my nix configs for my servers and desktop
nekomimi.pet
1/*https://hg.sr.ht/~dermetfan/seaweedfs-nixos/browse/seaweedfs.nix?rev=tip*/
2
3{ config, lib, pkgs, ... }:
4
5with lib;
6
7let
8 cfg = config.modules.seaweedfs;
9
10 clusterModule = cluster: {
11 options = {
12 package = mkOption {
13 type = types.package;
14 default = pkgs.seaweedfs;
15 };
16
17 security.grpc = let
18 auth = mkOption {
19 type = with types; nullOr (submodule {
20 options = {
21 cert = mkOption { type = path; };
22 key = mkOption { type = path; };
23 };
24 });
25 default = null;
26 };
27 in {
28 ca = mkOption {
29 type = with types; nullOr str;
30 default = null;
31 };
32
33 master = auth;
34 volume = auth;
35 filer = auth;
36 client = auth;
37 msgBroker = auth;
38 };
39
40 masters = mkOption {
41 type = with types; attrsOf (submodule (masterModule cluster.config));
42 default = {};
43 description = "SeaweedFS masters";
44 };
45
46 volumes = mkOption {
47 type = with types; attrsOf (submodule (volumeModule cluster.config));
48 default = {};
49 description = "SeaweedFS volumes";
50 };
51
52 filers = mkOption {
53 type = with types; attrsOf (submodule (filerModule cluster.config));
54 default = {};
55 description = "SeaweedFS filers";
56 };
57
58 webdavs = mkOption {
59 type = with types; attrsOf (submodule (webdavModule cluster.config));
60 default = {};
61 description = "SeaweedFS WebDAV servers";
62 };
63
64 instances = mkOption {
65 type = with types; attrsOf (submodule instanceModule);
66 description = "SeaweedFS instances";
67 default =
68 mapAttrs' (name: master: nameValuePair
69 "master-${name}"
70 {
71 inherit (master) cluster configs;
72
73 command = "master";
74
75 args = with master;
76 [
77 "-port=${toString port}"
78 "-volumeSizeLimitMB=${toString volumeSizeLimitMB}"
79 ] ++
80 optional (cpuprofile != "") "-cpuprofile=${cpuprofile}" ++
81 optional (defaultReplication != null) ("-defaultReplication=${defaultReplication.code}") ++
82 optional disableHttp "-disableHttp" ++
83 optional (garbageThreshold != "") "-garbageThreshold=${garbageThreshold}" ++
84 optional (ip != "") "-ip=${ip}" ++
85 optional (master."ip.bind" != "") "-ip.bind=${master."ip.bind"}" ++
86 optional (mdir != "") "-mdir=${mdir}" ++
87 optional (memprofile != "") "-memprofile=${memprofile}" ++
88 optional metrics.enable "-metrics.address=${metrics.address.text}" ++
89 optional (metrics.intervalSeconds != null) "-metrics.intervalSeconds=${toString metrics.intervalSeconds}" ++
90 optional (peers != []) ("-peers=" + (concatStringsSep "," (map (peer: peer.text) peers))) ++
91 optional resumeState "-resumeState" ++
92 optional volumePreallocate "-volumePreallocate" ++
93 optional (whiteList != []) ("-whiteList=" + (concatStringsSep "," whiteList));
94 }
95 ) cluster.config.masters //
96 mapAttrs' (name: volume: nameValuePair
97 "volume-${name}"
98 {
99 inherit (volume) cluster configs;
100
101 command = "volume";
102
103 args = with volume;
104 [
105 "-port=${toString port}"
106 "-dir=${concatStringsSep "," dir}"
107 "-fileSizeLimitMB=${toString fileSizeLimitMB}"
108 "-idleTimeout=${toString idleTimeout}"
109 "-index=${index}"
110 "-minFreeSpacePercent=${toString minFreeSpacePercent}"
111 "-preStopSeconds=${toString preStopSeconds}"
112 ] ++
113 optional (compactionMBps != null) ("-compactionMBps=${compactionMBps}") ++
114 optional (cpuprofile != "") "-cpuprofile=${cpuprofile}" ++
115 optional (dataCenter != "") "-dataCenter=${dataCenter}" ++
116 optional volume."images.fix.orientation" "-images.fix.orientation" ++
117 optional (ip != "") "-ip=${ip}" ++
118 optional (volume."ip.bind" != "") "-ip.bind=${volume."ip.bind"}" ++
119 optional (max != []) "-max=${concatStringsSep "," (map toString max)}" ++
120 optional (memprofile != "") "-memprofile=${memprofile}" ++
121 optional (metricsPort != null) "-metricsPort=${toString metricsPort}" ++
122 optional (mserver != []) ("-mserver=" + (concatStringsSep "," (map (mserver: mserver.text) mserver))) ++
123 optional (volume."port.public" != null) "-port.public=${toString volume."port.public"}" ++
124 optional pprof "-pprof" ++
125 optional (publicUrl != "") "-publicUrl=${publicUrl}" ++
126 optional (rack != "") "-rack=${rack}" ++
127 optional (!volume."read.redirect") "-read.redirect=false" ++
128 optional (whiteList != []) ("-whiteList=" + (concatStringsSep "," whiteList));
129
130 systemdService.preStart = "mkdir -p ${concatStringsSep " " volume.dir}";
131 }
132 ) cluster.config.volumes //
133 mapAttrs' (name: filer: nameValuePair
134 "filer-${name}"
135 {
136 inherit (filer) cluster configs;
137
138 command = "filer";
139
140 args = with filer;
141 [
142 "-port=${toString port}"
143 "-dirListLimit=${toString dirListLimit}"
144 "-maxMB=${toString maxMB}"
145 ] ++
146 optional (collection != "") "-collection=${collection}" ++
147 optional (dataCenter != "") "-dataCenter=${dataCenter}" ++
148 optional (defaultReplicaPlacement != null) ("-defaultReplicaPlacement=${defaultReplicaPlacement.code}") ++
149 optional disableDirListing "-disableDirListing" ++
150 optional disableHttp "-disableHttp" ++
151 optional encryptVolumeData "-encryptVolumeData" ++
152 optional (ip != "") "-ip=${ip}" ++
153 optional (filer."ip.bind" != "") "-ip.bind=${filer."ip.bind"}" ++
154 optional (master != []) ("-master=" + (concatStringsSep "," (map (master: master.text) master))) ++
155 optional (metricsPort != null) "-metricsPort=${toString metricsPort}" ++
156 optional (peers != []) ("-peers=" + (concatStringsSep "," (map (peer: peer.text) peers))) ++
157 optional (filer."port.readonly" != null) "-port.readonly=${toString filer."port.readonly"}" ++
158 optional (rack != "") "-rack=${rack}" ++
159 optionals s3.enable [
160 "-s3"
161 "-s3.port=${toString filer.s3.port}"
162 ] ++
163 optional (s3.enable && s3."cert.file" != "") "-s3.cert.file=${s3."cert.file"}" ++
164 optional (s3.enable && s3."key.file" != "") "-s3.key.file=${s3."key.file"}" ++
165 optional (s3.enable && s3.config != "") "-s3.config=${s3.config}" ++
166 optional (s3.enable && s3.domainName != []) "-s3.domainName=${concatStringsSep "," s3.domainName}";
167
168 systemdService.preStart = let
169 conf = filer.configs.filer.leveldb2 or {};
170 in optionalString (conf ? "dir") "mkdir -p ${conf.dir}";
171 }
172 ) cluster.config.filers //
173 mapAttrs' (name: webdav: nameValuePair
174 "webdav-${name}"
175 {
176 inherit (webdav) cluster;
177
178 command = "webdav";
179
180 args = with webdav;
181 [
182 "-port=${toString port}"
183 "-filer=${filer.text}"
184 "-cacheCapacityMB=${toString cacheCapacityMB}"
185 ] ++
186 optional (collection != "") "-collection=${collection}" ++
187 optional (cacheDir != "") "-cacheDir=${cacheDir}";
188 }
189 ) cluster.config.webdavs;
190 };
191 };
192 };
193
194 commonModule = cluster: common: {
195 options = {
196 cluster = mkOption {
197 type = types.submodule clusterModule;
198 internal = true;
199 };
200
201 openFirewall = mkEnableOption "open the firewall";
202 };
203
204 config = { inherit cluster; };
205 };
206
207 masterModule = cluster: master: {
208 imports = [ (commonModule cluster) ];
209
210 options = {
211 configs = mkOption {
212 type = with types; attrsOf attrs;
213 default.master.maintenance = {
214 scripts = ''
215 ec.encode -fullPercent=95 -quietFor=1h
216 ec.rebuild -force
217 ec.balance -force
218 volume.balance -force
219 volume.fix.replication
220 '';
221 sleep_minutes = 17;
222 };
223 };
224
225 cpuprofile = mkOption {
226 type = types.str;
227 default = "";
228 };
229
230 defaultReplication = mkOption {
231 type = types.submodule replicationModule;
232 default = {};
233 };
234
235 disableHttp = mkEnableOption "disable HTTP requests, gRPC only";
236
237 garbageThreshold = mkOption {
238 type = types.str;
239 default = "";
240 };
241
242 ip = mkOption {
243 type = types.str;
244 default = config.networking.hostName;
245 };
246
247 "ip.bind" = mkOption {
248 type = types.str;
249 default = "0.0.0.0";
250 };
251
252 mdir = mkOption {
253 type = types.str;
254 default = ".";
255 };
256
257 memprofile = mkOption {
258 type = types.str;
259 default = "";
260 };
261
262 metrics = {
263 enable = mkEnableOption "Prometheus";
264
265 address = mkOption {
266 type = types.submodule ipPortModule;
267 default = {};
268 };
269
270 intervalSeconds = mkOption {
271 type = types.ints.unsigned;
272 default = 15;
273 };
274 };
275
276 peers = mkOption {
277 type = peersType;
278 default = mapAttrsIpPort master.config.cluster.masters;
279 };
280
281 port = mkOption {
282 type = types.port;
283 default = 9333;
284 };
285
286 resumeState = mkEnableOption "resume previous state on master server";
287
288 volumePreallocate = mkEnableOption "preallocate disk space for volumes";
289
290 volumeSizeLimitMB = mkOption {
291 type = types.ints.unsigned;
292 default = 30000;
293 };
294
295 whiteList = mkOption {
296 type = with types; listOf str;
297 default = [];
298 };
299 };
300 };
301
302 volumeModule = cluster: volume: {
303 imports = [ (commonModule cluster) ];
304
305 options = {
306 configs = mkOption {
307 type = with types; attrsOf attrs;
308 default = {};
309 };
310
311 compactionMBps = mkOption {
312 type = with types; nullOr ints.unsigned;
313 default = null;
314 };
315
316 cpuprofile = mkOption {
317 type = types.str;
318 default = "";
319 };
320
321 dataCenter = mkOption {
322 type = types.str;
323 default = "";
324 };
325
326 dir = mkOption {
327 type = with types; listOf str;
328 default = [ "/var/lib/seaweedfs/${cluster._module.args.name}/volume-${volume.config._module.args.name}" ];
329 };
330
331 fileSizeLimitMB = mkOption {
332 type = types.ints.unsigned;
333 default = 256;
334 };
335
336 idleTimeout = mkOption{
337 type = types.ints.unsigned;
338 default = 30;
339 };
340
341 "images.fix.orientation" = mkEnableOption "adjustment of jpg orientation when uploading";
342
343 index = mkOption {
344 type = types.enum [
345 "memory"
346 "leveldb"
347 "leveldbMedium"
348 "leveldbLarge"
349 ];
350 default = "memory";
351 };
352
353 ip = mkOption {
354 type = types.str;
355 default = config.networking.hostName;
356 };
357
358 "ip.bind" = mkOption {
359 type = types.str;
360 default = "0.0.0.0";
361 };
362
363 max = mkOption {
364 type = with types; listOf ints.unsigned;
365 default = [ 8 ];
366 };
367
368 memprofile = mkOption {
369 type = types.str;
370 default = "";
371 };
372
373 metricsPort = mkOption {
374 type = with types; nullOr port;
375 default = null;
376 };
377
378 minFreeSpacePercent = mkOption {
379 type = types.ints.unsigned;
380 default = 1;
381 };
382
383 mserver = mkOption {
384 type = peersType;
385 default = mapAttrsIpPort volume.config.cluster.masters;
386 };
387
388 port = mkOption {
389 type = types.port;
390 default = 8080;
391 };
392
393 "port.public" = mkOption {
394 type = with types; nullOr port;
395 default = null;
396 };
397
398 pprof = mkEnableOption "pprof http handlers. precludes -memprofile and -cpuprofile";
399
400 preStopSeconds = mkOption {
401 type = types.int;
402 default = 10;
403 };
404
405 publicUrl = mkOption {
406 type = types.str;
407 default = "";
408 };
409
410 rack = mkOption {
411 type = types.str;
412 default = "";
413 };
414
415 "read.redirect" = mkOption {
416 type = types.bool;
417 default = true;
418 };
419
420 whiteList = mkOption {
421 type = with types; listOf str;
422 default = [];
423 };
424 };
425 };
426
427 filerModule = cluster: filer: {
428 imports = [ (commonModule cluster) ];
429
430 options = {
431 configs = mkOption {
432 type = with types; attrsOf attrs;
433 default.filer.leveldb2 = {
434 enabled = true;
435 dir = "/var/lib/seaweedfs/${cluster._module.args.name}/filer-${filer.config._module.args.name}/filerldb2";
436 };
437 };
438
439 collection = mkOption {
440 type = types.str;
441 default = "";
442 };
443
444 dataCenter = mkOption {
445 type = types.str;
446 default = "";
447 };
448
449 defaultReplicaPlacement = mkOption {
450 type = with types; nullOr (submodule replicationModule);
451 default = null;
452 };
453
454 dirListLimit = mkOption {
455 type = types.ints.unsigned;
456 default = 100000;
457 };
458
459 disableDirListing = mkEnableOption "turn off directory listing";
460
461 disableHttp = mkEnableOption "disable http request, only gRpc operations are allowed";
462
463 encryptVolumeData = mkEnableOption "encrypt data on volume servers";
464
465 ip = mkOption {
466 type = types.str;
467 default = config.networking.hostName;
468 };
469
470 "ip.bind" = mkOption {
471 type = types.str;
472 default = "0.0.0.0";
473 };
474
475 master = mkOption {
476 type = peersType;
477 default = mapAttrsIpPort filer.config.cluster.masters;
478 };
479
480 maxMB = mkOption {
481 type = types.ints.unsigned;
482 default = 32;
483 };
484
485 metricsPort = mkOption {
486 type = with types; nullOr port;
487 default = null;
488 };
489
490 peers = mkOption {
491 type = peersType;
492 default = mapAttrsIpPort filer.config.cluster.filers;
493 };
494
495 port = mkOption {
496 type = types.port;
497 default = 8888;
498 };
499
500 "port.readonly" = mkOption {
501 type = with types; nullOr port;
502 default = null;
503 };
504
505 rack = mkOption {
506 type = types.str;
507 default = "";
508 };
509
510 s3 = {
511 enable = mkEnableOption "whether to start S3 gateway";
512
513 "cert.file" = mkOption {
514 type = types.path;
515 default = "";
516 };
517
518 config = mkOption {
519 type = types.path;
520 default = "";
521 };
522
523 domainName = mkOption {
524 type = with types; listOf str;
525 default = [];
526 };
527
528 "key.file" = mkOption {
529 type = types.path;
530 default = "";
531 };
532
533 port = mkOption {
534 type = types.port;
535 default = 8333;
536 };
537 };
538 };
539 };
540
541 webdavModule = cluster: webdav: {
542 imports = [ (commonModule cluster) ];
543
544 options = {
545 cacheCapacityMB = mkOption {
546 type = types.int;
547 default = 1000;
548 };
549
550 cacheDir = mkOption {
551 type = types.str;
552 default = ".";
553 };
554
555 collection = mkOption {
556 type = types.str;
557 default = "";
558 };
559
560 filer = mkOption {
561 type = types.submodule ipPortModule;
562 default = {
563 ip = "127.0.0.1";
564 port = 8888;
565 };
566 };
567
568 port = mkOption {
569 type = types.port;
570 default = 7333;
571 };
572 };
573 };
574
575 instanceModule = instance: {
576 options = {
577 cluster = mkOption {
578 type = types.submodule clusterModule;
579 internal = true;
580 };
581
582 command = mkOption {
583 type = types.enum [
584 "server"
585 "master"
586 "volume"
587 "mount"
588 "filer"
589 "filer.replicate"
590 "filer.sync"
591 "s3"
592 "msgBroker"
593 "watch"
594 "webdav"
595 ];
596 };
597
598 logArgs = mkOption {
599 type = with types; listOf str;
600 default = [];
601 };
602
603 args = mkOption {
604 type = with types; listOf str;
605 default = [];
606 };
607
608 configs = mkOption {
609 type = with types; attrsOf attrs;
610 default = {};
611 };
612
613 package = mkOption {
614 type = types.package;
615 default = instance.config.cluster.package;
616 };
617
618 systemdService = mkOption {
619 type = types.attrs;
620 default = {};
621 };
622 };
623
624 config = {
625 logArgs = [ "-logtostderr" ];
626
627 systemdService.path = optional (instance.config.command == "mount") pkgs.fuse;
628 };
629 };
630
631 replicationModule = replication: {
632 options = {
633 dataCenter = mkOption {
634 type = types.ints.between 0 9;
635 default = 0;
636 };
637
638 rack = mkOption {
639 type = types.ints.between 0 9;
640 default = 0;
641 };
642
643 server = mkOption {
644 type = types.ints.between 0 9;
645 default = 0;
646 };
647
648 code = mkOption {
649 readOnly = true;
650 internal = true;
651 type = types.str;
652 default = with replication.config; "${toString dataCenter}${toString rack}${toString server}";
653 };
654 };
655 };
656
657 peersType = with types; listOf (submodule ipPortModule);
658
659 ipPortModule = ipPort: {
660 options = {
661 ip = mkOption {
662 type = types.str;
663 };
664
665 port = mkOption {
666 type = types.port;
667 };
668
669 text = mkOption {
670 internal = true;
671 readOnly = true;
672 type = types.str;
673 default = with ipPort.config; "${ip}:${toString port}";
674 };
675 };
676 };
677
678 mapAttrsIpPort = attrs: mapAttrsToList (name: value: { inherit (value) ip port; }) attrs;
679
680 toTOML = with generators; toINI {
681 mkKeyValue = mkKeyValueDefault {
682 mkValueString = v:
683 if isString v
684 then (
685 if hasInfix "\n" v
686 then ''
687 """
688 ${removeSuffix "\n" v}
689 """
690 ''
691 else ''"${v}"''
692 )
693 else mkValueStringDefault {} v;
694 } "=";
695 };
696
697 flattenAttrs = separator: attrs: let
698 /*
699 attrs = {
700 a = {
701 m1 = {};
702 m2 = {};
703 };
704 b = {
705 m1 = {};
706 };
707 }
708 */
709
710 /*
711 step1 = {
712 a = [
713 { name = "a-m1"; value = {}; }
714 { name = "a-m2"; value = {}; }
715 ];
716 b = [
717 { name = "b-m1"; value = {}; }
718 ];
719 };
720 */
721 step1 = mapAttrs (outerName: outerValues:
722 mapAttrsToList (innerName: innerValues: nameValuePair
723 "${outerName}${separator}${innerName}"
724 innerValues
725 ) outerValues
726 ) attrs;
727
728 /*
729 step2 = [
730 [
731 { name = "a-m1"; value = {}; }
732 { name = "a-m2"; value = {}; }
733 ]
734 [
735 { name = "b-m1"; value = {}; }
736 ]
737 ];
738 */
739 step2 = mapAttrsToList (name: value: value) step1;
740
741 /*
742 step3 = [
743 { name = "a-m1"; value = {}; }
744 { name = "a-m2"; value = {}; }
745 { name = "b-m1"; value = {}; }
746 ];
747 */
748 step3 = flatten step2;
749 in
750 /*
751 {
752 a-m1 = {};
753 a-m2 = {};
754 b-m1 = {};
755 };
756 */
757 builtins.listToAttrs step3;
758in {
759 options.modules.seaweedfs = {
760 clusters = mkOption {
761 type = with types; attrsOf (submodule clusterModule);
762 default = {};
763 description = "SeaweedFS clusters";
764 };
765 };
766
767 config = {
768 systemd.services = mapAttrs'
769 (name: instance: nameValuePair "seaweedfs-${name}" instance)
770 (flattenAttrs "-" (
771 mapAttrs (clusterName: cluster:
772 mapAttrs (instanceName: instance: with instance; recursiveUpdate systemdService rec {
773 description = "SeaweedFS ${clusterName} ${instanceName}";
774 wants = [ "network.target" ];
775 after = wants;
776 wantedBy = [ "multi-user.target" ];
777 preStart = with serviceConfig; ''
778 ${
779 let securityFile = config.environment.etc."seaweedfs/${clusterName}/security.toml";
780 in optionalString securityFile.enable "ln -s /etc/${securityFile.target} ${WorkingDirectory}/"
781 }
782
783 # TODO replace find usage with statically known condition
784 find -L /etc/${ConfigurationDirectory} -type f -exec ln -s '{}' ${WorkingDirectory}/ \;
785
786 ${optionalString (systemdService ? preStart) systemdService.preStart}
787 '';
788 serviceConfig = rec {
789 ExecStart = "${package}/bin/weed ${concatStringsSep " " logArgs} ${command} ${concatStringsSep " " args}";
790 Restart = "on-failure";
791 Type = "exec";
792 ConfigurationDirectory = "seaweedfs/${clusterName}/${instanceName}";
793 RuntimeDirectory = ConfigurationDirectory;
794 RuntimeDirectoryPreserve = "restart";
795 WorkingDirectory = "/run/${RuntimeDirectory}";
796 };
797 }) cluster.instances
798 ) cfg.clusters
799 ));
800
801 environment.etc =
802 (mapAttrs' (name: cluster:
803 let file = "seaweedfs/${name}/security.toml";
804 in nameValuePair file {
805 enable = config.environment.etc.${file}.text != "";
806 text = with cluster.security.grpc; toTOML (
807 (if ca == null then {} else { grpc.ca = ca; }) //
808 (if master == null then {} else { "grpc.master" = { inherit (master) cert key; }; }) //
809 (if volume == null then {} else { "grpc.volume" = { inherit (volume) cert key; }; }) //
810 (if filer == null then {} else { "grpc.filer" = { inherit (filer) cert key; }; }) //
811 (if client == null then {} else { "grpc.client" = { inherit (client) cert key; }; }) //
812 (if msgBroker == null then {} else { "grpc.msg_broker" = { inherit (msgBroker) cert key; }; })
813 );
814 }
815 ) cfg.clusters) //
816 (mapAttrs'
817 (name: config: nameValuePair
818 "seaweedfs/${name}.toml"
819 { text = toTOML config; }
820 )
821 (flattenAttrs "/" (
822 mapAttrs (clusterName: cluster:
823 flattenAttrs "/" (
824 mapAttrs
825 (instanceName: instance: instance.configs)
826 cluster.instances
827 )
828 ) cfg.clusters
829 ))
830 );
831
832 networking.firewall.allowedTCPPorts = let
833 modulesToPorts = extraPorts: mapAttrsToList (name: module:
834 with module;
835 optionals openFirewall (
836 [ port (port + 10000) ] ++
837 (filter (p: p != null) (extraPorts module))
838 )
839 );
840 in flatten (mapAttrsToList (clusterName: cluster:
841 modulesToPorts
842 (master: [])
843 cluster.masters ++
844
845 modulesToPorts
846 (volume: with volume; [ metricsPort volume."port.public" ])
847 cluster.volumes ++
848
849 modulesToPorts
850 (filer: with filer; [ metricsPort filer."port.readonly" s3.port])
851 cluster.filers ++
852
853 modulesToPorts
854 (webdav: [])
855 cluster.webdavs
856 ) cfg.clusters);
857 };
858}