pyrox.dev
1{ config, lib, pkgs, ...} :
2
3with lib;
4
5let
6 cfg = config.services.beegfs;
7
8 # functions for the generations of config files
9
10 configMgmtd = name: cfg: pkgs.writeText "mgmt-${name}.conf" ''
11 storeMgmtdDirectory = ${cfg.mgmtd.storeDir}
12 storeAllowFirstRunInit = false
13 connAuthFile = ${cfg.connAuthFile}
14 connPortShift = ${toString cfg.connPortShift}
15
16 ${cfg.mgmtd.extraConfig}
17 '';
18
19 configAdmon = name: cfg: pkgs.writeText "admon-${name}.conf" ''
20 sysMgmtdHost = ${cfg.mgmtdHost}
21 connAuthFile = ${cfg.connAuthFile}
22 connPortShift = ${toString cfg.connPortShift}
23
24 ${cfg.admon.extraConfig}
25 '';
26
27 configMeta = name: cfg: pkgs.writeText "meta-${name}.conf" ''
28 storeMetaDirectory = ${cfg.meta.storeDir}
29 sysMgmtdHost = ${cfg.mgmtdHost}
30 connAuthFile = ${cfg.connAuthFile}
31 connPortShift = ${toString cfg.connPortShift}
32 storeAllowFirstRunInit = false
33
34 ${cfg.meta.extraConfig}
35 '';
36
37 configStorage = name: cfg: pkgs.writeText "storage-${name}.conf" ''
38 storeStorageDirectory = ${cfg.storage.storeDir}
39 sysMgmtdHost = ${cfg.mgmtdHost}
40 connAuthFile = ${cfg.connAuthFile}
41 connPortShift = ${toString cfg.connPortShift}
42 storeAllowFirstRunInit = false
43
44 ${cfg.storage.extraConfig}
45 '';
46
47 configHelperd = name: cfg: pkgs.writeText "helperd-${name}.conf" ''
48 connAuthFile = ${cfg.connAuthFile}
49 ${cfg.helperd.extraConfig}
50 '';
51
52 configClientFilename = name : "/etc/beegfs/client-${name}.conf";
53
54 configClient = name: cfg: ''
55 sysMgmtdHost = ${cfg.mgmtdHost}
56 connAuthFile = ${cfg.connAuthFile}
57 connPortShift = ${toString cfg.connPortShift}
58
59 ${cfg.client.extraConfig}
60 '';
61
62 serviceList = [
63 { service = "admon"; cfgFile = configAdmon; }
64 { service = "meta"; cfgFile = configMeta; }
65 { service = "mgmtd"; cfgFile = configMgmtd; }
66 { service = "storage"; cfgFile = configStorage; }
67 ];
68
69 # functions to generate systemd.service entries
70
71 systemdEntry = service: cfgFile: (mapAttrs' ( name: cfg:
72 (nameValuePair "beegfs-${service}-${name}" (mkIf cfg."${service}".enable {
73 wantedBy = [ "multi-user.target" ];
74 requires = [ "network-online.target" ];
75 after = [ "network-online.target" ];
76 serviceConfig = rec {
77 ExecStart = ''
78 ${pkgs.beegfs}/bin/beegfs-${service} \
79 cfgFile=${cfgFile name cfg} \
80 pidFile=${PIDFile}
81 '';
82 PIDFile = "/run/beegfs-${service}-${name}.pid";
83 TimeoutStopSec = "300";
84 };
85 }))) cfg);
86
87 systemdHelperd = mapAttrs' ( name: cfg:
88 (nameValuePair "beegfs-helperd-${name}" (mkIf cfg.client.enable {
89 wantedBy = [ "multi-user.target" ];
90 requires = [ "network-online.target" ];
91 after = [ "network-online.target" ];
92 serviceConfig = rec {
93 ExecStart = ''
94 ${pkgs.beegfs}/bin/beegfs-helperd \
95 cfgFile=${configHelperd name cfg} \
96 pidFile=${PIDFile}
97 '';
98 PIDFile = "/run/beegfs-helperd-${name}.pid";
99 TimeoutStopSec = "300";
100 };
101 }))) cfg;
102
103 # wrappers to beegfs tools. Avoid typing path of config files
104 utilWrappers = mapAttrsToList ( name: cfg:
105 ( pkgs.runCommand "beegfs-utils-${name}" { nativeBuildInputs = [ pkgs.makeWrapper ]; } ''
106 mkdir -p $out/bin
107
108 makeWrapper ${pkgs.beegfs}/bin/beegfs-check-servers \
109 $out/bin/beegfs-check-servers-${name} \
110 --add-flags "-c ${configClientFilename name}" \
111 --prefix PATH : ${lib.makeBinPath [ pkgs.beegfs ]}
112
113 makeWrapper ${pkgs.beegfs}/bin/beegfs-ctl \
114 $out/bin/beegfs-ctl-${name} \
115 --add-flags "--cfgFile=${configClientFilename name}"
116
117 makeWrapper ${pkgs.beegfs}/bin/beegfs-ctl \
118 $out/bin/beegfs-df-${name} \
119 --add-flags "--cfgFile=${configClientFilename name}" \
120 --add-flags --listtargets \
121 --add-flags --hidenodeid \
122 --add-flags --pools \
123 --add-flags --spaceinfo
124
125 makeWrapper ${pkgs.beegfs}/bin/beegfs-fsck \
126 $out/bin/beegfs-fsck-${name} \
127 --add-flags "--cfgFile=${configClientFilename name}"
128 ''
129 )) cfg;
130in
131{
132 ###### interface
133
134 options = {
135 services.beegfsEnable = mkEnableOption "BeeGFS";
136
137 services.beegfs = mkOption {
138 default = {};
139 description = ''
140 BeeGFS configurations. Every mount point requires a separate configuration.
141 '';
142 type = with types; attrsOf (submodule ({ ... } : {
143 options = {
144 mgmtdHost = mkOption {
145 type = types.str;
146 default = null;
147 example = "master";
148 description = ''Hostname of managament host.'';
149 };
150
151 connAuthFile = mkOption {
152 type = types.str;
153 default = "";
154 example = "/etc/my.key";
155 description = "File containing shared secret authentication.";
156 };
157
158 connPortShift = mkOption {
159 type = types.int;
160 default = 0;
161 example = 5;
162 description = ''
163 For each additional beegfs configuration shift all
164 service TCP/UDP ports by at least 5.
165 '';
166 };
167
168 client = {
169 enable = mkEnableOption "BeeGFS client";
170
171 mount = mkOption {
172 type = types.bool;
173 default = true;
174 description = "Create fstab entry automatically";
175 };
176
177 mountPoint = mkOption {
178 type = types.str;
179 default = "/run/beegfs";
180 description = ''
181 Mount point under which the beegfs filesytem should be mounted.
182 If mounted manually the mount option specifing the config file is needed:
183 cfgFile=/etc/beegfs/beegfs-client-<name>.conf
184 '';
185 };
186
187 extraConfig = mkOption {
188 type = types.lines;
189 default = "";
190 description = ''
191 Additional lines for beegfs-client.conf.
192 See documentation for further details.
193 '';
194 };
195 };
196
197 helperd = {
198 enable = mkOption {
199 type = types.bool;
200 default = true;
201 description = ''
202 Enable the BeeGFS helperd.
203 The helpered is need for logging purposes on the client.
204 Disabling <literal>helperd</literal> allows for runing the client
205 with <literal>allowUnfree = false</literal>.
206 '';
207 };
208
209 extraConfig = mkOption {
210 type = types.lines;
211 default = "";
212 description = ''
213 Additional lines for beegfs-helperd.conf. See documentation
214 for further details.
215 '';
216 };
217 };
218
219 mgmtd = {
220 enable = mkEnableOption "BeeGFS mgmtd daemon";
221
222 storeDir = mkOption {
223 type = types.path;
224 default = null;
225 example = "/data/beegfs-mgmtd";
226 description = ''
227 Data directory for mgmtd.
228 Must not be shared with other beegfs daemons.
229 This directory must exist and it must be initialized
230 with beegfs-setup-mgmtd, e.g. "beegfs-setup-mgmtd -C -p <storeDir>"
231 '';
232 };
233
234 extraConfig = mkOption {
235 type = types.lines;
236 default = "";
237 description = ''
238 Additional lines for beegfs-mgmtd.conf. See documentation
239 for further details.
240 '';
241 };
242 };
243
244 admon = {
245 enable = mkEnableOption "BeeGFS admon daemon";
246
247 extraConfig = mkOption {
248 type = types.lines;
249 default = "";
250 description = ''
251 Additional lines for beegfs-admon.conf. See documentation
252 for further details.
253 '';
254 };
255 };
256
257 meta = {
258 enable = mkEnableOption "BeeGFS meta data daemon";
259
260 storeDir = mkOption {
261 type = types.path;
262 default = null;
263 example = "/data/beegfs-meta";
264 description = ''
265 Data directory for meta data service.
266 Must not be shared with other beegfs daemons.
267 The underlying filesystem must be mounted with xattr turned on.
268 This directory must exist and it must be initialized
269 with beegfs-setup-meta, e.g.
270 "beegfs-setup-meta -C -s <serviceID> -p <storeDir>"
271 '';
272 };
273
274 extraConfig = mkOption {
275 type = types.str;
276 default = "";
277 description = ''
278 Additional lines for beegfs-meta.conf. See documentation
279 for further details.
280 '';
281 };
282 };
283
284 storage = {
285 enable = mkEnableOption "BeeGFS storage daemon";
286
287 storeDir = mkOption {
288 type = types.path;
289 default = null;
290 example = "/data/beegfs-storage";
291 description = ''
292 Data directories for storage service.
293 Must not be shared with other beegfs daemons.
294 The underlying filesystem must be mounted with xattr turned on.
295 This directory must exist and it must be initialized
296 with beegfs-setup-storage, e.g.
297 "beegfs-setup-storage -C -s <serviceID> -i <storageTargetID> -p <storeDir>"
298 '';
299 };
300
301 extraConfig = mkOption {
302 type = types.str;
303 default = "";
304 description = ''
305 Addional lines for beegfs-storage.conf. See documentation
306 for further details.
307 '';
308 };
309 };
310 };
311 }));
312 };
313 };
314
315 ###### implementation
316
317 config =
318 mkIf config.services.beegfsEnable {
319
320 environment.systemPackages = utilWrappers;
321
322 # Put the client.conf files in /etc since they are needed
323 # by the commandline tools
324 environment.etc = mapAttrs' ( name: cfg:
325 (nameValuePair "beegfs/client-${name}.conf" (mkIf (cfg.client.enable)
326 {
327 enable = true;
328 text = configClient name cfg;
329 }))) cfg;
330
331 # Kernel module, we need it only once per host.
332 boot = mkIf (
333 foldr (a: b: a || b) false
334 (map (x: x.client.enable) (collect (x: x ? client) cfg)))
335 {
336 kernelModules = [ "beegfs" ];
337 extraModulePackages = [ pkgs.linuxPackages.beegfs-module ];
338 };
339
340 # generate fstab entries
341 fileSystems = mapAttrs' (name: cfg:
342 (nameValuePair cfg.client.mountPoint (optionalAttrs cfg.client.mount (mkIf cfg.client.enable {
343 device = "beegfs_nodev";
344 fsType = "beegfs";
345 mountPoint = cfg.client.mountPoint;
346 options = [ "cfgFile=${configClientFilename name}" "_netdev" ];
347 })))) cfg;
348
349 # generate systemd services
350 systemd.services = systemdHelperd //
351 foldr (a: b: a // b) {}
352 (map (x: systemdEntry x.service x.cfgFile) serviceList);
353 };
354}