pyrox.dev
1{
2 config,
3 lib,
4 pkgs,
5 ...
6}:
7let
8 cfg = config.services.ceph;
9
10 # function that translates "camelCaseOptions" to "camel case options", credits to tilpner in #nixos@freenode
11 expandCamelCase = lib.replaceStrings lib.upperChars (map (s: " ${s}") lib.lowerChars);
12 expandCamelCaseAttrs = lib.mapAttrs' (name: value: lib.nameValuePair (expandCamelCase name) value);
13
14 makeServices =
15 daemonType: daemonIds:
16 lib.mkMerge (
17 map (daemonId: {
18 "ceph-${daemonType}-${daemonId}" =
19 makeService daemonType daemonId cfg.global.clusterName
20 cfg.${daemonType}.package;
21 }) daemonIds
22 );
23
24 makeService =
25 daemonType: daemonId: clusterName: ceph:
26 let
27 stateDirectory = "ceph/${
28 if daemonType == "rgw" then "radosgw" else daemonType
29 }/${clusterName}-${daemonId}";
30 in
31 {
32 enable = true;
33 description = "Ceph ${
34 builtins.replaceStrings lib.lowerChars lib.upperChars daemonType
35 } daemon ${daemonId}";
36 after = [
37 "network-online.target"
38 "time-sync.target"
39 ] ++ lib.optional (daemonType == "osd") "ceph-mon.target";
40 wants = [
41 "network-online.target"
42 "time-sync.target"
43 ];
44 partOf = [ "ceph-${daemonType}.target" ];
45 wantedBy = [ "ceph-${daemonType}.target" ];
46
47 path = [ pkgs.getopt ];
48
49 # Don't start services that are not yet initialized
50 unitConfig.ConditionPathExists = "/var/lib/${stateDirectory}/keyring";
51 startLimitBurst =
52 if daemonType == "osd" then
53 30
54 else if
55 lib.elem daemonType [
56 "mgr"
57 "mds"
58 ]
59 then
60 3
61 else
62 5;
63 startLimitIntervalSec = 60 * 30; # 30 mins
64
65 serviceConfig =
66 {
67 LimitNOFILE = 1048576;
68 LimitNPROC = 1048576;
69 Environment = "CLUSTER=${clusterName}";
70 ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
71 PrivateDevices = "yes";
72 PrivateTmp = "true";
73 ProtectHome = "true";
74 ProtectSystem = "full";
75 Restart = "on-failure";
76 StateDirectory = stateDirectory;
77 User = "ceph";
78 Group = if daemonType == "osd" then "disk" else "ceph";
79 ExecStart = ''
80 ${ceph.out}/bin/${if daemonType == "rgw" then "radosgw" else "ceph-${daemonType}"} \
81 -f --cluster ${clusterName} --id ${daemonId}'';
82 }
83 // lib.optionalAttrs (daemonType == "osd") {
84 ExecStartPre = "${ceph.lib}/libexec/ceph/ceph-osd-prestart.sh --id ${daemonId} --cluster ${clusterName}";
85 RestartSec = "20s";
86 PrivateDevices = "no"; # osd needs disk access
87 }
88 // lib.optionalAttrs (daemonType == "mon") {
89 RestartSec = "10";
90 };
91 };
92
93 makeTarget = daemonType: {
94 "ceph-${daemonType}" = {
95 description = "Ceph target allowing to start/stop all ceph-${daemonType} services at once";
96 partOf = [ "ceph.target" ];
97 wantedBy = [ "ceph.target" ];
98 before = [ "ceph.target" ];
99 unitConfig.StopWhenUnneeded = true;
100 };
101 };
102in
103{
104 options.services.ceph = {
105 # Ceph has a monolithic configuration file but different sections for
106 # each daemon, a separate client section and a global section
107 enable = lib.mkEnableOption "Ceph global configuration";
108
109 global = {
110 fsid = lib.mkOption {
111 type = lib.types.str;
112 example = ''
113 433a2193-4f8a-47a0-95d2-209d7ca2cca5
114 '';
115 description = ''
116 Filesystem ID, a generated uuid, its must be generated and set before
117 attempting to start a cluster
118 '';
119 };
120
121 clusterName = lib.mkOption {
122 type = lib.types.str;
123 default = "ceph";
124 description = ''
125 Name of cluster
126 '';
127 };
128
129 mgrModulePath = lib.mkOption {
130 type = lib.types.path;
131 default = "${pkgs.ceph.lib}/lib/ceph/mgr";
132 defaultText = lib.literalExpression ''"''${pkgs.ceph.lib}/lib/ceph/mgr"'';
133 description = ''
134 Path at which to find ceph-mgr modules.
135 '';
136 };
137
138 monInitialMembers = lib.mkOption {
139 type = with lib.types; nullOr commas;
140 default = null;
141 example = ''
142 node0, node1, node2
143 '';
144 description = ''
145 List of hosts that will be used as monitors at startup.
146 '';
147 };
148
149 monHost = lib.mkOption {
150 type = with lib.types; nullOr commas;
151 default = null;
152 example = ''
153 10.10.0.1, 10.10.0.2, 10.10.0.3
154 '';
155 description = ''
156 List of hostname shortnames/IP addresses of the initial monitors.
157 '';
158 };
159
160 maxOpenFiles = lib.mkOption {
161 type = lib.types.int;
162 default = 131072;
163 description = ''
164 Max open files for each OSD daemon.
165 '';
166 };
167
168 authClusterRequired = lib.mkOption {
169 type = lib.types.enum [
170 "cephx"
171 "none"
172 ];
173 default = "cephx";
174 description = ''
175 Enables requiring daemons to authenticate with eachother in the cluster.
176 '';
177 };
178
179 authServiceRequired = lib.mkOption {
180 type = lib.types.enum [
181 "cephx"
182 "none"
183 ];
184 default = "cephx";
185 description = ''
186 Enables requiring clients to authenticate with the cluster to access services in the cluster (e.g. radosgw, mds or osd).
187 '';
188 };
189
190 authClientRequired = lib.mkOption {
191 type = lib.types.enum [
192 "cephx"
193 "none"
194 ];
195 default = "cephx";
196 description = ''
197 Enables requiring the cluster to authenticate itself to the client.
198 '';
199 };
200
201 publicNetwork = lib.mkOption {
202 type = with lib.types; nullOr commas;
203 default = null;
204 example = ''
205 10.20.0.0/24, 192.168.1.0/24
206 '';
207 description = ''
208 A comma-separated list of subnets that will be used as public networks in the cluster.
209 '';
210 };
211
212 clusterNetwork = lib.mkOption {
213 type = with lib.types; nullOr commas;
214 default = null;
215 example = ''
216 10.10.0.0/24, 192.168.0.0/24
217 '';
218 description = ''
219 A comma-separated list of subnets that will be used as cluster networks in the cluster.
220 '';
221 };
222
223 rgwMimeTypesFile = lib.mkOption {
224 type = with lib.types; nullOr path;
225 default = "${pkgs.mailcap}/etc/mime.types";
226 defaultText = lib.literalExpression ''"''${pkgs.mailcap}/etc/mime.types"'';
227 description = ''
228 Path to mime types used by radosgw.
229 '';
230 };
231 };
232
233 extraConfig = lib.mkOption {
234 type = with lib.types; attrsOf str;
235 default = { };
236 example = {
237 "ms bind ipv6" = "true";
238 };
239 description = ''
240 Extra configuration to add to the global section. Use for setting values that are common for all daemons in the cluster.
241 '';
242 };
243
244 mgr = {
245 enable = lib.mkEnableOption "Ceph MGR daemon";
246 daemons = lib.mkOption {
247 type = with lib.types; listOf str;
248 default = [ ];
249 example = [
250 "name1"
251 "name2"
252 ];
253 description = ''
254 A list of names for manager daemons that should have a service created. The names correspond
255 to the id part in ceph i.e. [ "name1" ] would result in mgr.name1
256 '';
257 };
258 package = lib.mkPackageOption pkgs "ceph" { };
259 extraConfig = lib.mkOption {
260 type = with lib.types; attrsOf str;
261 default = { };
262 description = ''
263 Extra configuration to add to the global section for manager daemons.
264 '';
265 };
266 };
267
268 mon = {
269 enable = lib.mkEnableOption "Ceph MON daemon";
270 daemons = lib.mkOption {
271 type = with lib.types; listOf str;
272 default = [ ];
273 example = [
274 "name1"
275 "name2"
276 ];
277 description = ''
278 A list of monitor daemons that should have a service created. The names correspond
279 to the id part in ceph i.e. [ "name1" ] would result in mon.name1
280 '';
281 };
282 package = lib.mkPackageOption pkgs "ceph" { };
283 extraConfig = lib.mkOption {
284 type = with lib.types; attrsOf str;
285 default = { };
286 description = ''
287 Extra configuration to add to the monitor section.
288 '';
289 };
290 };
291
292 osd = {
293 enable = lib.mkEnableOption "Ceph OSD daemon";
294 daemons = lib.mkOption {
295 type = with lib.types; listOf str;
296 default = [ ];
297 example = [
298 "name1"
299 "name2"
300 ];
301 description = ''
302 A list of OSD daemons that should have a service created. The names correspond
303 to the id part in ceph i.e. [ "name1" ] would result in osd.name1
304 '';
305 };
306 package = lib.mkPackageOption pkgs "ceph" { };
307 extraConfig = lib.mkOption {
308 type = with lib.types; attrsOf str;
309 default = {
310 "osd journal size" = "10000";
311 "osd pool default size" = "3";
312 "osd pool default min size" = "2";
313 "osd pool default pg num" = "200";
314 "osd pool default pgp num" = "200";
315 "osd crush chooseleaf type" = "1";
316 };
317 description = ''
318 Extra configuration to add to the OSD section.
319 '';
320 };
321 };
322
323 mds = {
324 enable = lib.mkEnableOption "Ceph MDS daemon";
325 daemons = lib.mkOption {
326 type = with lib.types; listOf str;
327 default = [ ];
328 example = [
329 "name1"
330 "name2"
331 ];
332 description = ''
333 A list of metadata service daemons that should have a service created. The names correspond
334 to the id part in ceph i.e. [ "name1" ] would result in mds.name1
335 '';
336 };
337 package = lib.mkPackageOption pkgs "ceph" { };
338 extraConfig = lib.mkOption {
339 type = with lib.types; attrsOf str;
340 default = { };
341 description = ''
342 Extra configuration to add to the MDS section.
343 '';
344 };
345 };
346
347 rgw = {
348 enable = lib.mkEnableOption "Ceph RadosGW daemon";
349 package = lib.mkPackageOption pkgs "ceph" { };
350 daemons = lib.mkOption {
351 type = with lib.types; listOf str;
352 default = [ ];
353 example = [
354 "name1"
355 "name2"
356 ];
357 description = ''
358 A list of rados gateway daemons that should have a service created. The names correspond
359 to the id part in ceph i.e. [ "name1" ] would result in client.name1, radosgw daemons
360 aren't daemons to cluster in the sense that OSD, MGR or MON daemons are. They are simply
361 daemons, from ceph, that uses the cluster as a backend.
362 '';
363 };
364 };
365
366 client = {
367 enable = lib.mkEnableOption "Ceph client configuration";
368 extraConfig = lib.mkOption {
369 type = with lib.types; attrsOf (attrsOf str);
370 default = { };
371 example = lib.literalExpression ''
372 {
373 # This would create a section for a radosgw daemon named node0 and related
374 # configuration for it
375 "client.radosgw.node0" = { "some config option" = "true"; };
376 };
377 '';
378 description = ''
379 Extra configuration to add to the client section. Configuration for rados gateways
380 would be added here, with their own sections, see example.
381 '';
382 };
383 };
384 };
385
386 config = lib.mkIf config.services.ceph.enable {
387 assertions = [
388 {
389 assertion = cfg.global.fsid != "";
390 message = "fsid has to be set to a valid uuid for the cluster to function";
391 }
392 {
393 assertion = cfg.mon.enable -> cfg.mon.daemons != [ ];
394 message = "have to set id of atleast one MON if you're going to enable Monitor";
395 }
396 {
397 assertion = cfg.mds.enable -> cfg.mds.daemons != [ ];
398 message = "have to set id of atleast one MDS if you're going to enable Metadata Service";
399 }
400 {
401 assertion = cfg.osd.enable -> cfg.osd.daemons != [ ];
402 message = "have to set id of atleast one OSD if you're going to enable OSD";
403 }
404 {
405 assertion = cfg.mgr.enable -> cfg.mgr.daemons != [ ];
406 message = "have to set id of atleast one MGR if you're going to enable MGR";
407 }
408 ];
409
410 warnings =
411 lib.optional (cfg.global.monInitialMembers == null)
412 "Not setting up a list of members in monInitialMembers requires that you set the host variable for each mon daemon or else the cluster won't function";
413
414 environment.etc."ceph/ceph.conf".text =
415 let
416 # Merge the extraConfig set for mgr daemons, as mgr don't have their own section
417 globalSection = expandCamelCaseAttrs (
418 cfg.global // cfg.extraConfig // lib.optionalAttrs cfg.mgr.enable cfg.mgr.extraConfig
419 );
420 # Remove all name-value pairs with null values from the attribute set to avoid making empty sections in the ceph.conf
421 globalSection' = lib.filterAttrs (name: value: value != null) globalSection;
422 totalConfig =
423 {
424 global = globalSection';
425 }
426 // lib.optionalAttrs (cfg.mon.enable && cfg.mon.extraConfig != { }) { mon = cfg.mon.extraConfig; }
427 // lib.optionalAttrs (cfg.mds.enable && cfg.mds.extraConfig != { }) { mds = cfg.mds.extraConfig; }
428 // lib.optionalAttrs (cfg.osd.enable && cfg.osd.extraConfig != { }) { osd = cfg.osd.extraConfig; }
429 // lib.optionalAttrs (cfg.client.enable && cfg.client.extraConfig != { }) cfg.client.extraConfig;
430 in
431 lib.generators.toINI { } totalConfig;
432
433 users.users.ceph = {
434 uid = config.ids.uids.ceph;
435 description = "Ceph daemon user";
436 group = "ceph";
437 extraGroups = [ "disk" ];
438 };
439
440 users.groups.ceph = {
441 gid = config.ids.gids.ceph;
442 };
443
444 systemd.services =
445 let
446 services =
447 [ ]
448 ++ lib.optional cfg.mon.enable (makeServices "mon" cfg.mon.daemons)
449 ++ lib.optional cfg.mds.enable (makeServices "mds" cfg.mds.daemons)
450 ++ lib.optional cfg.osd.enable (makeServices "osd" cfg.osd.daemons)
451 ++ lib.optional cfg.rgw.enable (makeServices "rgw" cfg.rgw.daemons)
452 ++ lib.optional cfg.mgr.enable (makeServices "mgr" cfg.mgr.daemons);
453 in
454 lib.mkMerge services;
455
456 systemd.targets =
457 let
458 targets =
459 [
460 {
461 ceph = {
462 description = "Ceph target allowing to start/stop all ceph service instances at once";
463 wantedBy = [ "multi-user.target" ];
464 unitConfig.StopWhenUnneeded = true;
465 };
466 }
467 ]
468 ++ lib.optional cfg.mon.enable (makeTarget "mon")
469 ++ lib.optional cfg.mds.enable (makeTarget "mds")
470 ++ lib.optional cfg.osd.enable (makeTarget "osd")
471 ++ lib.optional cfg.rgw.enable (makeTarget "rgw")
472 ++ lib.optional cfg.mgr.enable (makeTarget "mgr");
473 in
474 lib.mkMerge targets;
475
476 systemd.tmpfiles.settings."10-ceph" =
477 let
478 defaultConfig = {
479 user = "ceph";
480 group = "ceph";
481 };
482 in
483 {
484 "/etc/ceph".d = defaultConfig;
485 "/run/ceph".d = defaultConfig // {
486 mode = "0770";
487 };
488 "/var/lib/ceph".d = defaultConfig;
489 "/var/lib/ceph/mgr".d = lib.mkIf (cfg.mgr.enable) defaultConfig;
490 "/var/lib/ceph/mon".d = lib.mkIf (cfg.mon.enable) defaultConfig;
491 "/var/lib/ceph/osd".d = lib.mkIf (cfg.osd.enable) defaultConfig;
492 };
493 };
494}