pyrox.dev
1{ config, lib, pkgs, ... }:
2
3with lib;
4
5let
6 cfg = config.services.ceph;
7
8 # function that translates "camelCaseOptions" to "camel case options", credits to tilpner in #nixos@freenode
9 expandCamelCase = replaceStrings upperChars (map (s: " ${s}") lowerChars);
10 expandCamelCaseAttrs = mapAttrs' (name: value: nameValuePair (expandCamelCase name) value);
11
12 makeServices = (daemonType: daemonIds:
13 mkMerge (map (daemonId:
14 { "ceph-${daemonType}-${daemonId}" = makeService daemonType daemonId cfg.global.clusterName pkgs.ceph; })
15 daemonIds));
16
17 makeService = (daemonType: daemonId: clusterName: ceph:
18 let
19 stateDirectory = "ceph/${if daemonType == "rgw" then "radosgw" else daemonType}/${clusterName}-${daemonId}"; in {
20 enable = true;
21 description = "Ceph ${builtins.replaceStrings lowerChars upperChars daemonType} daemon ${daemonId}";
22 after = [ "network-online.target" "time-sync.target" ] ++ optional (daemonType == "osd") "ceph-mon.target";
23 wants = [ "network-online.target" "time-sync.target" ];
24 partOf = [ "ceph-${daemonType}.target" ];
25 wantedBy = [ "ceph-${daemonType}.target" ];
26
27 path = [ pkgs.getopt ];
28
29 # Don't start services that are not yet initialized
30 unitConfig.ConditionPathExists = "/var/lib/${stateDirectory}/keyring";
31 startLimitBurst =
32 if daemonType == "osd" then 30 else if lib.elem daemonType ["mgr" "mds"] then 3 else 5;
33 startLimitIntervalSec = 60 * 30; # 30 mins
34
35 serviceConfig = {
36 LimitNOFILE = 1048576;
37 LimitNPROC = 1048576;
38 Environment = "CLUSTER=${clusterName}";
39 ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
40 PrivateDevices = "yes";
41 PrivateTmp = "true";
42 ProtectHome = "true";
43 ProtectSystem = "full";
44 Restart = "on-failure";
45 StateDirectory = stateDirectory;
46 User = "ceph";
47 Group = if daemonType == "osd" then "disk" else "ceph";
48 ExecStart = ''${ceph.out}/bin/${if daemonType == "rgw" then "radosgw" else "ceph-${daemonType}"} \
49 -f --cluster ${clusterName} --id ${daemonId}'';
50 } // optionalAttrs (daemonType == "osd") {
51 ExecStartPre = "${ceph.lib}/libexec/ceph/ceph-osd-prestart.sh --id ${daemonId} --cluster ${clusterName}";
52 RestartSec = "20s";
53 PrivateDevices = "no"; # osd needs disk access
54 } // optionalAttrs ( daemonType == "mon") {
55 RestartSec = "10";
56 };
57 });
58
59 makeTarget = (daemonType:
60 {
61 "ceph-${daemonType}" = {
62 description = "Ceph target allowing to start/stop all ceph-${daemonType} services at once";
63 partOf = [ "ceph.target" ];
64 wantedBy = [ "ceph.target" ];
65 before = [ "ceph.target" ];
66 unitConfig.StopWhenUnneeded = true;
67 };
68 }
69 );
70in
71{
72 options.services.ceph = {
73 # Ceph has a monolithic configuration file but different sections for
74 # each daemon, a separate client section and a global section
75 enable = mkEnableOption "Ceph global configuration";
76
77 global = {
78 fsid = mkOption {
79 type = types.str;
80 example = ''
81 433a2193-4f8a-47a0-95d2-209d7ca2cca5
82 '';
83 description = ''
84 Filesystem ID, a generated uuid, its must be generated and set before
85 attempting to start a cluster
86 '';
87 };
88
89 clusterName = mkOption {
90 type = types.str;
91 default = "ceph";
92 description = ''
93 Name of cluster
94 '';
95 };
96
97 mgrModulePath = mkOption {
98 type = types.path;
99 default = "${pkgs.ceph.lib}/lib/ceph/mgr";
100 description = ''
101 Path at which to find ceph-mgr modules.
102 '';
103 };
104
105 monInitialMembers = mkOption {
106 type = with types; nullOr commas;
107 default = null;
108 example = ''
109 node0, node1, node2
110 '';
111 description = ''
112 List of hosts that will be used as monitors at startup.
113 '';
114 };
115
116 monHost = mkOption {
117 type = with types; nullOr commas;
118 default = null;
119 example = ''
120 10.10.0.1, 10.10.0.2, 10.10.0.3
121 '';
122 description = ''
123 List of hostname shortnames/IP addresses of the initial monitors.
124 '';
125 };
126
127 maxOpenFiles = mkOption {
128 type = types.int;
129 default = 131072;
130 description = ''
131 Max open files for each OSD daemon.
132 '';
133 };
134
135 authClusterRequired = mkOption {
136 type = types.enum [ "cephx" "none" ];
137 default = "cephx";
138 description = ''
139 Enables requiring daemons to authenticate with eachother in the cluster.
140 '';
141 };
142
143 authServiceRequired = mkOption {
144 type = types.enum [ "cephx" "none" ];
145 default = "cephx";
146 description = ''
147 Enables requiring clients to authenticate with the cluster to access services in the cluster (e.g. radosgw, mds or osd).
148 '';
149 };
150
151 authClientRequired = mkOption {
152 type = types.enum [ "cephx" "none" ];
153 default = "cephx";
154 description = ''
155 Enables requiring the cluster to authenticate itself to the client.
156 '';
157 };
158
159 publicNetwork = mkOption {
160 type = with types; nullOr commas;
161 default = null;
162 example = ''
163 10.20.0.0/24, 192.168.1.0/24
164 '';
165 description = ''
166 A comma-separated list of subnets that will be used as public networks in the cluster.
167 '';
168 };
169
170 clusterNetwork = mkOption {
171 type = with types; nullOr commas;
172 default = null;
173 example = ''
174 10.10.0.0/24, 192.168.0.0/24
175 '';
176 description = ''
177 A comma-separated list of subnets that will be used as cluster networks in the cluster.
178 '';
179 };
180
181 rgwMimeTypesFile = mkOption {
182 type = with types; nullOr path;
183 default = "${pkgs.mime-types}/etc/mime.types";
184 description = ''
185 Path to mime types used by radosgw.
186 '';
187 };
188 };
189
190 extraConfig = mkOption {
191 type = with types; attrsOf str;
192 default = {};
193 example = ''
194 {
195 "ms bind ipv6" = "true";
196 };
197 '';
198 description = ''
199 Extra configuration to add to the global section. Use for setting values that are common for all daemons in the cluster.
200 '';
201 };
202
203 mgr = {
204 enable = mkEnableOption "Ceph MGR daemon";
205 daemons = mkOption {
206 type = with types; listOf str;
207 default = [];
208 example = ''
209 [ "name1" "name2" ];
210 '';
211 description = ''
212 A list of names for manager daemons that should have a service created. The names correspond
213 to the id part in ceph i.e. [ "name1" ] would result in mgr.name1
214 '';
215 };
216 extraConfig = mkOption {
217 type = with types; attrsOf str;
218 default = {};
219 description = ''
220 Extra configuration to add to the global section for manager daemons.
221 '';
222 };
223 };
224
225 mon = {
226 enable = mkEnableOption "Ceph MON daemon";
227 daemons = mkOption {
228 type = with types; listOf str;
229 default = [];
230 example = ''
231 [ "name1" "name2" ];
232 '';
233 description = ''
234 A list of monitor daemons that should have a service created. The names correspond
235 to the id part in ceph i.e. [ "name1" ] would result in mon.name1
236 '';
237 };
238 extraConfig = mkOption {
239 type = with types; attrsOf str;
240 default = {};
241 description = ''
242 Extra configuration to add to the monitor section.
243 '';
244 };
245 };
246
247 osd = {
248 enable = mkEnableOption "Ceph OSD daemon";
249 daemons = mkOption {
250 type = with types; listOf str;
251 default = [];
252 example = ''
253 [ "name1" "name2" ];
254 '';
255 description = ''
256 A list of OSD daemons that should have a service created. The names correspond
257 to the id part in ceph i.e. [ "name1" ] would result in osd.name1
258 '';
259 };
260
261 extraConfig = mkOption {
262 type = with types; attrsOf str;
263 default = {
264 "osd journal size" = "10000";
265 "osd pool default size" = "3";
266 "osd pool default min size" = "2";
267 "osd pool default pg num" = "200";
268 "osd pool default pgp num" = "200";
269 "osd crush chooseleaf type" = "1";
270 };
271 description = ''
272 Extra configuration to add to the OSD section.
273 '';
274 };
275 };
276
277 mds = {
278 enable = mkEnableOption "Ceph MDS daemon";
279 daemons = mkOption {
280 type = with types; listOf str;
281 default = [];
282 example = ''
283 [ "name1" "name2" ];
284 '';
285 description = ''
286 A list of metadata service daemons that should have a service created. The names correspond
287 to the id part in ceph i.e. [ "name1" ] would result in mds.name1
288 '';
289 };
290 extraConfig = mkOption {
291 type = with types; attrsOf str;
292 default = {};
293 description = ''
294 Extra configuration to add to the MDS section.
295 '';
296 };
297 };
298
299 rgw = {
300 enable = mkEnableOption "Ceph RadosGW daemon";
301 daemons = mkOption {
302 type = with types; listOf str;
303 default = [];
304 example = ''
305 [ "name1" "name2" ];
306 '';
307 description = ''
308 A list of rados gateway daemons that should have a service created. The names correspond
309 to the id part in ceph i.e. [ "name1" ] would result in client.name1, radosgw daemons
310 aren't daemons to cluster in the sense that OSD, MGR or MON daemons are. They are simply
311 daemons, from ceph, that uses the cluster as a backend.
312 '';
313 };
314 };
315
316 client = {
317 enable = mkEnableOption "Ceph client configuration";
318 extraConfig = mkOption {
319 type = with types; attrsOf (attrsOf str);
320 default = {};
321 example = ''
322 {
323 # This would create a section for a radosgw daemon named node0 and related
324 # configuration for it
325 "client.radosgw.node0" = { "some config option" = "true"; };
326 };
327 '';
328 description = ''
329 Extra configuration to add to the client section. Configuration for rados gateways
330 would be added here, with their own sections, see example.
331 '';
332 };
333 };
334 };
335
336 config = mkIf config.services.ceph.enable {
337 assertions = [
338 { assertion = cfg.global.fsid != "";
339 message = "fsid has to be set to a valid uuid for the cluster to function";
340 }
341 { assertion = cfg.mon.enable == true -> cfg.mon.daemons != [];
342 message = "have to set id of atleast one MON if you're going to enable Monitor";
343 }
344 { assertion = cfg.mds.enable == true -> cfg.mds.daemons != [];
345 message = "have to set id of atleast one MDS if you're going to enable Metadata Service";
346 }
347 { assertion = cfg.osd.enable == true -> cfg.osd.daemons != [];
348 message = "have to set id of atleast one OSD if you're going to enable OSD";
349 }
350 { assertion = cfg.mgr.enable == true -> cfg.mgr.daemons != [];
351 message = "have to set id of atleast one MGR if you're going to enable MGR";
352 }
353 ];
354
355 warnings = optional (cfg.global.monInitialMembers == null)
356 "Not setting up a list of members in monInitialMembers requires that you set the host variable for each mon daemon or else the cluster won't function";
357
358 environment.etc."ceph/ceph.conf".text = let
359 # Merge the extraConfig set for mgr daemons, as mgr don't have their own section
360 globalSection = expandCamelCaseAttrs (cfg.global // cfg.extraConfig // optionalAttrs cfg.mgr.enable cfg.mgr.extraConfig);
361 # Remove all name-value pairs with null values from the attribute set to avoid making empty sections in the ceph.conf
362 globalSection' = filterAttrs (name: value: value != null) globalSection;
363 totalConfig = {
364 global = globalSection';
365 } // optionalAttrs (cfg.mon.enable && cfg.mon.extraConfig != {}) { mon = cfg.mon.extraConfig; }
366 // optionalAttrs (cfg.mds.enable && cfg.mds.extraConfig != {}) { mds = cfg.mds.extraConfig; }
367 // optionalAttrs (cfg.osd.enable && cfg.osd.extraConfig != {}) { osd = cfg.osd.extraConfig; }
368 // optionalAttrs (cfg.client.enable && cfg.client.extraConfig != {}) cfg.client.extraConfig;
369 in
370 generators.toINI {} totalConfig;
371
372 users.users.ceph = {
373 uid = config.ids.uids.ceph;
374 description = "Ceph daemon user";
375 group = "ceph";
376 extraGroups = [ "disk" ];
377 };
378
379 users.groups.ceph = {
380 gid = config.ids.gids.ceph;
381 };
382
383 systemd.services = let
384 services = []
385 ++ optional cfg.mon.enable (makeServices "mon" cfg.mon.daemons)
386 ++ optional cfg.mds.enable (makeServices "mds" cfg.mds.daemons)
387 ++ optional cfg.osd.enable (makeServices "osd" cfg.osd.daemons)
388 ++ optional cfg.rgw.enable (makeServices "rgw" cfg.rgw.daemons)
389 ++ optional cfg.mgr.enable (makeServices "mgr" cfg.mgr.daemons);
390 in
391 mkMerge services;
392
393 systemd.targets = let
394 targets = [
395 { ceph = {
396 description = "Ceph target allowing to start/stop all ceph service instances at once";
397 wantedBy = [ "multi-user.target" ];
398 unitConfig.StopWhenUnneeded = true;
399 }; } ]
400 ++ optional cfg.mon.enable (makeTarget "mon")
401 ++ optional cfg.mds.enable (makeTarget "mds")
402 ++ optional cfg.osd.enable (makeTarget "osd")
403 ++ optional cfg.rgw.enable (makeTarget "rgw")
404 ++ optional cfg.mgr.enable (makeTarget "mgr");
405 in
406 mkMerge targets;
407
408 systemd.tmpfiles.rules = [
409 "d /etc/ceph - ceph ceph - -"
410 "d /run/ceph 0770 ceph ceph -"
411 "d /var/lib/ceph - ceph ceph - -"]
412 ++ optionals cfg.mgr.enable [ "d /var/lib/ceph/mgr - ceph ceph - -"]
413 ++ optionals cfg.mon.enable [ "d /var/lib/ceph/mon - ceph ceph - -"]
414 ++ optionals cfg.osd.enable [ "d /var/lib/ceph/osd - ceph ceph - -"];
415 };
416}