pyrox.dev
1{ config, lib, pkgs, ... }:
2
3with lib;
4
5let
6 cfg = config.services.ceph;
7
8 # function that translates "camelCaseOptions" to "camel case options", credits to tilpner in #nixos@freenode
9 expandCamelCase = replaceStrings upperChars (map (s: " ${s}") lowerChars);
10 expandCamelCaseAttrs = mapAttrs' (name: value: nameValuePair (expandCamelCase name) value);
11
12 makeServices = daemonType: daemonIds:
13 mkMerge (map (daemonId:
14 { "ceph-${daemonType}-${daemonId}" = makeService daemonType daemonId cfg.global.clusterName cfg.${daemonType}.package; })
15 daemonIds);
16
17 makeService = daemonType: daemonId: clusterName: ceph:
18 let
19 stateDirectory = "ceph/${if daemonType == "rgw" then "radosgw" else daemonType}/${clusterName}-${daemonId}"; in {
20 enable = true;
21 description = "Ceph ${builtins.replaceStrings lowerChars upperChars daemonType} daemon ${daemonId}";
22 after = [ "network-online.target" "time-sync.target" ] ++ optional (daemonType == "osd") "ceph-mon.target";
23 wants = [ "network-online.target" "time-sync.target" ];
24 partOf = [ "ceph-${daemonType}.target" ];
25 wantedBy = [ "ceph-${daemonType}.target" ];
26
27 path = [ pkgs.getopt ];
28
29 # Don't start services that are not yet initialized
30 unitConfig.ConditionPathExists = "/var/lib/${stateDirectory}/keyring";
31 startLimitBurst =
32 if daemonType == "osd" then 30 else if lib.elem daemonType ["mgr" "mds"] then 3 else 5;
33 startLimitIntervalSec = 60 * 30; # 30 mins
34
35 serviceConfig = {
36 LimitNOFILE = 1048576;
37 LimitNPROC = 1048576;
38 Environment = "CLUSTER=${clusterName}";
39 ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
40 PrivateDevices = "yes";
41 PrivateTmp = "true";
42 ProtectHome = "true";
43 ProtectSystem = "full";
44 Restart = "on-failure";
45 StateDirectory = stateDirectory;
46 User = "ceph";
47 Group = if daemonType == "osd" then "disk" else "ceph";
48 ExecStart = ''${ceph.out}/bin/${if daemonType == "rgw" then "radosgw" else "ceph-${daemonType}"} \
49 -f --cluster ${clusterName} --id ${daemonId}'';
50 } // optionalAttrs (daemonType == "osd") {
51 ExecStartPre = "${ceph.lib}/libexec/ceph/ceph-osd-prestart.sh --id ${daemonId} --cluster ${clusterName}";
52 RestartSec = "20s";
53 PrivateDevices = "no"; # osd needs disk access
54 } // optionalAttrs ( daemonType == "mon") {
55 RestartSec = "10";
56 };
57 };
58
59 makeTarget = daemonType:
60 {
61 "ceph-${daemonType}" = {
62 description = "Ceph target allowing to start/stop all ceph-${daemonType} services at once";
63 partOf = [ "ceph.target" ];
64 wantedBy = [ "ceph.target" ];
65 before = [ "ceph.target" ];
66 unitConfig.StopWhenUnneeded = true;
67 };
68 };
69in
70{
71 options.services.ceph = {
72 # Ceph has a monolithic configuration file but different sections for
73 # each daemon, a separate client section and a global section
74 enable = mkEnableOption (lib.mdDoc "Ceph global configuration");
75
76 global = {
77 fsid = mkOption {
78 type = types.str;
79 example = ''
80 433a2193-4f8a-47a0-95d2-209d7ca2cca5
81 '';
82 description = lib.mdDoc ''
83 Filesystem ID, a generated uuid, its must be generated and set before
84 attempting to start a cluster
85 '';
86 };
87
88 clusterName = mkOption {
89 type = types.str;
90 default = "ceph";
91 description = lib.mdDoc ''
92 Name of cluster
93 '';
94 };
95
96 mgrModulePath = mkOption {
97 type = types.path;
98 default = "${pkgs.ceph.lib}/lib/ceph/mgr";
99 defaultText = literalExpression ''"''${pkgs.ceph.lib}/lib/ceph/mgr"'';
100 description = lib.mdDoc ''
101 Path at which to find ceph-mgr modules.
102 '';
103 };
104
105 monInitialMembers = mkOption {
106 type = with types; nullOr commas;
107 default = null;
108 example = ''
109 node0, node1, node2
110 '';
111 description = lib.mdDoc ''
112 List of hosts that will be used as monitors at startup.
113 '';
114 };
115
116 monHost = mkOption {
117 type = with types; nullOr commas;
118 default = null;
119 example = ''
120 10.10.0.1, 10.10.0.2, 10.10.0.3
121 '';
122 description = lib.mdDoc ''
123 List of hostname shortnames/IP addresses of the initial monitors.
124 '';
125 };
126
127 maxOpenFiles = mkOption {
128 type = types.int;
129 default = 131072;
130 description = lib.mdDoc ''
131 Max open files for each OSD daemon.
132 '';
133 };
134
135 authClusterRequired = mkOption {
136 type = types.enum [ "cephx" "none" ];
137 default = "cephx";
138 description = lib.mdDoc ''
139 Enables requiring daemons to authenticate with eachother in the cluster.
140 '';
141 };
142
143 authServiceRequired = mkOption {
144 type = types.enum [ "cephx" "none" ];
145 default = "cephx";
146 description = lib.mdDoc ''
147 Enables requiring clients to authenticate with the cluster to access services in the cluster (e.g. radosgw, mds or osd).
148 '';
149 };
150
151 authClientRequired = mkOption {
152 type = types.enum [ "cephx" "none" ];
153 default = "cephx";
154 description = lib.mdDoc ''
155 Enables requiring the cluster to authenticate itself to the client.
156 '';
157 };
158
159 publicNetwork = mkOption {
160 type = with types; nullOr commas;
161 default = null;
162 example = ''
163 10.20.0.0/24, 192.168.1.0/24
164 '';
165 description = lib.mdDoc ''
166 A comma-separated list of subnets that will be used as public networks in the cluster.
167 '';
168 };
169
170 clusterNetwork = mkOption {
171 type = with types; nullOr commas;
172 default = null;
173 example = ''
174 10.10.0.0/24, 192.168.0.0/24
175 '';
176 description = lib.mdDoc ''
177 A comma-separated list of subnets that will be used as cluster networks in the cluster.
178 '';
179 };
180
181 rgwMimeTypesFile = mkOption {
182 type = with types; nullOr path;
183 default = "${pkgs.mailcap}/etc/mime.types";
184 defaultText = literalExpression ''"''${pkgs.mailcap}/etc/mime.types"'';
185 description = lib.mdDoc ''
186 Path to mime types used by radosgw.
187 '';
188 };
189 };
190
191 extraConfig = mkOption {
192 type = with types; attrsOf str;
193 default = {};
194 example = {
195 "ms bind ipv6" = "true";
196 };
197 description = lib.mdDoc ''
198 Extra configuration to add to the global section. Use for setting values that are common for all daemons in the cluster.
199 '';
200 };
201
202 mgr = {
203 enable = mkEnableOption (lib.mdDoc "Ceph MGR daemon");
204 daemons = mkOption {
205 type = with types; listOf str;
206 default = [];
207 example = [ "name1" "name2" ];
208 description = lib.mdDoc ''
209 A list of names for manager daemons that should have a service created. The names correspond
210 to the id part in ceph i.e. [ "name1" ] would result in mgr.name1
211 '';
212 };
213 package = mkPackageOptionMD pkgs "ceph" { };
214 extraConfig = mkOption {
215 type = with types; attrsOf str;
216 default = {};
217 description = lib.mdDoc ''
218 Extra configuration to add to the global section for manager daemons.
219 '';
220 };
221 };
222
223 mon = {
224 enable = mkEnableOption (lib.mdDoc "Ceph MON daemon");
225 daemons = mkOption {
226 type = with types; listOf str;
227 default = [];
228 example = [ "name1" "name2" ];
229 description = lib.mdDoc ''
230 A list of monitor daemons that should have a service created. The names correspond
231 to the id part in ceph i.e. [ "name1" ] would result in mon.name1
232 '';
233 };
234 package = mkPackageOptionMD pkgs "ceph" { };
235 extraConfig = mkOption {
236 type = with types; attrsOf str;
237 default = {};
238 description = lib.mdDoc ''
239 Extra configuration to add to the monitor section.
240 '';
241 };
242 };
243
244 osd = {
245 enable = mkEnableOption (lib.mdDoc "Ceph OSD daemon");
246 daemons = mkOption {
247 type = with types; listOf str;
248 default = [];
249 example = [ "name1" "name2" ];
250 description = lib.mdDoc ''
251 A list of OSD daemons that should have a service created. The names correspond
252 to the id part in ceph i.e. [ "name1" ] would result in osd.name1
253 '';
254 };
255 package = mkPackageOptionMD pkgs "ceph" { };
256 extraConfig = mkOption {
257 type = with types; attrsOf str;
258 default = {
259 "osd journal size" = "10000";
260 "osd pool default size" = "3";
261 "osd pool default min size" = "2";
262 "osd pool default pg num" = "200";
263 "osd pool default pgp num" = "200";
264 "osd crush chooseleaf type" = "1";
265 };
266 description = lib.mdDoc ''
267 Extra configuration to add to the OSD section.
268 '';
269 };
270 };
271
272 mds = {
273 enable = mkEnableOption (lib.mdDoc "Ceph MDS daemon");
274 daemons = mkOption {
275 type = with types; listOf str;
276 default = [];
277 example = [ "name1" "name2" ];
278 description = lib.mdDoc ''
279 A list of metadata service daemons that should have a service created. The names correspond
280 to the id part in ceph i.e. [ "name1" ] would result in mds.name1
281 '';
282 };
283 package = mkPackageOptionMD pkgs "ceph" { };
284 extraConfig = mkOption {
285 type = with types; attrsOf str;
286 default = {};
287 description = lib.mdDoc ''
288 Extra configuration to add to the MDS section.
289 '';
290 };
291 };
292
293 rgw = {
294 enable = mkEnableOption (lib.mdDoc "Ceph RadosGW daemon");
295 package = mkPackageOptionMD pkgs "ceph" { };
296 daemons = mkOption {
297 type = with types; listOf str;
298 default = [];
299 example = [ "name1" "name2" ];
300 description = lib.mdDoc ''
301 A list of rados gateway daemons that should have a service created. The names correspond
302 to the id part in ceph i.e. [ "name1" ] would result in client.name1, radosgw daemons
303 aren't daemons to cluster in the sense that OSD, MGR or MON daemons are. They are simply
304 daemons, from ceph, that uses the cluster as a backend.
305 '';
306 };
307 };
308
309 client = {
310 enable = mkEnableOption (lib.mdDoc "Ceph client configuration");
311 extraConfig = mkOption {
312 type = with types; attrsOf (attrsOf str);
313 default = {};
314 example = literalExpression ''
315 {
316 # This would create a section for a radosgw daemon named node0 and related
317 # configuration for it
318 "client.radosgw.node0" = { "some config option" = "true"; };
319 };
320 '';
321 description = lib.mdDoc ''
322 Extra configuration to add to the client section. Configuration for rados gateways
323 would be added here, with their own sections, see example.
324 '';
325 };
326 };
327 };
328
329 config = mkIf config.services.ceph.enable {
330 assertions = [
331 { assertion = cfg.global.fsid != "";
332 message = "fsid has to be set to a valid uuid for the cluster to function";
333 }
334 { assertion = cfg.mon.enable -> cfg.mon.daemons != [];
335 message = "have to set id of atleast one MON if you're going to enable Monitor";
336 }
337 { assertion = cfg.mds.enable -> cfg.mds.daemons != [];
338 message = "have to set id of atleast one MDS if you're going to enable Metadata Service";
339 }
340 { assertion = cfg.osd.enable -> cfg.osd.daemons != [];
341 message = "have to set id of atleast one OSD if you're going to enable OSD";
342 }
343 { assertion = cfg.mgr.enable -> cfg.mgr.daemons != [];
344 message = "have to set id of atleast one MGR if you're going to enable MGR";
345 }
346 ];
347
348 warnings = optional (cfg.global.monInitialMembers == null)
349 "Not setting up a list of members in monInitialMembers requires that you set the host variable for each mon daemon or else the cluster won't function";
350
351 environment.etc."ceph/ceph.conf".text = let
352 # Merge the extraConfig set for mgr daemons, as mgr don't have their own section
353 globalSection = expandCamelCaseAttrs (cfg.global // cfg.extraConfig // optionalAttrs cfg.mgr.enable cfg.mgr.extraConfig);
354 # Remove all name-value pairs with null values from the attribute set to avoid making empty sections in the ceph.conf
355 globalSection' = filterAttrs (name: value: value != null) globalSection;
356 totalConfig = {
357 global = globalSection';
358 } // optionalAttrs (cfg.mon.enable && cfg.mon.extraConfig != {}) { mon = cfg.mon.extraConfig; }
359 // optionalAttrs (cfg.mds.enable && cfg.mds.extraConfig != {}) { mds = cfg.mds.extraConfig; }
360 // optionalAttrs (cfg.osd.enable && cfg.osd.extraConfig != {}) { osd = cfg.osd.extraConfig; }
361 // optionalAttrs (cfg.client.enable && cfg.client.extraConfig != {}) cfg.client.extraConfig;
362 in
363 generators.toINI {} totalConfig;
364
365 users.users.ceph = {
366 uid = config.ids.uids.ceph;
367 description = "Ceph daemon user";
368 group = "ceph";
369 extraGroups = [ "disk" ];
370 };
371
372 users.groups.ceph = {
373 gid = config.ids.gids.ceph;
374 };
375
376 systemd.services = let
377 services = []
378 ++ optional cfg.mon.enable (makeServices "mon" cfg.mon.daemons)
379 ++ optional cfg.mds.enable (makeServices "mds" cfg.mds.daemons)
380 ++ optional cfg.osd.enable (makeServices "osd" cfg.osd.daemons)
381 ++ optional cfg.rgw.enable (makeServices "rgw" cfg.rgw.daemons)
382 ++ optional cfg.mgr.enable (makeServices "mgr" cfg.mgr.daemons);
383 in
384 mkMerge services;
385
386 systemd.targets = let
387 targets = [
388 { ceph = {
389 description = "Ceph target allowing to start/stop all ceph service instances at once";
390 wantedBy = [ "multi-user.target" ];
391 unitConfig.StopWhenUnneeded = true;
392 }; } ]
393 ++ optional cfg.mon.enable (makeTarget "mon")
394 ++ optional cfg.mds.enable (makeTarget "mds")
395 ++ optional cfg.osd.enable (makeTarget "osd")
396 ++ optional cfg.rgw.enable (makeTarget "rgw")
397 ++ optional cfg.mgr.enable (makeTarget "mgr");
398 in
399 mkMerge targets;
400
401 systemd.tmpfiles.rules = [
402 "d /etc/ceph - ceph ceph - -"
403 "d /run/ceph 0770 ceph ceph -"
404 "d /var/lib/ceph - ceph ceph - -"]
405 ++ optionals cfg.mgr.enable [ "d /var/lib/ceph/mgr - ceph ceph - -"]
406 ++ optionals cfg.mon.enable [ "d /var/lib/ceph/mon - ceph ceph - -"]
407 ++ optionals cfg.osd.enable [ "d /var/lib/ceph/osd - ceph ceph - -"];
408 };
409}