pyrox.dev
1{ config, lib, pkgs, ... }:
2
3with lib;
4
5let
6 cfg = config.services.ceph;
7
8 # function that translates "camelCaseOptions" to "camel case options", credits to tilpner in #nixos@freenode
9 expandCamelCase = replaceStrings upperChars (map (s: " ${s}") lowerChars);
10 expandCamelCaseAttrs = mapAttrs' (name: value: nameValuePair (expandCamelCase name) value);
11
12 makeServices = (daemonType: daemonIds:
13 mkMerge (map (daemonId:
14 { "ceph-${daemonType}-${daemonId}" = makeService daemonType daemonId cfg.global.clusterName pkgs.ceph; })
15 daemonIds));
16
17 makeService = (daemonType: daemonId: clusterName: ceph:
18 let
19 stateDirectory = "ceph/${if daemonType == "rgw" then "radosgw" else daemonType}/${clusterName}-${daemonId}"; in {
20 enable = true;
21 description = "Ceph ${builtins.replaceStrings lowerChars upperChars daemonType} daemon ${daemonId}";
22 after = [ "network-online.target" "time-sync.target" ] ++ optional (daemonType == "osd") "ceph-mon.target";
23 wants = [ "network-online.target" "time-sync.target" ];
24 partOf = [ "ceph-${daemonType}.target" ];
25 wantedBy = [ "ceph-${daemonType}.target" ];
26
27 path = [ pkgs.getopt ];
28
29 # Don't start services that are not yet initialized
30 unitConfig.ConditionPathExists = "/var/lib/${stateDirectory}/keyring";
31 startLimitBurst =
32 if daemonType == "osd" then 30 else if lib.elem daemonType ["mgr" "mds"] then 3 else 5;
33 startLimitIntervalSec = 60 * 30; # 30 mins
34
35 serviceConfig = {
36 LimitNOFILE = 1048576;
37 LimitNPROC = 1048576;
38 Environment = "CLUSTER=${clusterName}";
39 ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
40 PrivateDevices = "yes";
41 PrivateTmp = "true";
42 ProtectHome = "true";
43 ProtectSystem = "full";
44 Restart = "on-failure";
45 StateDirectory = stateDirectory;
46 User = "ceph";
47 Group = if daemonType == "osd" then "disk" else "ceph";
48 ExecStart = ''${ceph.out}/bin/${if daemonType == "rgw" then "radosgw" else "ceph-${daemonType}"} \
49 -f --cluster ${clusterName} --id ${daemonId}'';
50 } // optionalAttrs (daemonType == "osd") {
51 ExecStartPre = "${ceph.lib}/libexec/ceph/ceph-osd-prestart.sh --id ${daemonId} --cluster ${clusterName}";
52 RestartSec = "20s";
53 PrivateDevices = "no"; # osd needs disk access
54 } // optionalAttrs ( daemonType == "mon") {
55 RestartSec = "10";
56 };
57 });
58
59 makeTarget = (daemonType:
60 {
61 "ceph-${daemonType}" = {
62 description = "Ceph target allowing to start/stop all ceph-${daemonType} services at once";
63 partOf = [ "ceph.target" ];
64 wantedBy = [ "ceph.target" ];
65 before = [ "ceph.target" ];
66 unitConfig.StopWhenUnneeded = true;
67 };
68 }
69 );
70in
71{
72 options.services.ceph = {
73 # Ceph has a monolithic configuration file but different sections for
74 # each daemon, a separate client section and a global section
75 enable = mkEnableOption (lib.mdDoc "Ceph global configuration");
76
77 global = {
78 fsid = mkOption {
79 type = types.str;
80 example = ''
81 433a2193-4f8a-47a0-95d2-209d7ca2cca5
82 '';
83 description = lib.mdDoc ''
84 Filesystem ID, a generated uuid, its must be generated and set before
85 attempting to start a cluster
86 '';
87 };
88
89 clusterName = mkOption {
90 type = types.str;
91 default = "ceph";
92 description = lib.mdDoc ''
93 Name of cluster
94 '';
95 };
96
97 mgrModulePath = mkOption {
98 type = types.path;
99 default = "${pkgs.ceph.lib}/lib/ceph/mgr";
100 defaultText = literalExpression ''"''${pkgs.ceph.lib}/lib/ceph/mgr"'';
101 description = lib.mdDoc ''
102 Path at which to find ceph-mgr modules.
103 '';
104 };
105
106 monInitialMembers = mkOption {
107 type = with types; nullOr commas;
108 default = null;
109 example = ''
110 node0, node1, node2
111 '';
112 description = lib.mdDoc ''
113 List of hosts that will be used as monitors at startup.
114 '';
115 };
116
117 monHost = mkOption {
118 type = with types; nullOr commas;
119 default = null;
120 example = ''
121 10.10.0.1, 10.10.0.2, 10.10.0.3
122 '';
123 description = lib.mdDoc ''
124 List of hostname shortnames/IP addresses of the initial monitors.
125 '';
126 };
127
128 maxOpenFiles = mkOption {
129 type = types.int;
130 default = 131072;
131 description = lib.mdDoc ''
132 Max open files for each OSD daemon.
133 '';
134 };
135
136 authClusterRequired = mkOption {
137 type = types.enum [ "cephx" "none" ];
138 default = "cephx";
139 description = lib.mdDoc ''
140 Enables requiring daemons to authenticate with eachother in the cluster.
141 '';
142 };
143
144 authServiceRequired = mkOption {
145 type = types.enum [ "cephx" "none" ];
146 default = "cephx";
147 description = lib.mdDoc ''
148 Enables requiring clients to authenticate with the cluster to access services in the cluster (e.g. radosgw, mds or osd).
149 '';
150 };
151
152 authClientRequired = mkOption {
153 type = types.enum [ "cephx" "none" ];
154 default = "cephx";
155 description = lib.mdDoc ''
156 Enables requiring the cluster to authenticate itself to the client.
157 '';
158 };
159
160 publicNetwork = mkOption {
161 type = with types; nullOr commas;
162 default = null;
163 example = ''
164 10.20.0.0/24, 192.168.1.0/24
165 '';
166 description = lib.mdDoc ''
167 A comma-separated list of subnets that will be used as public networks in the cluster.
168 '';
169 };
170
171 clusterNetwork = mkOption {
172 type = with types; nullOr commas;
173 default = null;
174 example = ''
175 10.10.0.0/24, 192.168.0.0/24
176 '';
177 description = lib.mdDoc ''
178 A comma-separated list of subnets that will be used as cluster networks in the cluster.
179 '';
180 };
181
182 rgwMimeTypesFile = mkOption {
183 type = with types; nullOr path;
184 default = "${pkgs.mailcap}/etc/mime.types";
185 defaultText = literalExpression ''"''${pkgs.mailcap}/etc/mime.types"'';
186 description = lib.mdDoc ''
187 Path to mime types used by radosgw.
188 '';
189 };
190 };
191
192 extraConfig = mkOption {
193 type = with types; attrsOf str;
194 default = {};
195 example = {
196 "ms bind ipv6" = "true";
197 };
198 description = lib.mdDoc ''
199 Extra configuration to add to the global section. Use for setting values that are common for all daemons in the cluster.
200 '';
201 };
202
203 mgr = {
204 enable = mkEnableOption (lib.mdDoc "Ceph MGR daemon");
205 daemons = mkOption {
206 type = with types; listOf str;
207 default = [];
208 example = [ "name1" "name2" ];
209 description = lib.mdDoc ''
210 A list of names for manager daemons that should have a service created. The names correspond
211 to the id part in ceph i.e. [ "name1" ] would result in mgr.name1
212 '';
213 };
214 extraConfig = mkOption {
215 type = with types; attrsOf str;
216 default = {};
217 description = lib.mdDoc ''
218 Extra configuration to add to the global section for manager daemons.
219 '';
220 };
221 };
222
223 mon = {
224 enable = mkEnableOption (lib.mdDoc "Ceph MON daemon");
225 daemons = mkOption {
226 type = with types; listOf str;
227 default = [];
228 example = [ "name1" "name2" ];
229 description = lib.mdDoc ''
230 A list of monitor daemons that should have a service created. The names correspond
231 to the id part in ceph i.e. [ "name1" ] would result in mon.name1
232 '';
233 };
234 extraConfig = mkOption {
235 type = with types; attrsOf str;
236 default = {};
237 description = lib.mdDoc ''
238 Extra configuration to add to the monitor section.
239 '';
240 };
241 };
242
243 osd = {
244 enable = mkEnableOption (lib.mdDoc "Ceph OSD daemon");
245 daemons = mkOption {
246 type = with types; listOf str;
247 default = [];
248 example = [ "name1" "name2" ];
249 description = lib.mdDoc ''
250 A list of OSD daemons that should have a service created. The names correspond
251 to the id part in ceph i.e. [ "name1" ] would result in osd.name1
252 '';
253 };
254
255 extraConfig = mkOption {
256 type = with types; attrsOf str;
257 default = {
258 "osd journal size" = "10000";
259 "osd pool default size" = "3";
260 "osd pool default min size" = "2";
261 "osd pool default pg num" = "200";
262 "osd pool default pgp num" = "200";
263 "osd crush chooseleaf type" = "1";
264 };
265 description = lib.mdDoc ''
266 Extra configuration to add to the OSD section.
267 '';
268 };
269 };
270
271 mds = {
272 enable = mkEnableOption (lib.mdDoc "Ceph MDS daemon");
273 daemons = mkOption {
274 type = with types; listOf str;
275 default = [];
276 example = [ "name1" "name2" ];
277 description = lib.mdDoc ''
278 A list of metadata service daemons that should have a service created. The names correspond
279 to the id part in ceph i.e. [ "name1" ] would result in mds.name1
280 '';
281 };
282 extraConfig = mkOption {
283 type = with types; attrsOf str;
284 default = {};
285 description = lib.mdDoc ''
286 Extra configuration to add to the MDS section.
287 '';
288 };
289 };
290
291 rgw = {
292 enable = mkEnableOption (lib.mdDoc "Ceph RadosGW daemon");
293 daemons = mkOption {
294 type = with types; listOf str;
295 default = [];
296 example = [ "name1" "name2" ];
297 description = lib.mdDoc ''
298 A list of rados gateway daemons that should have a service created. The names correspond
299 to the id part in ceph i.e. [ "name1" ] would result in client.name1, radosgw daemons
300 aren't daemons to cluster in the sense that OSD, MGR or MON daemons are. They are simply
301 daemons, from ceph, that uses the cluster as a backend.
302 '';
303 };
304 };
305
306 client = {
307 enable = mkEnableOption (lib.mdDoc "Ceph client configuration");
308 extraConfig = mkOption {
309 type = with types; attrsOf (attrsOf str);
310 default = {};
311 example = literalExpression ''
312 {
313 # This would create a section for a radosgw daemon named node0 and related
314 # configuration for it
315 "client.radosgw.node0" = { "some config option" = "true"; };
316 };
317 '';
318 description = lib.mdDoc ''
319 Extra configuration to add to the client section. Configuration for rados gateways
320 would be added here, with their own sections, see example.
321 '';
322 };
323 };
324 };
325
326 config = mkIf config.services.ceph.enable {
327 assertions = [
328 { assertion = cfg.global.fsid != "";
329 message = "fsid has to be set to a valid uuid for the cluster to function";
330 }
331 { assertion = cfg.mon.enable == true -> cfg.mon.daemons != [];
332 message = "have to set id of atleast one MON if you're going to enable Monitor";
333 }
334 { assertion = cfg.mds.enable == true -> cfg.mds.daemons != [];
335 message = "have to set id of atleast one MDS if you're going to enable Metadata Service";
336 }
337 { assertion = cfg.osd.enable == true -> cfg.osd.daemons != [];
338 message = "have to set id of atleast one OSD if you're going to enable OSD";
339 }
340 { assertion = cfg.mgr.enable == true -> cfg.mgr.daemons != [];
341 message = "have to set id of atleast one MGR if you're going to enable MGR";
342 }
343 ];
344
345 warnings = optional (cfg.global.monInitialMembers == null)
346 "Not setting up a list of members in monInitialMembers requires that you set the host variable for each mon daemon or else the cluster won't function";
347
348 environment.etc."ceph/ceph.conf".text = let
349 # Merge the extraConfig set for mgr daemons, as mgr don't have their own section
350 globalSection = expandCamelCaseAttrs (cfg.global // cfg.extraConfig // optionalAttrs cfg.mgr.enable cfg.mgr.extraConfig);
351 # Remove all name-value pairs with null values from the attribute set to avoid making empty sections in the ceph.conf
352 globalSection' = filterAttrs (name: value: value != null) globalSection;
353 totalConfig = {
354 global = globalSection';
355 } // optionalAttrs (cfg.mon.enable && cfg.mon.extraConfig != {}) { mon = cfg.mon.extraConfig; }
356 // optionalAttrs (cfg.mds.enable && cfg.mds.extraConfig != {}) { mds = cfg.mds.extraConfig; }
357 // optionalAttrs (cfg.osd.enable && cfg.osd.extraConfig != {}) { osd = cfg.osd.extraConfig; }
358 // optionalAttrs (cfg.client.enable && cfg.client.extraConfig != {}) cfg.client.extraConfig;
359 in
360 generators.toINI {} totalConfig;
361
362 users.users.ceph = {
363 uid = config.ids.uids.ceph;
364 description = "Ceph daemon user";
365 group = "ceph";
366 extraGroups = [ "disk" ];
367 };
368
369 users.groups.ceph = {
370 gid = config.ids.gids.ceph;
371 };
372
373 systemd.services = let
374 services = []
375 ++ optional cfg.mon.enable (makeServices "mon" cfg.mon.daemons)
376 ++ optional cfg.mds.enable (makeServices "mds" cfg.mds.daemons)
377 ++ optional cfg.osd.enable (makeServices "osd" cfg.osd.daemons)
378 ++ optional cfg.rgw.enable (makeServices "rgw" cfg.rgw.daemons)
379 ++ optional cfg.mgr.enable (makeServices "mgr" cfg.mgr.daemons);
380 in
381 mkMerge services;
382
383 systemd.targets = let
384 targets = [
385 { ceph = {
386 description = "Ceph target allowing to start/stop all ceph service instances at once";
387 wantedBy = [ "multi-user.target" ];
388 unitConfig.StopWhenUnneeded = true;
389 }; } ]
390 ++ optional cfg.mon.enable (makeTarget "mon")
391 ++ optional cfg.mds.enable (makeTarget "mds")
392 ++ optional cfg.osd.enable (makeTarget "osd")
393 ++ optional cfg.rgw.enable (makeTarget "rgw")
394 ++ optional cfg.mgr.enable (makeTarget "mgr");
395 in
396 mkMerge targets;
397
398 systemd.tmpfiles.rules = [
399 "d /etc/ceph - ceph ceph - -"
400 "d /run/ceph 0770 ceph ceph -"
401 "d /var/lib/ceph - ceph ceph - -"]
402 ++ optionals cfg.mgr.enable [ "d /var/lib/ceph/mgr - ceph ceph - -"]
403 ++ optionals cfg.mon.enable [ "d /var/lib/ceph/mon - ceph ceph - -"]
404 ++ optionals cfg.osd.enable [ "d /var/lib/ceph/osd - ceph ceph - -"];
405 };
406}