pyrox.dev
1{ config, lib, pkgs, ... }:
2
3with lib;
4
5let
6 ceph = pkgs.ceph;
7 cfg = config.services.ceph;
8 # function that translates "camelCaseOptions" to "camel case options", credits to tilpner in #nixos@freenode
9 translateOption = replaceStrings upperChars (map (s: " ${s}") lowerChars);
10 generateDaemonList = (daemonType: daemons: extraServiceConfig:
11 mkMerge (
12 map (daemon:
13 { "ceph-${daemonType}-${daemon}" = generateServiceFile daemonType daemon cfg.global.clusterName ceph extraServiceConfig; }
14 ) daemons
15 )
16 );
17 generateServiceFile = (daemonType: daemonId: clusterName: ceph: extraServiceConfig: {
18 enable = true;
19 description = "Ceph ${builtins.replaceStrings lowerChars upperChars daemonType} daemon ${daemonId}";
20 after = [ "network-online.target" "local-fs.target" "time-sync.target" ] ++ optional (daemonType == "osd") "ceph-mon.target";
21 wants = [ "network-online.target" "local-fs.target" "time-sync.target" ];
22 partOf = [ "ceph-${daemonType}.target" ];
23 wantedBy = [ "ceph-${daemonType}.target" ];
24
25 serviceConfig = {
26 LimitNOFILE = 1048576;
27 LimitNPROC = 1048576;
28 Environment = "CLUSTER=${clusterName}";
29 ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
30 PrivateDevices = "yes";
31 PrivateTmp = "true";
32 ProtectHome = "true";
33 ProtectSystem = "full";
34 Restart = "on-failure";
35 StartLimitBurst = "5";
36 StartLimitInterval = "30min";
37 ExecStart = "${ceph.out}/bin/${if daemonType == "rgw" then "radosgw" else "ceph-${daemonType}"} -f --cluster ${clusterName} --id ${if daemonType == "rgw" then "client.${daemonId}" else daemonId} --setuser ceph --setgroup ceph";
38 } // extraServiceConfig
39 // optionalAttrs (daemonType == "osd") { ExecStartPre = "${ceph.out}/libexec/ceph/ceph-osd-prestart.sh --id ${daemonId} --cluster ${clusterName}"; };
40 } // optionalAttrs (builtins.elem daemonType [ "mds" "mon" "rgw" "mgr" ]) { preStart = ''
41 daemonPath="/var/lib/ceph/${if daemonType == "rgw" then "radosgw" else daemonType}/${clusterName}-${daemonId}"
42 if [ ! -d ''$daemonPath ]; then
43 mkdir -m 755 -p ''$daemonPath
44 chown -R ceph:ceph ''$daemonPath
45 fi
46 '';
47 } // optionalAttrs (daemonType == "osd") { path = [ pkgs.getopt ]; }
48 );
49 generateTargetFile = (daemonType:
50 {
51 "ceph-${daemonType}" = {
52 description = "Ceph target allowing to start/stop all ceph-${daemonType} services at once";
53 partOf = [ "ceph.target" ];
54 before = [ "ceph.target" ];
55 };
56 }
57 );
58in
59{
60 options.services.ceph = {
61 # Ceph has a monolithic configuration file but different sections for
62 # each daemon, a separate client section and a global section
63 enable = mkEnableOption "Ceph global configuration";
64
65 global = {
66 fsid = mkOption {
67 type = types.str;
68 example = ''
69 433a2193-4f8a-47a0-95d2-209d7ca2cca5
70 '';
71 description = ''
72 Filesystem ID, a generated uuid, its must be generated and set before
73 attempting to start a cluster
74 '';
75 };
76
77 clusterName = mkOption {
78 type = types.str;
79 default = "ceph";
80 description = ''
81 Name of cluster
82 '';
83 };
84
85 monInitialMembers = mkOption {
86 type = with types; nullOr commas;
87 default = null;
88 example = ''
89 node0, node1, node2
90 '';
91 description = ''
92 List of hosts that will be used as monitors at startup.
93 '';
94 };
95
96 monHost = mkOption {
97 type = with types; nullOr commas;
98 default = null;
99 example = ''
100 10.10.0.1, 10.10.0.2, 10.10.0.3
101 '';
102 description = ''
103 List of hostname shortnames/IP addresses of the initial monitors.
104 '';
105 };
106
107 maxOpenFiles = mkOption {
108 type = types.int;
109 default = 131072;
110 description = ''
111 Max open files for each OSD daemon.
112 '';
113 };
114
115 authClusterRequired = mkOption {
116 type = types.enum [ "cephx" "none" ];
117 default = "cephx";
118 description = ''
119 Enables requiring daemons to authenticate with eachother in the cluster.
120 '';
121 };
122
123 authServiceRequired = mkOption {
124 type = types.enum [ "cephx" "none" ];
125 default = "cephx";
126 description = ''
127 Enables requiring clients to authenticate with the cluster to access services in the cluster (e.g. radosgw, mds or osd).
128 '';
129 };
130
131 authClientRequired = mkOption {
132 type = types.enum [ "cephx" "none" ];
133 default = "cephx";
134 description = ''
135 Enables requiring the cluster to authenticate itself to the client.
136 '';
137 };
138
139 publicNetwork = mkOption {
140 type = with types; nullOr commas;
141 default = null;
142 example = ''
143 10.20.0.0/24, 192.168.1.0/24
144 '';
145 description = ''
146 A comma-separated list of subnets that will be used as public networks in the cluster.
147 '';
148 };
149
150 clusterNetwork = mkOption {
151 type = with types; nullOr commas;
152 default = null;
153 example = ''
154 10.10.0.0/24, 192.168.0.0/24
155 '';
156 description = ''
157 A comma-separated list of subnets that will be used as cluster networks in the cluster.
158 '';
159 };
160 };
161
162 mgr = {
163 enable = mkEnableOption "Ceph MGR daemon";
164 daemons = mkOption {
165 type = with types; listOf str;
166 default = [];
167 example = ''
168 [ "name1" "name2" ];
169 '';
170 description = ''
171 A list of names for manager daemons that should have a service created. The names correspond
172 to the id part in ceph i.e. [ "name1" ] would result in mgr.name1
173 '';
174 };
175 extraConfig = mkOption {
176 type = with types; attrsOf str;
177 default = {};
178 description = ''
179 Extra configuration to add to the global section for manager daemons.
180 '';
181 };
182 };
183
184 mon = {
185 enable = mkEnableOption "Ceph MON daemon";
186 daemons = mkOption {
187 type = with types; listOf str;
188 default = [];
189 example = ''
190 [ "name1" "name2" ];
191 '';
192 description = ''
193 A list of monitor daemons that should have a service created. The names correspond
194 to the id part in ceph i.e. [ "name1" ] would result in mon.name1
195 '';
196 };
197 extraConfig = mkOption {
198 type = with types; attrsOf str;
199 default = {};
200 description = ''
201 Extra configuration to add to the monitor section.
202 '';
203 };
204 };
205
206 osd = {
207 enable = mkEnableOption "Ceph OSD daemon";
208 daemons = mkOption {
209 type = with types; listOf str;
210 default = [];
211 example = ''
212 [ "name1" "name2" ];
213 '';
214 description = ''
215 A list of OSD daemons that should have a service created. The names correspond
216 to the id part in ceph i.e. [ "name1" ] would result in osd.name1
217 '';
218 };
219 extraConfig = mkOption {
220 type = with types; attrsOf str;
221 default = {
222 "osd journal size" = "10000";
223 "osd pool default size" = "3";
224 "osd pool default min size" = "2";
225 "osd pool default pg num" = "200";
226 "osd pool default pgp num" = "200";
227 "osd crush chooseleaf type" = "1";
228 };
229 description = ''
230 Extra configuration to add to the OSD section.
231 '';
232 };
233 };
234
235 mds = {
236 enable = mkEnableOption "Ceph MDS daemon";
237 daemons = mkOption {
238 type = with types; listOf str;
239 default = [];
240 example = ''
241 [ "name1" "name2" ];
242 '';
243 description = ''
244 A list of metadata service daemons that should have a service created. The names correspond
245 to the id part in ceph i.e. [ "name1" ] would result in mds.name1
246 '';
247 };
248 extraConfig = mkOption {
249 type = with types; attrsOf str;
250 default = {};
251 description = ''
252 Extra configuration to add to the MDS section.
253 '';
254 };
255 };
256
257 rgw = {
258 enable = mkEnableOption "Ceph RadosGW daemon";
259 daemons = mkOption {
260 type = with types; listOf str;
261 default = [];
262 example = ''
263 [ "name1" "name2" ];
264 '';
265 description = ''
266 A list of rados gateway daemons that should have a service created. The names correspond
267 to the id part in ceph i.e. [ "name1" ] would result in client.name1, radosgw daemons
268 aren't daemons to cluster in the sense that OSD, MGR or MON daemons are. They are simply
269 daemons, from ceph, that uses the cluster as a backend.
270 '';
271 };
272 };
273
274 client = {
275 enable = mkEnableOption "Ceph client configuration";
276 extraConfig = mkOption {
277 type = with types; attrsOf str;
278 default = {};
279 example = ''
280 {
281 # This would create a section for a radosgw daemon named node0 and related
282 # configuration for it
283 "client.radosgw.node0" = { "some config option" = "true"; };
284 };
285 '';
286 description = ''
287 Extra configuration to add to the client section. Configuration for rados gateways
288 would be added here, with their own sections, see example.
289 '';
290 };
291 };
292 };
293
294 config = mkIf config.services.ceph.enable {
295 assertions = [
296 { assertion = cfg.global.fsid != "";
297 message = "fsid has to be set to a valid uuid for the cluster to function";
298 }
299 { assertion = cfg.mgr.enable == true;
300 message = "ceph 12.x requires atleast 1 MGR daemon enabled for the cluster to function";
301 }
302 { assertion = cfg.mon.enable == true -> cfg.mon.daemons != [];
303 message = "have to set id of atleast one MON if you're going to enable Monitor";
304 }
305 { assertion = cfg.mds.enable == true -> cfg.mds.daemons != [];
306 message = "have to set id of atleast one MDS if you're going to enable Metadata Service";
307 }
308 { assertion = cfg.osd.enable == true -> cfg.osd.daemons != [];
309 message = "have to set id of atleast one OSD if you're going to enable OSD";
310 }
311 { assertion = cfg.mgr.enable == true -> cfg.mgr.daemons != [];
312 message = "have to set id of atleast one MGR if you're going to enable MGR";
313 }
314 ];
315
316 warnings = optional (cfg.global.monInitialMembers == null)
317 ''Not setting up a list of members in monInitialMembers requires that you set the host variable for each mon daemon or else the cluster won't function'';
318
319 environment.etc."ceph/ceph.conf".text = let
320 # Translate camelCaseOptions to the expected camel case option for ceph.conf
321 translatedGlobalConfig = mapAttrs' (name: value: nameValuePair (translateOption name) value) cfg.global;
322 # Merge the extraConfig set for mgr daemons, as mgr don't have their own section
323 globalAndMgrConfig = translatedGlobalConfig // optionalAttrs cfg.mgr.enable cfg.mgr.extraConfig;
324 # Remove all name-value pairs with null values from the attribute set to avoid making empty sections in the ceph.conf
325 globalConfig = mapAttrs' (name: value: nameValuePair (translateOption name) value) (filterAttrs (name: value: value != null) globalAndMgrConfig);
326 totalConfig = {
327 "global" = globalConfig;
328 } // optionalAttrs (cfg.mon.enable && cfg.mon.extraConfig != {}) { "mon" = cfg.mon.extraConfig; }
329 // optionalAttrs (cfg.mds.enable && cfg.mds.extraConfig != {}) { "mds" = cfg.mds.extraConfig; }
330 // optionalAttrs (cfg.osd.enable && cfg.osd.extraConfig != {}) { "osd" = cfg.osd.extraConfig; }
331 // optionalAttrs (cfg.client.enable && cfg.client.extraConfig != {}) cfg.client.extraConfig;
332 in
333 generators.toINI {} totalConfig;
334
335 users.users = singleton {
336 name = "ceph";
337 uid = config.ids.uids.ceph;
338 description = "Ceph daemon user";
339 };
340
341 users.groups = singleton {
342 name = "ceph";
343 gid = config.ids.gids.ceph;
344 };
345
346 systemd.services = let
347 services = []
348 ++ optional cfg.mon.enable (generateDaemonList "mon" cfg.mon.daemons { RestartSec = "10"; })
349 ++ optional cfg.mds.enable (generateDaemonList "mds" cfg.mds.daemons { StartLimitBurst = "3"; })
350 ++ optional cfg.osd.enable (generateDaemonList "osd" cfg.osd.daemons { StartLimitBurst = "30"; RestartSec = "20s"; })
351 ++ optional cfg.rgw.enable (generateDaemonList "rgw" cfg.rgw.daemons { })
352 ++ optional cfg.mgr.enable (generateDaemonList "mgr" cfg.mgr.daemons { StartLimitBurst = "3"; });
353 in
354 mkMerge services;
355
356 systemd.targets = let
357 targets = [
358 { "ceph" = { description = "Ceph target allowing to start/stop all ceph service instances at once"; }; }
359 ] ++ optional cfg.mon.enable (generateTargetFile "mon")
360 ++ optional cfg.mds.enable (generateTargetFile "mds")
361 ++ optional cfg.osd.enable (generateTargetFile "osd")
362 ++ optional cfg.rgw.enable (generateTargetFile "rgw")
363 ++ optional cfg.mgr.enable (generateTargetFile "mgr");
364 in
365 mkMerge targets;
366
367 systemd.tmpfiles.rules = [
368 "d /run/ceph 0770 ceph ceph -"
369 ];
370 };
371}