pyrox.dev
1{
2 config,
3 lib,
4 pkgs,
5 ...
6}:
7let
8 cfg = config.services.foundationdb;
9 pkg = cfg.package;
10
11 # used for initial cluster configuration
12 initialIpAddr = if (cfg.publicAddress != "auto") then cfg.publicAddress else "127.0.0.1";
13
14 fdbServers =
15 n:
16 lib.concatStringsSep "\n" (
17 map (x: "[fdbserver.${toString (x + cfg.listenPortStart)}]") (lib.range 0 (n - 1))
18 );
19
20 backupAgents =
21 n: lib.concatStringsSep "\n" (map (x: "[backup_agent.${toString x}]") (lib.range 1 n));
22
23 configFile = pkgs.writeText "foundationdb.conf" ''
24 [general]
25 cluster_file = /etc/foundationdb/fdb.cluster
26
27 [fdbmonitor]
28 restart_delay = ${toString cfg.restartDelay}
29 user = ${cfg.user}
30 group = ${cfg.group}
31
32 [fdbserver]
33 command = ${pkg}/bin/fdbserver
34 public_address = ${cfg.publicAddress}:$ID
35 listen_address = ${cfg.listenAddress}
36 datadir = ${cfg.dataDir}/$ID
37 logdir = ${cfg.logDir}
38 logsize = ${cfg.logSize}
39 maxlogssize = ${cfg.maxLogSize}
40 ${lib.optionalString (cfg.class != null) "class = ${cfg.class}"}
41 memory = ${cfg.memory}
42 storage_memory = ${cfg.storageMemory}
43
44 ${lib.optionalString (lib.versionAtLeast cfg.package.version "6.1") ''
45 trace_format = ${cfg.traceFormat}
46 ''}
47
48 ${lib.optionalString (cfg.tls != null) ''
49 tls_plugin = ${pkg}/libexec/plugins/FDBLibTLS.so
50 tls_certificate_file = ${cfg.tls.certificate}
51 tls_key_file = ${cfg.tls.key}
52 tls_verify_peers = ${cfg.tls.allowedPeers}
53 ''}
54
55 ${lib.optionalString (
56 cfg.locality.machineId != null
57 ) "locality_machineid=${cfg.locality.machineId}"}
58 ${lib.optionalString (cfg.locality.zoneId != null) "locality_zoneid=${cfg.locality.zoneId}"}
59 ${lib.optionalString (
60 cfg.locality.datacenterId != null
61 ) "locality_dcid=${cfg.locality.datacenterId}"}
62 ${lib.optionalString (cfg.locality.dataHall != null) "locality_data_hall=${cfg.locality.dataHall}"}
63
64 ${fdbServers cfg.serverProcesses}
65
66 [backup_agent]
67 command = ${pkg}/libexec/backup_agent
68 ${backupAgents cfg.backupProcesses}
69 '';
70in
71{
72 options.services.foundationdb = {
73
74 enable = lib.mkEnableOption "FoundationDB Server";
75
76 package = lib.mkOption {
77 type = lib.types.package;
78 description = ''
79 The FoundationDB package to use for this server. This must be specified by the user
80 in order to ensure migrations and upgrades are controlled appropriately.
81 '';
82 };
83
84 publicAddress = lib.mkOption {
85 type = lib.types.str;
86 default = "auto";
87 description = "Publicly visible IP address of the process. Port is determined by process ID";
88 };
89
90 listenAddress = lib.mkOption {
91 type = lib.types.str;
92 default = "public";
93 description = "Publicly visible IP address of the process. Port is determined by process ID";
94 };
95
96 listenPortStart = lib.mkOption {
97 type = lib.types.int;
98 default = 4500;
99 description = ''
100 Starting port number for database listening sockets. Every FDB process binds to a
101 subsequent port, to this number reflects the start of the overall range. e.g. having
102 8 server processes will use all ports between 4500 and 4507.
103 '';
104 };
105
106 openFirewall = lib.mkOption {
107 type = lib.types.bool;
108 default = false;
109 description = ''
110 Open the firewall ports corresponding to FoundationDB processes and coordinators
111 using {option}`config.networking.firewall.*`.
112 '';
113 };
114
115 dataDir = lib.mkOption {
116 type = lib.types.path;
117 default = "/var/lib/foundationdb";
118 description = "Data directory. All cluster data will be put under here.";
119 };
120
121 logDir = lib.mkOption {
122 type = lib.types.path;
123 default = "/var/log/foundationdb";
124 description = "Log directory.";
125 };
126
127 user = lib.mkOption {
128 type = lib.types.str;
129 default = "foundationdb";
130 description = "User account under which FoundationDB runs.";
131 };
132
133 group = lib.mkOption {
134 type = lib.types.str;
135 default = "foundationdb";
136 description = "Group account under which FoundationDB runs.";
137 };
138
139 class = lib.mkOption {
140 type = lib.types.nullOr (
141 lib.types.enum [
142 "storage"
143 "transaction"
144 "stateless"
145 ]
146 );
147 default = null;
148 description = "Process class";
149 };
150
151 restartDelay = lib.mkOption {
152 type = lib.types.int;
153 default = 10;
154 description = "Number of seconds to wait before restarting servers.";
155 };
156
157 logSize = lib.mkOption {
158 type = lib.types.str;
159 default = "10MiB";
160 description = ''
161 Roll over to a new log file after the current log file
162 reaches the specified size.
163 '';
164 };
165
166 maxLogSize = lib.mkOption {
167 type = lib.types.str;
168 default = "100MiB";
169 description = ''
170 Delete the oldest log file when the total size of all log
171 files exceeds the specified size. If set to 0, old log files
172 will not be deleted.
173 '';
174 };
175
176 serverProcesses = lib.mkOption {
177 type = lib.types.int;
178 default = 1;
179 description = "Number of fdbserver processes to run.";
180 };
181
182 backupProcesses = lib.mkOption {
183 type = lib.types.int;
184 default = 1;
185 description = "Number of backup_agent processes to run for snapshots.";
186 };
187
188 memory = lib.mkOption {
189 type = lib.types.str;
190 default = "8GiB";
191 description = ''
192 Maximum memory used by the process. The default value is
193 `8GiB`. When specified without a unit,
194 `MiB` is assumed. This parameter does not
195 change the memory allocation of the program. Rather, it sets
196 a hard limit beyond which the process will kill itself and
197 be restarted. The default value of `8GiB`
198 is double the intended memory usage in the default
199 configuration (providing an emergency buffer to deal with
200 memory leaks or similar problems). It is not recommended to
201 decrease the value of this parameter below its default
202 value. It may be increased if you wish to allocate a very
203 large amount of storage engine memory or cache. In
204 particular, when the `storageMemory`
205 parameter is increased, the `memory`
206 parameter should be increased by an equal amount.
207 '';
208 };
209
210 storageMemory = lib.mkOption {
211 type = lib.types.str;
212 default = "1GiB";
213 description = ''
214 Maximum memory used for data storage. The default value is
215 `1GiB`. When specified without a unit,
216 `MB` is assumed. Clusters using the memory
217 storage engine will be restricted to using this amount of
218 memory per process for purposes of data storage. Memory
219 overhead associated with storing the data is counted against
220 this total. If you increase the
221 `storageMemory`, you should also increase
222 the `memory` parameter by the same amount.
223 '';
224 };
225
226 tls = lib.mkOption {
227 default = null;
228 description = ''
229 FoundationDB Transport Security Layer (TLS) settings.
230 '';
231
232 type = lib.types.nullOr (
233 lib.types.submodule ({
234 options = {
235 certificate = lib.mkOption {
236 type = lib.types.str;
237 description = ''
238 Path to the TLS certificate file. This certificate will
239 be offered to, and may be verified by, clients.
240 '';
241 };
242
243 key = lib.mkOption {
244 type = lib.types.str;
245 description = "Private key file for the certificate.";
246 };
247
248 allowedPeers = lib.mkOption {
249 type = lib.types.str;
250 default = "Check.Valid=1,Check.Unexpired=1";
251 description = ''
252 "Peer verification string". This may be used to adjust which TLS
253 client certificates a server will accept, as a form of user
254 authorization; for example, it may only accept TLS clients who
255 offer a certificate abiding by some locality or organization name.
256
257 For more information, please see the FoundationDB documentation.
258 '';
259 };
260 };
261 })
262 );
263 };
264
265 locality = lib.mkOption {
266 default = {
267 machineId = null;
268 zoneId = null;
269 datacenterId = null;
270 dataHall = null;
271 };
272
273 description = ''
274 FoundationDB locality settings.
275 '';
276
277 type = lib.types.submodule ({
278 options = {
279 machineId = lib.mkOption {
280 default = null;
281 type = lib.types.nullOr lib.types.str;
282 description = ''
283 Machine identifier key. All processes on a machine should share a
284 unique id. By default, processes on a machine determine a unique id to share.
285 This does not generally need to be set.
286 '';
287 };
288
289 zoneId = lib.mkOption {
290 default = null;
291 type = lib.types.nullOr lib.types.str;
292 description = ''
293 Zone identifier key. Processes that share a zone id are
294 considered non-unique for the purposes of data replication.
295 If unset, defaults to machine id.
296 '';
297 };
298
299 datacenterId = lib.mkOption {
300 default = null;
301 type = lib.types.nullOr lib.types.str;
302 description = ''
303 Data center identifier key. All processes physically located in a
304 data center should share the id. If you are depending on data
305 center based replication this must be set on all processes.
306 '';
307 };
308
309 dataHall = lib.mkOption {
310 default = null;
311 type = lib.types.nullOr lib.types.str;
312 description = ''
313 Data hall identifier key. All processes physically located in a
314 data hall should share the id. If you are depending on data
315 hall based replication this must be set on all processes.
316 '';
317 };
318 };
319 });
320 };
321
322 extraReadWritePaths = lib.mkOption {
323 default = [ ];
324 type = lib.types.listOf lib.types.path;
325 description = ''
326 An extra set of filesystem paths that FoundationDB can read to
327 and write from. By default, FoundationDB runs under a heavily
328 namespaced systemd environment without write access to most of
329 the filesystem outside of its data and log directories. By
330 adding paths to this list, the set of writeable paths will be
331 expanded. This is useful for allowing e.g. backups to local files,
332 which must be performed on behalf of the foundationdb service.
333 '';
334 };
335
336 pidfile = lib.mkOption {
337 type = lib.types.path;
338 default = "/run/foundationdb.pid";
339 description = "Path to pidfile for fdbmonitor.";
340 };
341
342 traceFormat = lib.mkOption {
343 type = lib.types.enum [
344 "xml"
345 "json"
346 ];
347 default = "xml";
348 description = "Trace logging format.";
349 };
350 };
351
352 config = lib.mkIf cfg.enable {
353 assertions = [
354 {
355 assertion = lib.versionOlder cfg.package.version "6.1" -> cfg.traceFormat == "xml";
356 message =
357 ''
358 Versions of FoundationDB before 6.1 do not support configurable trace formats (only XML is supported).
359 This option has no effect for version ''
360 + cfg.package.version
361 + ''
362 , and enabling it is an error.
363 '';
364 }
365 ];
366
367 environment.systemPackages = [ pkg ];
368
369 users.users = lib.optionalAttrs (cfg.user == "foundationdb") {
370 foundationdb = {
371 description = "FoundationDB User";
372 uid = config.ids.uids.foundationdb;
373 group = cfg.group;
374 };
375 };
376
377 users.groups = lib.optionalAttrs (cfg.group == "foundationdb") {
378 foundationdb.gid = config.ids.gids.foundationdb;
379 };
380
381 networking.firewall.allowedTCPPortRanges = lib.mkIf cfg.openFirewall [
382 {
383 from = cfg.listenPortStart;
384 to = (cfg.listenPortStart + cfg.serverProcesses) - 1;
385 }
386 ];
387
388 systemd.tmpfiles.rules = [
389 "d /etc/foundationdb 0755 ${cfg.user} ${cfg.group} - -"
390 "d '${cfg.dataDir}' 0770 ${cfg.user} ${cfg.group} - -"
391 "d '${cfg.logDir}' 0770 ${cfg.user} ${cfg.group} - -"
392 "F '${cfg.pidfile}' - ${cfg.user} ${cfg.group} - -"
393 ];
394
395 systemd.services.foundationdb = {
396 description = "FoundationDB Service";
397
398 after = [ "network.target" ];
399 wantedBy = [ "multi-user.target" ];
400 unitConfig = {
401 RequiresMountsFor = "${cfg.dataDir} ${cfg.logDir}";
402 };
403
404 serviceConfig =
405 let
406 rwpaths = [
407 cfg.dataDir
408 cfg.logDir
409 cfg.pidfile
410 "/etc/foundationdb"
411 ] ++ cfg.extraReadWritePaths;
412 in
413 {
414 Type = "simple";
415 Restart = "always";
416 RestartSec = 5;
417 User = cfg.user;
418 Group = cfg.group;
419 PIDFile = "${cfg.pidfile}";
420
421 PermissionsStartOnly = true; # setup needs root perms
422 TimeoutSec = 120; # give reasonable time to shut down
423
424 # Security options
425 NoNewPrivileges = true;
426 ProtectHome = true;
427 ProtectSystem = "strict";
428 ProtectKernelTunables = true;
429 ProtectControlGroups = true;
430 PrivateTmp = true;
431 PrivateDevices = true;
432 ReadWritePaths = lib.concatStringsSep " " (map (x: "-" + x) rwpaths);
433 };
434
435 path = [
436 pkg
437 pkgs.coreutils
438 ];
439
440 preStart = ''
441 if [ ! -f /etc/foundationdb/fdb.cluster ]; then
442 cf=/etc/foundationdb/fdb.cluster
443 desc=$(tr -dc A-Za-z0-9 </dev/urandom 2>/dev/null | head -c8)
444 rand=$(tr -dc A-Za-z0-9 </dev/urandom 2>/dev/null | head -c8)
445 echo ''${desc}:''${rand}@${initialIpAddr}:${builtins.toString cfg.listenPortStart} > $cf
446 chmod 0664 $cf
447 touch "${cfg.dataDir}/.first_startup"
448 fi
449 '';
450
451 script = "exec fdbmonitor --lockfile ${cfg.pidfile} --conffile ${configFile}";
452
453 postStart = ''
454 if [ -e "${cfg.dataDir}/.first_startup" ]; then
455 fdbcli --exec "configure new single ssd"
456 rm -f "${cfg.dataDir}/.first_startup";
457 fi
458 '';
459 };
460 };
461
462 meta.doc = ./foundationdb.md;
463 meta.maintainers = with lib.maintainers; [ thoughtpolice ];
464}