pyrox.dev
1{ config, lib, pkgs, ... }:
2
3with lib;
4
5let
6 cfg = config.services.foundationdb;
7 pkg = cfg.package;
8
9 # used for initial cluster configuration
10 initialIpAddr = if (cfg.publicAddress != "auto") then cfg.publicAddress else "127.0.0.1";
11
12 fdbServers = n:
13 concatStringsSep "\n" (map (x: "[fdbserver.${toString (x+cfg.listenPortStart)}]") (range 0 (n - 1)));
14
15 backupAgents = n:
16 concatStringsSep "\n" (map (x: "[backup_agent.${toString x}]") (range 1 n));
17
18 configFile = pkgs.writeText "foundationdb.conf" ''
19 [general]
20 cluster_file = /etc/foundationdb/fdb.cluster
21
22 [fdbmonitor]
23 restart_delay = ${toString cfg.restartDelay}
24 user = ${cfg.user}
25 group = ${cfg.group}
26
27 [fdbserver]
28 command = ${pkg}/bin/fdbserver
29 public_address = ${cfg.publicAddress}:$ID
30 listen_address = ${cfg.listenAddress}
31 datadir = ${cfg.dataDir}/$ID
32 logdir = ${cfg.logDir}
33 logsize = ${cfg.logSize}
34 maxlogssize = ${cfg.maxLogSize}
35 ${optionalString (cfg.class != null) "class = ${cfg.class}"}
36 memory = ${cfg.memory}
37 storage_memory = ${cfg.storageMemory}
38
39 ${optionalString (lib.versionAtLeast cfg.package.version "6.1") ''
40 trace_format = ${cfg.traceFormat}
41 ''}
42
43 ${optionalString (cfg.tls != null) ''
44 tls_plugin = ${pkg}/libexec/plugins/FDBLibTLS.so
45 tls_certificate_file = ${cfg.tls.certificate}
46 tls_key_file = ${cfg.tls.key}
47 tls_verify_peers = ${cfg.tls.allowedPeers}
48 ''}
49
50 ${optionalString (cfg.locality.machineId != null) "locality_machineid=${cfg.locality.machineId}"}
51 ${optionalString (cfg.locality.zoneId != null) "locality_zoneid=${cfg.locality.zoneId}"}
52 ${optionalString (cfg.locality.datacenterId != null) "locality_dcid=${cfg.locality.datacenterId}"}
53 ${optionalString (cfg.locality.dataHall != null) "locality_data_hall=${cfg.locality.dataHall}"}
54
55 ${fdbServers cfg.serverProcesses}
56
57 [backup_agent]
58 command = ${pkg}/libexec/backup_agent
59 ${backupAgents cfg.backupProcesses}
60 '';
61in
62{
63 options.services.foundationdb = {
64
65 enable = mkEnableOption (lib.mdDoc "FoundationDB Server");
66
67 package = mkOption {
68 type = types.package;
69 description = lib.mdDoc ''
70 The FoundationDB package to use for this server. This must be specified by the user
71 in order to ensure migrations and upgrades are controlled appropriately.
72 '';
73 };
74
75 publicAddress = mkOption {
76 type = types.str;
77 default = "auto";
78 description = lib.mdDoc "Publicly visible IP address of the process. Port is determined by process ID";
79 };
80
81 listenAddress = mkOption {
82 type = types.str;
83 default = "public";
84 description = lib.mdDoc "Publicly visible IP address of the process. Port is determined by process ID";
85 };
86
87 listenPortStart = mkOption {
88 type = types.int;
89 default = 4500;
90 description = lib.mdDoc ''
91 Starting port number for database listening sockets. Every FDB process binds to a
92 subsequent port, to this number reflects the start of the overall range. e.g. having
93 8 server processes will use all ports between 4500 and 4507.
94 '';
95 };
96
97 openFirewall = mkOption {
98 type = types.bool;
99 default = false;
100 description = lib.mdDoc ''
101 Open the firewall ports corresponding to FoundationDB processes and coordinators
102 using {option}`config.networking.firewall.*`.
103 '';
104 };
105
106 dataDir = mkOption {
107 type = types.path;
108 default = "/var/lib/foundationdb";
109 description = lib.mdDoc "Data directory. All cluster data will be put under here.";
110 };
111
112 logDir = mkOption {
113 type = types.path;
114 default = "/var/log/foundationdb";
115 description = lib.mdDoc "Log directory.";
116 };
117
118 user = mkOption {
119 type = types.str;
120 default = "foundationdb";
121 description = lib.mdDoc "User account under which FoundationDB runs.";
122 };
123
124 group = mkOption {
125 type = types.str;
126 default = "foundationdb";
127 description = lib.mdDoc "Group account under which FoundationDB runs.";
128 };
129
130 class = mkOption {
131 type = types.nullOr (types.enum [ "storage" "transaction" "stateless" ]);
132 default = null;
133 description = lib.mdDoc "Process class";
134 };
135
136 restartDelay = mkOption {
137 type = types.int;
138 default = 10;
139 description = lib.mdDoc "Number of seconds to wait before restarting servers.";
140 };
141
142 logSize = mkOption {
143 type = types.str;
144 default = "10MiB";
145 description = lib.mdDoc ''
146 Roll over to a new log file after the current log file
147 reaches the specified size.
148 '';
149 };
150
151 maxLogSize = mkOption {
152 type = types.str;
153 default = "100MiB";
154 description = lib.mdDoc ''
155 Delete the oldest log file when the total size of all log
156 files exceeds the specified size. If set to 0, old log files
157 will not be deleted.
158 '';
159 };
160
161 serverProcesses = mkOption {
162 type = types.int;
163 default = 1;
164 description = lib.mdDoc "Number of fdbserver processes to run.";
165 };
166
167 backupProcesses = mkOption {
168 type = types.int;
169 default = 1;
170 description = lib.mdDoc "Number of backup_agent processes to run for snapshots.";
171 };
172
173 memory = mkOption {
174 type = types.str;
175 default = "8GiB";
176 description = lib.mdDoc ''
177 Maximum memory used by the process. The default value is
178 `8GiB`. When specified without a unit,
179 `MiB` is assumed. This parameter does not
180 change the memory allocation of the program. Rather, it sets
181 a hard limit beyond which the process will kill itself and
182 be restarted. The default value of `8GiB`
183 is double the intended memory usage in the default
184 configuration (providing an emergency buffer to deal with
185 memory leaks or similar problems). It is not recommended to
186 decrease the value of this parameter below its default
187 value. It may be increased if you wish to allocate a very
188 large amount of storage engine memory or cache. In
189 particular, when the `storageMemory`
190 parameter is increased, the `memory`
191 parameter should be increased by an equal amount.
192 '';
193 };
194
195 storageMemory = mkOption {
196 type = types.str;
197 default = "1GiB";
198 description = lib.mdDoc ''
199 Maximum memory used for data storage. The default value is
200 `1GiB`. When specified without a unit,
201 `MB` is assumed. Clusters using the memory
202 storage engine will be restricted to using this amount of
203 memory per process for purposes of data storage. Memory
204 overhead associated with storing the data is counted against
205 this total. If you increase the
206 `storageMemory`, you should also increase
207 the `memory` parameter by the same amount.
208 '';
209 };
210
211 tls = mkOption {
212 default = null;
213 description = lib.mdDoc ''
214 FoundationDB Transport Security Layer (TLS) settings.
215 '';
216
217 type = types.nullOr (types.submodule ({
218 options = {
219 certificate = mkOption {
220 type = types.str;
221 description = lib.mdDoc ''
222 Path to the TLS certificate file. This certificate will
223 be offered to, and may be verified by, clients.
224 '';
225 };
226
227 key = mkOption {
228 type = types.str;
229 description = lib.mdDoc "Private key file for the certificate.";
230 };
231
232 allowedPeers = mkOption {
233 type = types.str;
234 default = "Check.Valid=1,Check.Unexpired=1";
235 description = lib.mdDoc ''
236 "Peer verification string". This may be used to adjust which TLS
237 client certificates a server will accept, as a form of user
238 authorization; for example, it may only accept TLS clients who
239 offer a certificate abiding by some locality or organization name.
240
241 For more information, please see the FoundationDB documentation.
242 '';
243 };
244 };
245 }));
246 };
247
248 locality = mkOption {
249 default = {
250 machineId = null;
251 zoneId = null;
252 datacenterId = null;
253 dataHall = null;
254 };
255
256 description = lib.mdDoc ''
257 FoundationDB locality settings.
258 '';
259
260 type = types.submodule ({
261 options = {
262 machineId = mkOption {
263 default = null;
264 type = types.nullOr types.str;
265 description = lib.mdDoc ''
266 Machine identifier key. All processes on a machine should share a
267 unique id. By default, processes on a machine determine a unique id to share.
268 This does not generally need to be set.
269 '';
270 };
271
272 zoneId = mkOption {
273 default = null;
274 type = types.nullOr types.str;
275 description = lib.mdDoc ''
276 Zone identifier key. Processes that share a zone id are
277 considered non-unique for the purposes of data replication.
278 If unset, defaults to machine id.
279 '';
280 };
281
282 datacenterId = mkOption {
283 default = null;
284 type = types.nullOr types.str;
285 description = lib.mdDoc ''
286 Data center identifier key. All processes physically located in a
287 data center should share the id. If you are depending on data
288 center based replication this must be set on all processes.
289 '';
290 };
291
292 dataHall = mkOption {
293 default = null;
294 type = types.nullOr types.str;
295 description = lib.mdDoc ''
296 Data hall identifier key. All processes physically located in a
297 data hall should share the id. If you are depending on data
298 hall based replication this must be set on all processes.
299 '';
300 };
301 };
302 });
303 };
304
305 extraReadWritePaths = mkOption {
306 default = [ ];
307 type = types.listOf types.path;
308 description = lib.mdDoc ''
309 An extra set of filesystem paths that FoundationDB can read to
310 and write from. By default, FoundationDB runs under a heavily
311 namespaced systemd environment without write access to most of
312 the filesystem outside of its data and log directories. By
313 adding paths to this list, the set of writeable paths will be
314 expanded. This is useful for allowing e.g. backups to local files,
315 which must be performed on behalf of the foundationdb service.
316 '';
317 };
318
319 pidfile = mkOption {
320 type = types.path;
321 default = "/run/foundationdb.pid";
322 description = lib.mdDoc "Path to pidfile for fdbmonitor.";
323 };
324
325 traceFormat = mkOption {
326 type = types.enum [ "xml" "json" ];
327 default = "xml";
328 description = lib.mdDoc "Trace logging format.";
329 };
330 };
331
332 config = mkIf cfg.enable {
333 assertions = [
334 { assertion = lib.versionOlder cfg.package.version "6.1" -> cfg.traceFormat == "xml";
335 message = ''
336 Versions of FoundationDB before 6.1 do not support configurable trace formats (only XML is supported).
337 This option has no effect for version '' + cfg.package.version + '', and enabling it is an error.
338 '';
339 }
340 ];
341
342 environment.systemPackages = [ pkg ];
343
344 users.users = optionalAttrs (cfg.user == "foundationdb") {
345 foundationdb = {
346 description = "FoundationDB User";
347 uid = config.ids.uids.foundationdb;
348 group = cfg.group;
349 };
350 };
351
352 users.groups = optionalAttrs (cfg.group == "foundationdb") {
353 foundationdb.gid = config.ids.gids.foundationdb;
354 };
355
356 networking.firewall.allowedTCPPortRanges = mkIf cfg.openFirewall
357 [ { from = cfg.listenPortStart;
358 to = (cfg.listenPortStart + cfg.serverProcesses) - 1;
359 }
360 ];
361
362 systemd.tmpfiles.rules = [
363 "d /etc/foundationdb 0755 ${cfg.user} ${cfg.group} - -"
364 "d '${cfg.dataDir}' 0770 ${cfg.user} ${cfg.group} - -"
365 "d '${cfg.logDir}' 0770 ${cfg.user} ${cfg.group} - -"
366 "F '${cfg.pidfile}' - ${cfg.user} ${cfg.group} - -"
367 ];
368
369 systemd.services.foundationdb = {
370 description = "FoundationDB Service";
371
372 after = [ "network.target" ];
373 wantedBy = [ "multi-user.target" ];
374 unitConfig =
375 { RequiresMountsFor = "${cfg.dataDir} ${cfg.logDir}";
376 };
377
378 serviceConfig =
379 let rwpaths = [ cfg.dataDir cfg.logDir cfg.pidfile "/etc/foundationdb" ]
380 ++ cfg.extraReadWritePaths;
381 in
382 { Type = "simple";
383 Restart = "always";
384 RestartSec = 5;
385 User = cfg.user;
386 Group = cfg.group;
387 PIDFile = "${cfg.pidfile}";
388
389 PermissionsStartOnly = true; # setup needs root perms
390 TimeoutSec = 120; # give reasonable time to shut down
391
392 # Security options
393 NoNewPrivileges = true;
394 ProtectHome = true;
395 ProtectSystem = "strict";
396 ProtectKernelTunables = true;
397 ProtectControlGroups = true;
398 PrivateTmp = true;
399 PrivateDevices = true;
400 ReadWritePaths = lib.concatStringsSep " " (map (x: "-" + x) rwpaths);
401 };
402
403 path = [ pkg pkgs.coreutils ];
404
405 preStart = ''
406 if [ ! -f /etc/foundationdb/fdb.cluster ]; then
407 cf=/etc/foundationdb/fdb.cluster
408 desc=$(tr -dc A-Za-z0-9 </dev/urandom 2>/dev/null | head -c8)
409 rand=$(tr -dc A-Za-z0-9 </dev/urandom 2>/dev/null | head -c8)
410 echo ''${desc}:''${rand}@${initialIpAddr}:${builtins.toString cfg.listenPortStart} > $cf
411 chmod 0664 $cf
412 touch "${cfg.dataDir}/.first_startup"
413 fi
414 '';
415
416 script = "exec fdbmonitor --lockfile ${cfg.pidfile} --conffile ${configFile}";
417
418 postStart = ''
419 if [ -e "${cfg.dataDir}/.first_startup" ]; then
420 fdbcli --exec "configure new single ssd"
421 rm -f "${cfg.dataDir}/.first_startup";
422 fi
423 '';
424 };
425 };
426
427 meta.doc = ./foundationdb.md;
428 meta.maintainers = with lib.maintainers; [ thoughtpolice ];
429}