pyrox.dev
1{
2 config,
3 lib,
4 pkgs,
5 ...
6}:
7let
8
9 cfg = config.services.xtreemfs;
10
11 xtreemfs = pkgs.xtreemfs;
12
13 home = cfg.homeDir;
14
15 startupScript =
16 class: configPath:
17 pkgs.writeScript "xtreemfs-osd.sh" ''
18 #! ${pkgs.runtimeShell}
19 JAVA_HOME="${pkgs.jdk}"
20 JAVADIR="${xtreemfs}/share/java"
21 JAVA_CALL="$JAVA_HOME/bin/java -ea -cp $JAVADIR/XtreemFS.jar:$JAVADIR/BabuDB.jar:$JAVADIR/Flease.jar:$JAVADIR/protobuf-java-2.5.0.jar:$JAVADIR/Foundation.jar:$JAVADIR/jdmkrt.jar:$JAVADIR/jdmktk.jar:$JAVADIR/commons-codec-1.3.jar"
22 $JAVA_CALL ${class} ${configPath}
23 '';
24
25 dirReplicationConfig = pkgs.writeText "xtreemfs-dir-replication-plugin.properties" ''
26 babudb.repl.backupDir = ${home}/server-repl-dir
27 plugin.jar = ${xtreemfs}/share/java/BabuDB_replication_plugin.jar
28 babudb.repl.dependency.0 = ${xtreemfs}/share/java/Flease.jar
29
30 ${cfg.dir.replication.extraConfig}
31 '';
32
33 dirConfig = pkgs.writeText "xtreemfs-dir-config.properties" ''
34 uuid = ${cfg.dir.uuid}
35 listen.port = ${toString cfg.dir.port}
36 ${lib.optionalString (cfg.dir.address != "") "listen.address = ${cfg.dir.address}"}
37 http_port = ${toString cfg.dir.httpPort}
38 babudb.baseDir = ${home}/dir/database
39 babudb.logDir = ${home}/dir/db-log
40 babudb.sync = ${if cfg.dir.replication.enable then "FDATASYNC" else cfg.dir.syncMode}
41
42 ${lib.optionalString cfg.dir.replication.enable "babudb.plugin.0 = ${dirReplicationConfig}"}
43
44 ${cfg.dir.extraConfig}
45 '';
46
47 mrcReplicationConfig = pkgs.writeText "xtreemfs-mrc-replication-plugin.properties" ''
48 babudb.repl.backupDir = ${home}/server-repl-mrc
49 plugin.jar = ${xtreemfs}/share/java/BabuDB_replication_plugin.jar
50 babudb.repl.dependency.0 = ${xtreemfs}/share/java/Flease.jar
51
52 ${cfg.mrc.replication.extraConfig}
53 '';
54
55 mrcConfig = pkgs.writeText "xtreemfs-mrc-config.properties" ''
56 uuid = ${cfg.mrc.uuid}
57 listen.port = ${toString cfg.mrc.port}
58 ${lib.optionalString (cfg.mrc.address != "") "listen.address = ${cfg.mrc.address}"}
59 http_port = ${toString cfg.mrc.httpPort}
60 babudb.baseDir = ${home}/mrc/database
61 babudb.logDir = ${home}/mrc/db-log
62 babudb.sync = ${if cfg.mrc.replication.enable then "FDATASYNC" else cfg.mrc.syncMode}
63
64 ${lib.optionalString cfg.mrc.replication.enable "babudb.plugin.0 = ${mrcReplicationConfig}"}
65
66 ${cfg.mrc.extraConfig}
67 '';
68
69 osdConfig = pkgs.writeText "xtreemfs-osd-config.properties" ''
70 uuid = ${cfg.osd.uuid}
71 listen.port = ${toString cfg.osd.port}
72 ${lib.optionalString (cfg.osd.address != "") "listen.address = ${cfg.osd.address}"}
73 http_port = ${toString cfg.osd.httpPort}
74 object_dir = ${home}/osd/
75
76 ${cfg.osd.extraConfig}
77 '';
78
79 optionalDir = lib.optionals cfg.dir.enable [ "xtreemfs-dir.service" ];
80
81 systemdOptionalDependencies = {
82 after = [ "network.target" ] ++ optionalDir;
83 wantedBy = [ "multi-user.target" ] ++ optionalDir;
84 };
85
86in
87
88{
89
90 ###### interface
91
92 options = {
93
94 services.xtreemfs = {
95
96 enable = lib.mkEnableOption "XtreemFS";
97
98 homeDir = lib.mkOption {
99 type = lib.types.path;
100 default = "/var/lib/xtreemfs";
101 description = ''
102 XtreemFS home dir for the xtreemfs user.
103 '';
104 };
105
106 dir = {
107 enable = lib.mkOption {
108 type = lib.types.bool;
109 default = true;
110 description = ''
111 Whether to enable XtreemFS DIR service.
112 '';
113 };
114
115 uuid = lib.mkOption {
116 example = "eacb6bab-f444-4ebf-a06a-3f72d7465e40";
117 type = lib.types.str;
118 description = ''
119 Must be set to a unique identifier, preferably a UUID according to
120 RFC 4122. UUIDs can be generated with `uuidgen` command, found in
121 the `util-linux` package.
122 '';
123 };
124 port = lib.mkOption {
125 default = 32638;
126 type = lib.types.port;
127 description = ''
128 The port to listen on for incoming connections (TCP).
129 '';
130 };
131 address = lib.mkOption {
132 type = lib.types.str;
133 example = "127.0.0.1";
134 default = "";
135 description = ''
136 If specified, it defines the interface to listen on. If not
137 specified, the service will listen on all interfaces (any).
138 '';
139 };
140 httpPort = lib.mkOption {
141 default = 30638;
142 type = lib.types.port;
143 description = ''
144 Specifies the listen port for the HTTP service that returns the
145 status page.
146 '';
147 };
148 syncMode = lib.mkOption {
149 type = lib.types.enum [
150 "ASYNC"
151 "SYNC_WRITE_METADATA"
152 "SYNC_WRITE"
153 "FDATASYNC"
154 "FSYNC"
155 ];
156 default = "FSYNC";
157 example = "FDATASYNC";
158 description = ''
159 The sync mode influences how operations are committed to the disk
160 log before the operation is acknowledged to the caller.
161
162 -ASYNC mode the writes to the disk log are buffered in memory by the operating system. This is the fastest mode but will lead to data loss in case of a crash, kernel panic or power failure.
163 -SYNC_WRITE_METADATA opens the file with O_SYNC, the system will not buffer any writes. The operation will be acknowledged when data has been safely written to disk. This mode is slow but offers maximum data safety. However, BabuDB cannot influence the disk drive caches, this depends on the OS and hard disk model.
164 -SYNC_WRITE similar to SYNC_WRITE_METADATA but opens file with O_DSYNC which means that only the data is commit to disk. This can lead to some data loss depending on the implementation of the underlying file system. Linux does not implement this mode.
165 -FDATASYNC is similar to SYNC_WRITE but opens the file in asynchronous mode and calls fdatasync() after writing the data to disk.
166 -FSYNC is similar to SYNC_WRITE_METADATA but opens the file in asynchronous mode and calls fsync() after writing the data to disk.
167
168 For best throughput use ASYNC, for maximum data safety use FSYNC.
169
170 (If xtreemfs.dir.replication.enable is true then FDATASYNC is forced)
171 '';
172 };
173 extraConfig = lib.mkOption {
174 type = lib.types.lines;
175 default = "";
176 example = ''
177 # specify whether SSL is required
178 ssl.enabled = true
179 ssl.service_creds.pw = passphrase
180 ssl.service_creds.container = pkcs12
181 ssl.service_creds = /etc/xos/xtreemfs/truststore/certs/dir.p12
182 ssl.trusted_certs = /etc/xos/xtreemfs/truststore/certs/trusted.jks
183 ssl.trusted_certs.pw = jks_passphrase
184 ssl.trusted_certs.container = jks
185 '';
186 description = ''
187 Configuration of XtreemFS DIR service.
188 WARNING: configuration is saved as plaintext inside nix store.
189 For more options: https://www.xtreemfs.org/xtfs-guide-1.5.1/index.html
190 '';
191 };
192 replication = {
193 enable = lib.mkEnableOption "XtreemFS DIR replication plugin";
194 extraConfig = lib.mkOption {
195 type = lib.types.lines;
196 example = ''
197 # participants of the replication including this replica
198 babudb.repl.participant.0 = 192.168.0.10
199 babudb.repl.participant.0.port = 35676
200 babudb.repl.participant.1 = 192.168.0.11
201 babudb.repl.participant.1.port = 35676
202 babudb.repl.participant.2 = 192.168.0.12
203 babudb.repl.participant.2.port = 35676
204
205 # number of servers that at least have to be up to date
206 # To have a fault-tolerant system, this value has to be set to the
207 # majority of nodes i.e., if you have three replicas, set this to 2
208 # Please note that a setup with two nodes provides no fault-tolerance.
209 babudb.repl.sync.n = 2
210
211 # specify whether SSL is required
212 babudb.ssl.enabled = true
213
214 babudb.ssl.protocol = tlsv12
215
216 # server credentials for SSL handshakes
217 babudb.ssl.service_creds = /etc/xos/xtreemfs/truststore/certs/osd.p12
218 babudb.ssl.service_creds.pw = passphrase
219 babudb.ssl.service_creds.container = pkcs12
220
221 # trusted certificates for SSL handshakes
222 babudb.ssl.trusted_certs = /etc/xos/xtreemfs/truststore/certs/trusted.jks
223 babudb.ssl.trusted_certs.pw = jks_passphrase
224 babudb.ssl.trusted_certs.container = jks
225
226 babudb.ssl.authenticationWithoutEncryption = false
227 '';
228 description = ''
229 Configuration of XtreemFS DIR replication plugin.
230 WARNING: configuration is saved as plaintext inside nix store.
231 For more options: https://www.xtreemfs.org/xtfs-guide-1.5.1/index.html
232 '';
233 };
234 };
235 };
236
237 mrc = {
238 enable = lib.mkOption {
239 type = lib.types.bool;
240 default = true;
241 description = ''
242 Whether to enable XtreemFS MRC service.
243 '';
244 };
245
246 uuid = lib.mkOption {
247 example = "eacb6bab-f444-4ebf-a06a-3f72d7465e41";
248 type = lib.types.str;
249 description = ''
250 Must be set to a unique identifier, preferably a UUID according to
251 RFC 4122. UUIDs can be generated with `uuidgen` command, found in
252 the `util-linux` package.
253 '';
254 };
255 port = lib.mkOption {
256 default = 32636;
257 type = lib.types.port;
258 description = ''
259 The port to listen on for incoming connections (TCP).
260 '';
261 };
262 address = lib.mkOption {
263 example = "127.0.0.1";
264 type = lib.types.str;
265 default = "";
266 description = ''
267 If specified, it defines the interface to listen on. If not
268 specified, the service will listen on all interfaces (any).
269 '';
270 };
271 httpPort = lib.mkOption {
272 default = 30636;
273 type = lib.types.port;
274 description = ''
275 Specifies the listen port for the HTTP service that returns the
276 status page.
277 '';
278 };
279 syncMode = lib.mkOption {
280 default = "FSYNC";
281 type = lib.types.enum [
282 "ASYNC"
283 "SYNC_WRITE_METADATA"
284 "SYNC_WRITE"
285 "FDATASYNC"
286 "FSYNC"
287 ];
288 example = "FDATASYNC";
289 description = ''
290 The sync mode influences how operations are committed to the disk
291 log before the operation is acknowledged to the caller.
292
293 -ASYNC mode the writes to the disk log are buffered in memory by the operating system. This is the fastest mode but will lead to data loss in case of a crash, kernel panic or power failure.
294 -SYNC_WRITE_METADATA opens the file with O_SYNC, the system will not buffer any writes. The operation will be acknowledged when data has been safely written to disk. This mode is slow but offers maximum data safety. However, BabuDB cannot influence the disk drive caches, this depends on the OS and hard disk model.
295 -SYNC_WRITE similar to SYNC_WRITE_METADATA but opens file with O_DSYNC which means that only the data is commit to disk. This can lead to some data loss depending on the implementation of the underlying file system. Linux does not implement this mode.
296 -FDATASYNC is similar to SYNC_WRITE but opens the file in asynchronous mode and calls fdatasync() after writing the data to disk.
297 -FSYNC is similar to SYNC_WRITE_METADATA but opens the file in asynchronous mode and calls fsync() after writing the data to disk.
298
299 For best throughput use ASYNC, for maximum data safety use FSYNC.
300
301 (If xtreemfs.mrc.replication.enable is true then FDATASYNC is forced)
302 '';
303 };
304 extraConfig = lib.mkOption {
305 type = lib.types.lines;
306 example = ''
307 osd_check_interval = 300
308 no_atime = true
309 local_clock_renewal = 0
310 remote_time_sync = 30000
311 authentication_provider = org.xtreemfs.common.auth.NullAuthProvider
312
313 # shared secret between the MRC and all OSDs
314 capability_secret = iNG8UuQJrJ6XVDTe
315
316 dir_service.host = 192.168.0.10
317 dir_service.port = 32638
318
319 # if replication is enabled
320 dir_service.1.host = 192.168.0.11
321 dir_service.1.port = 32638
322 dir_service.2.host = 192.168.0.12
323 dir_service.2.port = 32638
324
325 # specify whether SSL is required
326 ssl.enabled = true
327 ssl.protocol = tlsv12
328 ssl.service_creds.pw = passphrase
329 ssl.service_creds.container = pkcs12
330 ssl.service_creds = /etc/xos/xtreemfs/truststore/certs/mrc.p12
331 ssl.trusted_certs = /etc/xos/xtreemfs/truststore/certs/trusted.jks
332 ssl.trusted_certs.pw = jks_passphrase
333 ssl.trusted_certs.container = jks
334 '';
335 description = ''
336 Configuration of XtreemFS MRC service.
337 WARNING: configuration is saved as plaintext inside nix store.
338 For more options: https://www.xtreemfs.org/xtfs-guide-1.5.1/index.html
339 '';
340 };
341 replication = {
342 enable = lib.mkEnableOption "XtreemFS MRC replication plugin";
343 extraConfig = lib.mkOption {
344 type = lib.types.lines;
345 example = ''
346 # participants of the replication including this replica
347 babudb.repl.participant.0 = 192.168.0.10
348 babudb.repl.participant.0.port = 35678
349 babudb.repl.participant.1 = 192.168.0.11
350 babudb.repl.participant.1.port = 35678
351 babudb.repl.participant.2 = 192.168.0.12
352 babudb.repl.participant.2.port = 35678
353
354 # number of servers that at least have to be up to date
355 # To have a fault-tolerant system, this value has to be set to the
356 # majority of nodes i.e., if you have three replicas, set this to 2
357 # Please note that a setup with two nodes provides no fault-tolerance.
358 babudb.repl.sync.n = 2
359
360 # specify whether SSL is required
361 babudb.ssl.enabled = true
362
363 babudb.ssl.protocol = tlsv12
364
365 # server credentials for SSL handshakes
366 babudb.ssl.service_creds = /etc/xos/xtreemfs/truststore/certs/osd.p12
367 babudb.ssl.service_creds.pw = passphrase
368 babudb.ssl.service_creds.container = pkcs12
369
370 # trusted certificates for SSL handshakes
371 babudb.ssl.trusted_certs = /etc/xos/xtreemfs/truststore/certs/trusted.jks
372 babudb.ssl.trusted_certs.pw = jks_passphrase
373 babudb.ssl.trusted_certs.container = jks
374
375 babudb.ssl.authenticationWithoutEncryption = false
376 '';
377 description = ''
378 Configuration of XtreemFS MRC replication plugin.
379 WARNING: configuration is saved as plaintext inside nix store.
380 For more options: https://www.xtreemfs.org/xtfs-guide-1.5.1/index.html
381 '';
382 };
383 };
384 };
385
386 osd = {
387 enable = lib.mkOption {
388 type = lib.types.bool;
389 default = true;
390 description = ''
391 Whether to enable XtreemFS OSD service.
392 '';
393 };
394
395 uuid = lib.mkOption {
396 example = "eacb6bab-f444-4ebf-a06a-3f72d7465e42";
397 type = lib.types.str;
398 description = ''
399 Must be set to a unique identifier, preferably a UUID according to
400 RFC 4122. UUIDs can be generated with `uuidgen` command, found in
401 the `util-linux` package.
402 '';
403 };
404 port = lib.mkOption {
405 default = 32640;
406 type = lib.types.port;
407 description = ''
408 The port to listen on for incoming connections (TCP and UDP).
409 '';
410 };
411 address = lib.mkOption {
412 example = "127.0.0.1";
413 type = lib.types.str;
414 default = "";
415 description = ''
416 If specified, it defines the interface to listen on. If not
417 specified, the service will listen on all interfaces (any).
418 '';
419 };
420 httpPort = lib.mkOption {
421 default = 30640;
422 type = lib.types.port;
423 description = ''
424 Specifies the listen port for the HTTP service that returns the
425 status page.
426 '';
427 };
428 extraConfig = lib.mkOption {
429 type = lib.types.lines;
430 example = ''
431 local_clock_renewal = 0
432 remote_time_sync = 30000
433 report_free_space = true
434 capability_secret = iNG8UuQJrJ6XVDTe
435
436 dir_service.host = 192.168.0.10
437 dir_service.port = 32638
438
439 # if replication is used
440 dir_service.1.host = 192.168.0.11
441 dir_service.1.port = 32638
442 dir_service.2.host = 192.168.0.12
443 dir_service.2.port = 32638
444
445 # specify whether SSL is required
446 ssl.enabled = true
447 ssl.service_creds.pw = passphrase
448 ssl.service_creds.container = pkcs12
449 ssl.service_creds = /etc/xos/xtreemfs/truststore/certs/osd.p12
450 ssl.trusted_certs = /etc/xos/xtreemfs/truststore/certs/trusted.jks
451 ssl.trusted_certs.pw = jks_passphrase
452 ssl.trusted_certs.container = jks
453 '';
454 description = ''
455 Configuration of XtreemFS OSD service.
456 WARNING: configuration is saved as plaintext inside nix store.
457 For more options: https://www.xtreemfs.org/xtfs-guide-1.5.1/index.html
458 '';
459 };
460 };
461 };
462
463 };
464
465 ###### implementation
466
467 config = lib.mkIf cfg.enable {
468
469 environment.systemPackages = [ xtreemfs ];
470
471 users.users.xtreemfs = {
472 uid = config.ids.uids.xtreemfs;
473 description = "XtreemFS user";
474 createHome = true;
475 home = home;
476 };
477
478 users.groups.xtreemfs = {
479 gid = config.ids.gids.xtreemfs;
480 };
481
482 systemd.services.xtreemfs-dir = lib.mkIf cfg.dir.enable {
483 description = "XtreemFS-DIR Server";
484 after = [ "network.target" ];
485 wantedBy = [ "multi-user.target" ];
486 serviceConfig = {
487 User = "xtreemfs";
488 ExecStart = "${startupScript "org.xtreemfs.dir.DIR" dirConfig}";
489 };
490 };
491
492 systemd.services.xtreemfs-mrc = lib.mkIf cfg.mrc.enable (
493 {
494 description = "XtreemFS-MRC Server";
495 serviceConfig = {
496 User = "xtreemfs";
497 ExecStart = "${startupScript "org.xtreemfs.mrc.MRC" mrcConfig}";
498 };
499 }
500 // systemdOptionalDependencies
501 );
502
503 systemd.services.xtreemfs-osd = lib.mkIf cfg.osd.enable (
504 {
505 description = "XtreemFS-OSD Server";
506 serviceConfig = {
507 User = "xtreemfs";
508 ExecStart = "${startupScript "org.xtreemfs.osd.OSD" osdConfig}";
509 };
510 }
511 // systemdOptionalDependencies
512 );
513
514 };
515
516}