pyrox.dev
1{ config, lib, pkgs, ... }:
2
3with lib;
4
5let
6
7 cfg = config.services.xtreemfs;
8
9 xtreemfs = pkgs.xtreemfs;
10
11 home = cfg.homeDir;
12
13 startupScript = class: configPath: pkgs.writeScript "xtreemfs-osd.sh" ''
14 #! ${pkgs.runtimeShell}
15 JAVA_HOME="${pkgs.jdk}"
16 JAVADIR="${xtreemfs}/share/java"
17 JAVA_CALL="$JAVA_HOME/bin/java -ea -cp $JAVADIR/XtreemFS.jar:$JAVADIR/BabuDB.jar:$JAVADIR/Flease.jar:$JAVADIR/protobuf-java-2.5.0.jar:$JAVADIR/Foundation.jar:$JAVADIR/jdmkrt.jar:$JAVADIR/jdmktk.jar:$JAVADIR/commons-codec-1.3.jar"
18 $JAVA_CALL ${class} ${configPath}
19 '';
20
21 dirReplicationConfig = pkgs.writeText "xtreemfs-dir-replication-plugin.properties" ''
22 babudb.repl.backupDir = ${home}/server-repl-dir
23 plugin.jar = ${xtreemfs}/share/java/BabuDB_replication_plugin.jar
24 babudb.repl.dependency.0 = ${xtreemfs}/share/java/Flease.jar
25
26 ${cfg.dir.replication.extraConfig}
27 '';
28
29 dirConfig = pkgs.writeText "xtreemfs-dir-config.properties" ''
30 uuid = ${cfg.dir.uuid}
31 listen.port = ${toString cfg.dir.port}
32 ${optionalString (cfg.dir.address != "") "listen.address = ${cfg.dir.address}"}
33 http_port = ${toString cfg.dir.httpPort}
34 babudb.baseDir = ${home}/dir/database
35 babudb.logDir = ${home}/dir/db-log
36 babudb.sync = ${if cfg.dir.replication.enable then "FDATASYNC" else cfg.dir.syncMode}
37
38 ${optionalString cfg.dir.replication.enable "babudb.plugin.0 = ${dirReplicationConfig}"}
39
40 ${cfg.dir.extraConfig}
41 '';
42
43 mrcReplicationConfig = pkgs.writeText "xtreemfs-mrc-replication-plugin.properties" ''
44 babudb.repl.backupDir = ${home}/server-repl-mrc
45 plugin.jar = ${xtreemfs}/share/java/BabuDB_replication_plugin.jar
46 babudb.repl.dependency.0 = ${xtreemfs}/share/java/Flease.jar
47
48 ${cfg.mrc.replication.extraConfig}
49 '';
50
51 mrcConfig = pkgs.writeText "xtreemfs-mrc-config.properties" ''
52 uuid = ${cfg.mrc.uuid}
53 listen.port = ${toString cfg.mrc.port}
54 ${optionalString (cfg.mrc.address != "") "listen.address = ${cfg.mrc.address}"}
55 http_port = ${toString cfg.mrc.httpPort}
56 babudb.baseDir = ${home}/mrc/database
57 babudb.logDir = ${home}/mrc/db-log
58 babudb.sync = ${if cfg.mrc.replication.enable then "FDATASYNC" else cfg.mrc.syncMode}
59
60 ${optionalString cfg.mrc.replication.enable "babudb.plugin.0 = ${mrcReplicationConfig}"}
61
62 ${cfg.mrc.extraConfig}
63 '';
64
65 osdConfig = pkgs.writeText "xtreemfs-osd-config.properties" ''
66 uuid = ${cfg.osd.uuid}
67 listen.port = ${toString cfg.osd.port}
68 ${optionalString (cfg.osd.address != "") "listen.address = ${cfg.osd.address}"}
69 http_port = ${toString cfg.osd.httpPort}
70 object_dir = ${home}/osd/
71
72 ${cfg.osd.extraConfig}
73 '';
74
75 optionalDir = optionals cfg.dir.enable ["xtreemfs-dir.service"];
76
77 systemdOptionalDependencies = {
78 after = [ "network.target" ] ++ optionalDir;
79 wantedBy = [ "multi-user.target" ] ++ optionalDir;
80 };
81
82in
83
84{
85
86 ###### interface
87
88 options = {
89
90 services.xtreemfs = {
91
92 enable = mkEnableOption (lib.mdDoc "XtreemFS");
93
94 homeDir = mkOption {
95 type = types.path;
96 default = "/var/lib/xtreemfs";
97 description = lib.mdDoc ''
98 XtreemFS home dir for the xtreemfs user.
99 '';
100 };
101
102 dir = {
103 enable = mkOption {
104 type = types.bool;
105 default = true;
106 description = lib.mdDoc ''
107 Whether to enable XtreemFS DIR service.
108 '';
109 };
110
111 uuid = mkOption {
112 example = "eacb6bab-f444-4ebf-a06a-3f72d7465e40";
113 type = types.str;
114 description = lib.mdDoc ''
115 Must be set to a unique identifier, preferably a UUID according to
116 RFC 4122. UUIDs can be generated with `uuidgen` command, found in
117 the `util-linux` package.
118 '';
119 };
120 port = mkOption {
121 default = 32638;
122 type = types.port;
123 description = lib.mdDoc ''
124 The port to listen on for incoming connections (TCP).
125 '';
126 };
127 address = mkOption {
128 type = types.str;
129 example = "127.0.0.1";
130 default = "";
131 description = lib.mdDoc ''
132 If specified, it defines the interface to listen on. If not
133 specified, the service will listen on all interfaces (any).
134 '';
135 };
136 httpPort = mkOption {
137 default = 30638;
138 type = types.port;
139 description = lib.mdDoc ''
140 Specifies the listen port for the HTTP service that returns the
141 status page.
142 '';
143 };
144 syncMode = mkOption {
145 type = types.enum [ "ASYNC" "SYNC_WRITE_METADATA" "SYNC_WRITE" "FDATASYNC" "FSYNC" ];
146 default = "FSYNC";
147 example = "FDATASYNC";
148 description = lib.mdDoc ''
149 The sync mode influences how operations are committed to the disk
150 log before the operation is acknowledged to the caller.
151
152 -ASYNC mode the writes to the disk log are buffered in memory by the operating system. This is the fastest mode but will lead to data loss in case of a crash, kernel panic or power failure.
153 -SYNC_WRITE_METADATA opens the file with O_SYNC, the system will not buffer any writes. The operation will be acknowledged when data has been safely written to disk. This mode is slow but offers maximum data safety. However, BabuDB cannot influence the disk drive caches, this depends on the OS and hard disk model.
154 -SYNC_WRITE similar to SYNC_WRITE_METADATA but opens file with O_DSYNC which means that only the data is commit to disk. This can lead to some data loss depending on the implementation of the underlying file system. Linux does not implement this mode.
155 -FDATASYNC is similar to SYNC_WRITE but opens the file in asynchronous mode and calls fdatasync() after writing the data to disk.
156 -FSYNC is similar to SYNC_WRITE_METADATA but opens the file in asynchronous mode and calls fsync() after writing the data to disk.
157
158 For best throughput use ASYNC, for maximum data safety use FSYNC.
159
160 (If xtreemfs.dir.replication.enable is true then FDATASYNC is forced)
161 '';
162 };
163 extraConfig = mkOption {
164 type = types.lines;
165 default = "";
166 example = ''
167 # specify whether SSL is required
168 ssl.enabled = true
169 ssl.service_creds.pw = passphrase
170 ssl.service_creds.container = pkcs12
171 ssl.service_creds = /etc/xos/xtreemfs/truststore/certs/dir.p12
172 ssl.trusted_certs = /etc/xos/xtreemfs/truststore/certs/trusted.jks
173 ssl.trusted_certs.pw = jks_passphrase
174 ssl.trusted_certs.container = jks
175 '';
176 description = lib.mdDoc ''
177 Configuration of XtreemFS DIR service.
178 WARNING: configuration is saved as plaintext inside nix store.
179 For more options: http://www.xtreemfs.org/xtfs-guide-1.5.1/index.html
180 '';
181 };
182 replication = {
183 enable = mkEnableOption (lib.mdDoc "XtreemFS DIR replication plugin");
184 extraConfig = mkOption {
185 type = types.lines;
186 example = ''
187 # participants of the replication including this replica
188 babudb.repl.participant.0 = 192.168.0.10
189 babudb.repl.participant.0.port = 35676
190 babudb.repl.participant.1 = 192.168.0.11
191 babudb.repl.participant.1.port = 35676
192 babudb.repl.participant.2 = 192.168.0.12
193 babudb.repl.participant.2.port = 35676
194
195 # number of servers that at least have to be up to date
196 # To have a fault-tolerant system, this value has to be set to the
197 # majority of nodes i.e., if you have three replicas, set this to 2
198 # Please note that a setup with two nodes provides no fault-tolerance.
199 babudb.repl.sync.n = 2
200
201 # specify whether SSL is required
202 babudb.ssl.enabled = true
203
204 babudb.ssl.protocol = tlsv12
205
206 # server credentials for SSL handshakes
207 babudb.ssl.service_creds = /etc/xos/xtreemfs/truststore/certs/osd.p12
208 babudb.ssl.service_creds.pw = passphrase
209 babudb.ssl.service_creds.container = pkcs12
210
211 # trusted certificates for SSL handshakes
212 babudb.ssl.trusted_certs = /etc/xos/xtreemfs/truststore/certs/trusted.jks
213 babudb.ssl.trusted_certs.pw = jks_passphrase
214 babudb.ssl.trusted_certs.container = jks
215
216 babudb.ssl.authenticationWithoutEncryption = false
217 '';
218 description = lib.mdDoc ''
219 Configuration of XtreemFS DIR replication plugin.
220 WARNING: configuration is saved as plaintext inside nix store.
221 For more options: http://www.xtreemfs.org/xtfs-guide-1.5.1/index.html
222 '';
223 };
224 };
225 };
226
227 mrc = {
228 enable = mkOption {
229 type = types.bool;
230 default = true;
231 description = lib.mdDoc ''
232 Whether to enable XtreemFS MRC service.
233 '';
234 };
235
236 uuid = mkOption {
237 example = "eacb6bab-f444-4ebf-a06a-3f72d7465e41";
238 type = types.str;
239 description = lib.mdDoc ''
240 Must be set to a unique identifier, preferably a UUID according to
241 RFC 4122. UUIDs can be generated with `uuidgen` command, found in
242 the `util-linux` package.
243 '';
244 };
245 port = mkOption {
246 default = 32636;
247 type = types.port;
248 description = lib.mdDoc ''
249 The port to listen on for incoming connections (TCP).
250 '';
251 };
252 address = mkOption {
253 example = "127.0.0.1";
254 type = types.str;
255 default = "";
256 description = lib.mdDoc ''
257 If specified, it defines the interface to listen on. If not
258 specified, the service will listen on all interfaces (any).
259 '';
260 };
261 httpPort = mkOption {
262 default = 30636;
263 type = types.port;
264 description = lib.mdDoc ''
265 Specifies the listen port for the HTTP service that returns the
266 status page.
267 '';
268 };
269 syncMode = mkOption {
270 default = "FSYNC";
271 type = types.enum [ "ASYNC" "SYNC_WRITE_METADATA" "SYNC_WRITE" "FDATASYNC" "FSYNC" ];
272 example = "FDATASYNC";
273 description = lib.mdDoc ''
274 The sync mode influences how operations are committed to the disk
275 log before the operation is acknowledged to the caller.
276
277 -ASYNC mode the writes to the disk log are buffered in memory by the operating system. This is the fastest mode but will lead to data loss in case of a crash, kernel panic or power failure.
278 -SYNC_WRITE_METADATA opens the file with O_SYNC, the system will not buffer any writes. The operation will be acknowledged when data has been safely written to disk. This mode is slow but offers maximum data safety. However, BabuDB cannot influence the disk drive caches, this depends on the OS and hard disk model.
279 -SYNC_WRITE similar to SYNC_WRITE_METADATA but opens file with O_DSYNC which means that only the data is commit to disk. This can lead to some data loss depending on the implementation of the underlying file system. Linux does not implement this mode.
280 -FDATASYNC is similar to SYNC_WRITE but opens the file in asynchronous mode and calls fdatasync() after writing the data to disk.
281 -FSYNC is similar to SYNC_WRITE_METADATA but opens the file in asynchronous mode and calls fsync() after writing the data to disk.
282
283 For best throughput use ASYNC, for maximum data safety use FSYNC.
284
285 (If xtreemfs.mrc.replication.enable is true then FDATASYNC is forced)
286 '';
287 };
288 extraConfig = mkOption {
289 type = types.lines;
290 example = ''
291 osd_check_interval = 300
292 no_atime = true
293 local_clock_renewal = 0
294 remote_time_sync = 30000
295 authentication_provider = org.xtreemfs.common.auth.NullAuthProvider
296
297 # shared secret between the MRC and all OSDs
298 capability_secret = iNG8UuQJrJ6XVDTe
299
300 dir_service.host = 192.168.0.10
301 dir_service.port = 32638
302
303 # if replication is enabled
304 dir_service.1.host = 192.168.0.11
305 dir_service.1.port = 32638
306 dir_service.2.host = 192.168.0.12
307 dir_service.2.port = 32638
308
309 # specify whether SSL is required
310 ssl.enabled = true
311 ssl.protocol = tlsv12
312 ssl.service_creds.pw = passphrase
313 ssl.service_creds.container = pkcs12
314 ssl.service_creds = /etc/xos/xtreemfs/truststore/certs/mrc.p12
315 ssl.trusted_certs = /etc/xos/xtreemfs/truststore/certs/trusted.jks
316 ssl.trusted_certs.pw = jks_passphrase
317 ssl.trusted_certs.container = jks
318 '';
319 description = lib.mdDoc ''
320 Configuration of XtreemFS MRC service.
321 WARNING: configuration is saved as plaintext inside nix store.
322 For more options: http://www.xtreemfs.org/xtfs-guide-1.5.1/index.html
323 '';
324 };
325 replication = {
326 enable = mkEnableOption (lib.mdDoc "XtreemFS MRC replication plugin");
327 extraConfig = mkOption {
328 type = types.lines;
329 example = ''
330 # participants of the replication including this replica
331 babudb.repl.participant.0 = 192.168.0.10
332 babudb.repl.participant.0.port = 35678
333 babudb.repl.participant.1 = 192.168.0.11
334 babudb.repl.participant.1.port = 35678
335 babudb.repl.participant.2 = 192.168.0.12
336 babudb.repl.participant.2.port = 35678
337
338 # number of servers that at least have to be up to date
339 # To have a fault-tolerant system, this value has to be set to the
340 # majority of nodes i.e., if you have three replicas, set this to 2
341 # Please note that a setup with two nodes provides no fault-tolerance.
342 babudb.repl.sync.n = 2
343
344 # specify whether SSL is required
345 babudb.ssl.enabled = true
346
347 babudb.ssl.protocol = tlsv12
348
349 # server credentials for SSL handshakes
350 babudb.ssl.service_creds = /etc/xos/xtreemfs/truststore/certs/osd.p12
351 babudb.ssl.service_creds.pw = passphrase
352 babudb.ssl.service_creds.container = pkcs12
353
354 # trusted certificates for SSL handshakes
355 babudb.ssl.trusted_certs = /etc/xos/xtreemfs/truststore/certs/trusted.jks
356 babudb.ssl.trusted_certs.pw = jks_passphrase
357 babudb.ssl.trusted_certs.container = jks
358
359 babudb.ssl.authenticationWithoutEncryption = false
360 '';
361 description = lib.mdDoc ''
362 Configuration of XtreemFS MRC replication plugin.
363 WARNING: configuration is saved as plaintext inside nix store.
364 For more options: http://www.xtreemfs.org/xtfs-guide-1.5.1/index.html
365 '';
366 };
367 };
368 };
369
370 osd = {
371 enable = mkOption {
372 type = types.bool;
373 default = true;
374 description = lib.mdDoc ''
375 Whether to enable XtreemFS OSD service.
376 '';
377 };
378
379 uuid = mkOption {
380 example = "eacb6bab-f444-4ebf-a06a-3f72d7465e42";
381 type = types.str;
382 description = lib.mdDoc ''
383 Must be set to a unique identifier, preferably a UUID according to
384 RFC 4122. UUIDs can be generated with `uuidgen` command, found in
385 the `util-linux` package.
386 '';
387 };
388 port = mkOption {
389 default = 32640;
390 type = types.port;
391 description = lib.mdDoc ''
392 The port to listen on for incoming connections (TCP and UDP).
393 '';
394 };
395 address = mkOption {
396 example = "127.0.0.1";
397 type = types.str;
398 default = "";
399 description = lib.mdDoc ''
400 If specified, it defines the interface to listen on. If not
401 specified, the service will listen on all interfaces (any).
402 '';
403 };
404 httpPort = mkOption {
405 default = 30640;
406 type = types.port;
407 description = lib.mdDoc ''
408 Specifies the listen port for the HTTP service that returns the
409 status page.
410 '';
411 };
412 extraConfig = mkOption {
413 type = types.lines;
414 example = ''
415 local_clock_renewal = 0
416 remote_time_sync = 30000
417 report_free_space = true
418 capability_secret = iNG8UuQJrJ6XVDTe
419
420 dir_service.host = 192.168.0.10
421 dir_service.port = 32638
422
423 # if replication is used
424 dir_service.1.host = 192.168.0.11
425 dir_service.1.port = 32638
426 dir_service.2.host = 192.168.0.12
427 dir_service.2.port = 32638
428
429 # specify whether SSL is required
430 ssl.enabled = true
431 ssl.service_creds.pw = passphrase
432 ssl.service_creds.container = pkcs12
433 ssl.service_creds = /etc/xos/xtreemfs/truststore/certs/osd.p12
434 ssl.trusted_certs = /etc/xos/xtreemfs/truststore/certs/trusted.jks
435 ssl.trusted_certs.pw = jks_passphrase
436 ssl.trusted_certs.container = jks
437 '';
438 description = lib.mdDoc ''
439 Configuration of XtreemFS OSD service.
440 WARNING: configuration is saved as plaintext inside nix store.
441 For more options: http://www.xtreemfs.org/xtfs-guide-1.5.1/index.html
442 '';
443 };
444 };
445 };
446
447 };
448
449
450 ###### implementation
451
452 config = lib.mkIf cfg.enable {
453
454 environment.systemPackages = [ xtreemfs ];
455
456 users.users.xtreemfs =
457 { uid = config.ids.uids.xtreemfs;
458 description = "XtreemFS user";
459 createHome = true;
460 home = home;
461 };
462
463 users.groups.xtreemfs =
464 { gid = config.ids.gids.xtreemfs;
465 };
466
467 systemd.services.xtreemfs-dir = mkIf cfg.dir.enable {
468 description = "XtreemFS-DIR Server";
469 after = [ "network.target" ];
470 wantedBy = [ "multi-user.target" ];
471 serviceConfig = {
472 User = "xtreemfs";
473 ExecStart = "${startupScript "org.xtreemfs.dir.DIR" dirConfig}";
474 };
475 };
476
477 systemd.services.xtreemfs-mrc = mkIf cfg.mrc.enable ({
478 description = "XtreemFS-MRC Server";
479 serviceConfig = {
480 User = "xtreemfs";
481 ExecStart = "${startupScript "org.xtreemfs.mrc.MRC" mrcConfig}";
482 };
483 } // systemdOptionalDependencies);
484
485 systemd.services.xtreemfs-osd = mkIf cfg.osd.enable ({
486 description = "XtreemFS-OSD Server";
487 serviceConfig = {
488 User = "xtreemfs";
489 ExecStart = "${startupScript "org.xtreemfs.osd.OSD" osdConfig}";
490 };
491 } // systemdOptionalDependencies);
492
493 };
494
495}