pyrox.dev
1{ config, lib, pkgs, ... }:
2
3with lib;
4
5let
6 cfg = config.services.exhibitor;
7 exhibitor = cfg.package;
8 exhibitorConfig = ''
9 zookeeper-install-directory=${cfg.baseDir}/zookeeper
10 zookeeper-data-directory=${cfg.zkDataDir}
11 zookeeper-log-directory=${cfg.zkLogDir}
12 zoo-cfg-extra=${cfg.zkExtraCfg}
13 client-port=${toString cfg.zkClientPort}
14 connect-port=${toString cfg.zkConnectPort}
15 election-port=${toString cfg.zkElectionPort}
16 cleanup-period-ms=${toString cfg.zkCleanupPeriod}
17 servers-spec=${concatStringsSep "," cfg.zkServersSpec}
18 auto-manage-instances=${toString cfg.autoManageInstances}
19 ${cfg.extraConf}
20 '';
21 # NB: toString rather than lib.boolToString on cfg.autoManageInstances is intended.
22 # Exhibitor tests if it's an integer not equal to 0, so the empty string (toString false)
23 # will operate in the same fashion as a 0.
24 configDir = pkgs.writeTextDir "exhibitor.properties" exhibitorConfig;
25 cliOptionsCommon = {
26 configtype = cfg.configType;
27 defaultconfig = "${configDir}/exhibitor.properties";
28 port = toString cfg.port;
29 hostname = cfg.hostname;
30 headingtext = if (cfg.headingText != null) then (lib.escapeShellArg cfg.headingText) else null;
31 nodemodification = lib.boolToString cfg.nodeModification;
32 configcheckms = toString cfg.configCheckMs;
33 jquerystyle = cfg.jqueryStyle;
34 loglines = toString cfg.logLines;
35 servo = lib.boolToString cfg.servo;
36 timeout = toString cfg.timeout;
37 };
38 s3CommonOptions = { s3region = cfg.s3Region; s3credentials = cfg.s3Credentials; };
39 cliOptionsPerConfig = {
40 s3 = {
41 s3config = "${cfg.s3Config.bucketName}:${cfg.s3Config.objectKey}";
42 s3configprefix = cfg.s3Config.configPrefix;
43 };
44 zookeeper = {
45 zkconfigconnect = concatStringsSep "," cfg.zkConfigConnect;
46 zkconfigexhibitorpath = cfg.zkConfigExhibitorPath;
47 zkconfigpollms = toString cfg.zkConfigPollMs;
48 zkconfigretry = "${toString cfg.zkConfigRetry.sleepMs}:${toString cfg.zkConfigRetry.retryQuantity}";
49 zkconfigzpath = cfg.zkConfigZPath;
50 zkconfigexhibitorport = toString cfg.zkConfigExhibitorPort; # NB: This might be null
51 };
52 file = {
53 fsconfigdir = cfg.fsConfigDir;
54 fsconfiglockprefix = cfg.fsConfigLockPrefix;
55 fsConfigName = fsConfigName;
56 };
57 none = {
58 noneconfigdir = configDir;
59 };
60 };
61 cliOptions = concatStringsSep " " (mapAttrsToList (k: v: "--${k} ${v}") (filterAttrs (k: v: v != null && v != "") (cliOptionsCommon //
62 cliOptionsPerConfig."${cfg.configType}" //
63 s3CommonOptions //
64 optionalAttrs cfg.s3Backup { s3backup = "true"; } //
65 optionalAttrs cfg.fileSystemBackup { filesystembackup = "true"; }
66 )));
67in
68{
69 options = {
70 services.exhibitor = {
71 enable = mkOption {
72 type = types.bool;
73 default = false;
74 description = "
75 Whether to enable the exhibitor server.
76 ";
77 };
78 # See https://github.com/soabase/exhibitor/wiki/Running-Exhibitor for what these mean
79 # General options for any type of config
80 port = mkOption {
81 type = types.int;
82 default = 8080;
83 description = ''
84 The port for exhibitor to listen on and communicate with other exhibitors.
85 '';
86 };
87 baseDir = mkOption {
88 type = types.str;
89 default = "/var/exhibitor";
90 description = ''
91 Baseline directory for exhibitor runtime config.
92 '';
93 };
94 configType = mkOption {
95 type = types.enum [ "file" "s3" "zookeeper" "none" ];
96 description = ''
97 Which configuration type you want to use. Additional config will be
98 required depending on which type you are using.
99 '';
100 };
101 hostname = mkOption {
102 type = types.nullOr types.str;
103 description = ''
104 Hostname to use and advertise
105 '';
106 default = null;
107 };
108 nodeModification = mkOption {
109 type = types.bool;
110 description = ''
111 Whether the Explorer UI will allow nodes to be modified (use with caution).
112 '';
113 default = true;
114 };
115 configCheckMs = mkOption {
116 type = types.int;
117 description = ''
118 Period (ms) to check for shared config updates.
119 '';
120 default = 30000;
121 };
122 headingText = mkOption {
123 type = types.nullOr types.str;
124 description = ''
125 Extra text to display in UI header
126 '';
127 default = null;
128 };
129 jqueryStyle = mkOption {
130 type = types.enum [ "red" "black" "custom" ];
131 description = ''
132 Styling used for the JQuery-based UI.
133 '';
134 default = "red";
135 };
136 logLines = mkOption {
137 type = types.int;
138 description = ''
139 Max lines of logging to keep in memory for display.
140 '';
141 default = 1000;
142 };
143 servo = mkOption {
144 type = types.bool;
145 description = ''
146 ZooKeeper will be queried once a minute for its state via the 'mntr' four
147 letter word (this requires ZooKeeper 3.4.x+). Servo will be used to publish
148 this data via JMX.
149 '';
150 default = false;
151 };
152 timeout = mkOption {
153 type = types.int;
154 description = ''
155 Connection timeout (ms) for ZK connections.
156 '';
157 default = 30000;
158 };
159 autoManageInstances = mkOption {
160 type = types.bool;
161 description = ''
162 Automatically manage ZooKeeper instances in the ensemble
163 '';
164 default = false;
165 };
166 zkDataDir = mkOption {
167 type = types.str;
168 default = "${cfg.baseDir}/zkData";
169 description = ''
170 The Zookeeper data directory
171 '';
172 };
173 zkLogDir = mkOption {
174 type = types.path;
175 default = "${cfg.baseDir}/zkLogs";
176 description = ''
177 The Zookeeper logs directory
178 '';
179 };
180 extraConf = mkOption {
181 type = types.str;
182 default = "";
183 description = ''
184 Extra Exhibitor configuration to put in the ZooKeeper config file.
185 '';
186 };
187 zkExtraCfg = mkOption {
188 type = types.str;
189 default = ''initLimit=5&syncLimit=2&tickTime=2000'';
190 description = ''
191 Extra options to pass into Zookeeper
192 '';
193 };
194 zkClientPort = mkOption {
195 type = types.int;
196 default = 2181;
197 description = ''
198 Zookeeper client port
199 '';
200 };
201 zkConnectPort = mkOption {
202 type = types.int;
203 default = 2888;
204 description = ''
205 The port to use for followers to talk to each other.
206 '';
207 };
208 zkElectionPort = mkOption {
209 type = types.int;
210 default = 3888;
211 description = ''
212 The port for Zookeepers to use for leader election.
213 '';
214 };
215 zkCleanupPeriod = mkOption {
216 type = types.int;
217 default = 0;
218 description = ''
219 How often (in milliseconds) to run the Zookeeper log cleanup task.
220 '';
221 };
222 zkServersSpec = mkOption {
223 type = types.listOf types.str;
224 default = [];
225 description = ''
226 Zookeeper server spec for all servers in the ensemble.
227 '';
228 example = [ "S:1:zk1.example.com" "S:2:zk2.example.com" "S:3:zk3.example.com" "O:4:zk-observer.example.com" ];
229 };
230
231 # Backup options
232 s3Backup = mkOption {
233 type = types.bool;
234 default = false;
235 description = ''
236 Whether to enable backups to S3
237 '';
238 };
239 fileSystemBackup = mkOption {
240 type = types.bool;
241 default = false;
242 description = ''
243 Enables file system backup of ZooKeeper log files
244 '';
245 };
246
247 # Options for using zookeeper configType
248 zkConfigConnect = mkOption {
249 type = types.listOf types.str;
250 description = ''
251 The initial connection string for ZooKeeper shared config storage
252 '';
253 example = ["host1:2181" "host2:2181"];
254 };
255 zkConfigExhibitorPath = mkOption {
256 type = types.string;
257 description = ''
258 If the ZooKeeper shared config is also running Exhibitor, the URI path for the REST call
259 '';
260 default = "/";
261 };
262 zkConfigExhibitorPort = mkOption {
263 type = types.nullOr types.int;
264 description = ''
265 If the ZooKeeper shared config is also running Exhibitor, the port that
266 Exhibitor is listening on. IMPORTANT: if this value is not set it implies
267 that Exhibitor is not being used on the ZooKeeper shared config.
268 '';
269 };
270 zkConfigPollMs = mkOption {
271 type = types.int;
272 description = ''
273 The period in ms to check for changes in the config ensemble
274 '';
275 default = 10000;
276 };
277 zkConfigRetry = {
278 sleepMs = mkOption {
279 type = types.int;
280 default = 1000;
281 description = ''
282 Retry sleep time connecting to the ZooKeeper config
283 '';
284 };
285 retryQuantity = mkOption {
286 type = types.int;
287 default = 3;
288 description = ''
289 Retries connecting to the ZooKeeper config
290 '';
291 };
292 };
293 zkConfigZPath = mkOption {
294 type = types.str;
295 description = ''
296 The base ZPath that Exhibitor should use
297 '';
298 example = "/exhibitor/config";
299 };
300
301 # Config options for s3 configType
302 s3Config = {
303 bucketName = mkOption {
304 type = types.str;
305 description = ''
306 Bucket name to store config
307 '';
308 };
309 objectKey = mkOption {
310 type = types.str;
311 description = ''
312 S3 key name to store the config
313 '';
314 };
315 configPrefix = mkOption {
316 type = types.str;
317 description = ''
318 When using AWS S3 shared config files, the prefix to use for values such as locks
319 '';
320 default = "exhibitor-";
321 };
322 };
323
324 # The next two are used for either s3backup or s3 configType
325 s3Credentials = mkOption {
326 type = types.nullOr types.path;
327 description = ''
328 Optional credentials to use for s3backup or s3config. Argument is the path
329 to an AWS credential properties file with two properties:
330 com.netflix.exhibitor.s3.access-key-id and com.netflix.exhibitor.s3.access-secret-key
331 '';
332 default = null;
333 };
334 s3Region = mkOption {
335 type = types.nullOr types.str;
336 description = ''
337 Optional region for S3 calls
338 '';
339 default = null;
340 };
341
342 # Config options for file config type
343 fsConfigDir = mkOption {
344 type = types.path;
345 description = ''
346 Directory to store Exhibitor properties (cannot be used with s3config).
347 Exhibitor uses file system locks so you can specify a shared location
348 so as to enable complete ensemble management.
349 '';
350 };
351 fsConfigLockPrefix = mkOption {
352 type = types.str;
353 description = ''
354 A prefix for a locking mechanism used in conjunction with fsconfigdir
355 '';
356 default = "exhibitor-lock-";
357 };
358 fsConfigName = mkOption {
359 type = types.str;
360 description = ''
361 The name of the file to store config in
362 '';
363 default = "exhibitor.properties";
364 };
365 };
366 };
367
368 config = mkIf cfg.enable {
369 systemd.services.exhibitor = {
370 description = "Exhibitor Daemon";
371 wantedBy = [ "multi-user.target" ];
372 after = [ "network.target" ];
373 environment = {
374 ZOO_LOG_DIR = cfg.baseDir;
375 };
376 serviceConfig = {
377 /***
378 Exhibitor is a bit un-nixy. It wants to present to you a user interface in order to
379 mutate the configuration of both itself and ZooKeeper, and to coordinate changes
380 among the members of the Zookeeper ensemble. I'm going for a different approach here,
381 which is to manage all the configuration via nix and have it write out the configuration
382 files that exhibitor will use, and to reduce the amount of inter-exhibitor orchestration.
383 ***/
384 ExecStart = ''
385 ${pkgs.exhibitor}/bin/startExhibitor.sh ${cliOptions}
386 '';
387 User = "zookeeper";
388 PermissionsStartOnly = true;
389 };
390 # This is a bit wonky, but the reason for this is that Exhibitor tries to write to
391 # ${cfg.baseDir}/zookeeper/bin/../conf/zoo.cfg
392 # I want everything but the conf directory to be in the immutable nix store, and I want defaults
393 # from the nix store
394 # If I symlink the bin directory in, then bin/../ will resolve to the parent of the symlink in the
395 # immutable nix store. Bind mounting a writable conf over the existing conf might work, but it gets very
396 # messy with trying to copy the existing out into a mutable store.
397 # Another option is to try to patch upstream exhibitor, but the current package just pulls down the
398 # prebuild JARs off of Maven, rather than building them ourselves, as Maven support in Nix isn't
399 # very mature. So, it seems like a reasonable compromise is to just copy out of the immutable store
400 # just before starting the service, so we're running binaries from the immutable store, but we work around
401 # Exhibitor's desire to mutate its current installation.
402 preStart = ''
403 mkdir -m 0700 -p ${cfg.baseDir}/zookeeper
404 # Not doing a chown -R to keep the base ZK files owned by root
405 chown zookeeper ${cfg.baseDir} ${cfg.baseDir}/zookeeper
406 cp -Rf ${pkgs.zookeeper}/* ${cfg.baseDir}/zookeeper
407 chown -R zookeeper ${cfg.baseDir}/zookeeper/conf
408 chmod -R u+w ${cfg.baseDir}/zookeeper/conf
409 '';
410 };
411 users.extraUsers = singleton {
412 name = "zookeeper";
413 uid = config.ids.uids.zookeeper;
414 description = "Zookeeper daemon user";
415 home = cfg.baseDir;
416 };
417 };
418}