pyrox.dev
1{ config, lib, options, pkgs, ... }:
2
3with lib;
4
5let
6 cfg = config.services.exhibitor;
7 opt = options.services.exhibitor;
8 exhibitorConfig = ''
9 zookeeper-install-directory=${cfg.baseDir}/zookeeper
10 zookeeper-data-directory=${cfg.zkDataDir}
11 zookeeper-log-directory=${cfg.zkLogDir}
12 zoo-cfg-extra=${cfg.zkExtraCfg}
13 client-port=${toString cfg.zkClientPort}
14 connect-port=${toString cfg.zkConnectPort}
15 election-port=${toString cfg.zkElectionPort}
16 cleanup-period-ms=${toString cfg.zkCleanupPeriod}
17 servers-spec=${concatStringsSep "," cfg.zkServersSpec}
18 auto-manage-instances=${toString cfg.autoManageInstances}
19 ${cfg.extraConf}
20 '';
21 # NB: toString rather than lib.boolToString on cfg.autoManageInstances is intended.
22 # Exhibitor tests if it's an integer not equal to 0, so the empty string (toString false)
23 # will operate in the same fashion as a 0.
24 configDir = pkgs.writeTextDir "exhibitor.properties" exhibitorConfig;
25 cliOptionsCommon = {
26 configtype = cfg.configType;
27 defaultconfig = "${configDir}/exhibitor.properties";
28 port = toString cfg.port;
29 hostname = cfg.hostname;
30 headingtext = if (cfg.headingText != null) then (lib.escapeShellArg cfg.headingText) else null;
31 nodemodification = lib.boolToString cfg.nodeModification;
32 configcheckms = toString cfg.configCheckMs;
33 jquerystyle = cfg.jqueryStyle;
34 loglines = toString cfg.logLines;
35 servo = lib.boolToString cfg.servo;
36 timeout = toString cfg.timeout;
37 };
38 s3CommonOptions = { s3region = cfg.s3Region; s3credentials = cfg.s3Credentials; };
39 cliOptionsPerConfig = {
40 s3 = {
41 s3config = "${cfg.s3Config.bucketName}:${cfg.s3Config.objectKey}";
42 s3configprefix = cfg.s3Config.configPrefix;
43 };
44 zookeeper = {
45 zkconfigconnect = concatStringsSep "," cfg.zkConfigConnect;
46 zkconfigexhibitorpath = cfg.zkConfigExhibitorPath;
47 zkconfigpollms = toString cfg.zkConfigPollMs;
48 zkconfigretry = "${toString cfg.zkConfigRetry.sleepMs}:${toString cfg.zkConfigRetry.retryQuantity}";
49 zkconfigzpath = cfg.zkConfigZPath;
50 zkconfigexhibitorport = toString cfg.zkConfigExhibitorPort; # NB: This might be null
51 };
52 file = {
53 fsconfigdir = cfg.fsConfigDir;
54 fsconfiglockprefix = cfg.fsConfigLockPrefix;
55 fsConfigName = fsConfigName;
56 };
57 none = {
58 noneconfigdir = configDir;
59 };
60 };
61 cliOptions = concatStringsSep " " (mapAttrsToList (k: v: "--${k} ${v}") (filterAttrs (k: v: v != null && v != "") (cliOptionsCommon //
62 cliOptionsPerConfig.${cfg.configType} //
63 s3CommonOptions //
64 optionalAttrs cfg.s3Backup { s3backup = "true"; } //
65 optionalAttrs cfg.fileSystemBackup { filesystembackup = "true"; }
66 )));
67in
68{
69 options = {
70 services.exhibitor = {
71 enable = mkEnableOption (lib.mdDoc "exhibitor server");
72
73 # See https://github.com/soabase/exhibitor/wiki/Running-Exhibitor for what these mean
74 # General options for any type of config
75 port = mkOption {
76 type = types.port;
77 default = 8080;
78 description = lib.mdDoc ''
79 The port for exhibitor to listen on and communicate with other exhibitors.
80 '';
81 };
82 baseDir = mkOption {
83 type = types.str;
84 default = "/var/exhibitor";
85 description = lib.mdDoc ''
86 Baseline directory for exhibitor runtime config.
87 '';
88 };
89 configType = mkOption {
90 type = types.enum [ "file" "s3" "zookeeper" "none" ];
91 description = lib.mdDoc ''
92 Which configuration type you want to use. Additional config will be
93 required depending on which type you are using.
94 '';
95 };
96 hostname = mkOption {
97 type = types.nullOr types.str;
98 description = lib.mdDoc ''
99 Hostname to use and advertise
100 '';
101 default = null;
102 };
103 nodeModification = mkOption {
104 type = types.bool;
105 description = lib.mdDoc ''
106 Whether the Explorer UI will allow nodes to be modified (use with caution).
107 '';
108 default = true;
109 };
110 configCheckMs = mkOption {
111 type = types.int;
112 description = lib.mdDoc ''
113 Period (ms) to check for shared config updates.
114 '';
115 default = 30000;
116 };
117 headingText = mkOption {
118 type = types.nullOr types.str;
119 description = lib.mdDoc ''
120 Extra text to display in UI header
121 '';
122 default = null;
123 };
124 jqueryStyle = mkOption {
125 type = types.enum [ "red" "black" "custom" ];
126 description = lib.mdDoc ''
127 Styling used for the JQuery-based UI.
128 '';
129 default = "red";
130 };
131 logLines = mkOption {
132 type = types.int;
133 description = lib.mdDoc ''
134 Max lines of logging to keep in memory for display.
135 '';
136 default = 1000;
137 };
138 servo = mkOption {
139 type = types.bool;
140 description = lib.mdDoc ''
141 ZooKeeper will be queried once a minute for its state via the 'mntr' four
142 letter word (this requires ZooKeeper 3.4.x+). Servo will be used to publish
143 this data via JMX.
144 '';
145 default = false;
146 };
147 timeout = mkOption {
148 type = types.int;
149 description = lib.mdDoc ''
150 Connection timeout (ms) for ZK connections.
151 '';
152 default = 30000;
153 };
154 autoManageInstances = mkOption {
155 type = types.bool;
156 description = lib.mdDoc ''
157 Automatically manage ZooKeeper instances in the ensemble
158 '';
159 default = false;
160 };
161 zkDataDir = mkOption {
162 type = types.str;
163 default = "${cfg.baseDir}/zkData";
164 defaultText = literalExpression ''"''${config.${opt.baseDir}}/zkData"'';
165 description = lib.mdDoc ''
166 The Zookeeper data directory
167 '';
168 };
169 zkLogDir = mkOption {
170 type = types.path;
171 default = "${cfg.baseDir}/zkLogs";
172 defaultText = literalExpression ''"''${config.${opt.baseDir}}/zkLogs"'';
173 description = lib.mdDoc ''
174 The Zookeeper logs directory
175 '';
176 };
177 extraConf = mkOption {
178 type = types.str;
179 default = "";
180 description = lib.mdDoc ''
181 Extra Exhibitor configuration to put in the ZooKeeper config file.
182 '';
183 };
184 zkExtraCfg = mkOption {
185 type = types.str;
186 default = "initLimit=5&syncLimit=2&tickTime=2000";
187 description = lib.mdDoc ''
188 Extra options to pass into Zookeeper
189 '';
190 };
191 zkClientPort = mkOption {
192 type = types.int;
193 default = 2181;
194 description = lib.mdDoc ''
195 Zookeeper client port
196 '';
197 };
198 zkConnectPort = mkOption {
199 type = types.int;
200 default = 2888;
201 description = lib.mdDoc ''
202 The port to use for followers to talk to each other.
203 '';
204 };
205 zkElectionPort = mkOption {
206 type = types.int;
207 default = 3888;
208 description = lib.mdDoc ''
209 The port for Zookeepers to use for leader election.
210 '';
211 };
212 zkCleanupPeriod = mkOption {
213 type = types.int;
214 default = 0;
215 description = lib.mdDoc ''
216 How often (in milliseconds) to run the Zookeeper log cleanup task.
217 '';
218 };
219 zkServersSpec = mkOption {
220 type = types.listOf types.str;
221 default = [];
222 description = lib.mdDoc ''
223 Zookeeper server spec for all servers in the ensemble.
224 '';
225 example = [ "S:1:zk1.example.com" "S:2:zk2.example.com" "S:3:zk3.example.com" "O:4:zk-observer.example.com" ];
226 };
227
228 # Backup options
229 s3Backup = mkOption {
230 type = types.bool;
231 default = false;
232 description = lib.mdDoc ''
233 Whether to enable backups to S3
234 '';
235 };
236 fileSystemBackup = mkOption {
237 type = types.bool;
238 default = false;
239 description = lib.mdDoc ''
240 Enables file system backup of ZooKeeper log files
241 '';
242 };
243
244 # Options for using zookeeper configType
245 zkConfigConnect = mkOption {
246 type = types.listOf types.str;
247 description = lib.mdDoc ''
248 The initial connection string for ZooKeeper shared config storage
249 '';
250 example = ["host1:2181" "host2:2181"];
251 };
252 zkConfigExhibitorPath = mkOption {
253 type = types.str;
254 description = lib.mdDoc ''
255 If the ZooKeeper shared config is also running Exhibitor, the URI path for the REST call
256 '';
257 default = "/";
258 };
259 zkConfigExhibitorPort = mkOption {
260 type = types.nullOr types.int;
261 description = lib.mdDoc ''
262 If the ZooKeeper shared config is also running Exhibitor, the port that
263 Exhibitor is listening on. IMPORTANT: if this value is not set it implies
264 that Exhibitor is not being used on the ZooKeeper shared config.
265 '';
266 };
267 zkConfigPollMs = mkOption {
268 type = types.int;
269 description = lib.mdDoc ''
270 The period in ms to check for changes in the config ensemble
271 '';
272 default = 10000;
273 };
274 zkConfigRetry = {
275 sleepMs = mkOption {
276 type = types.int;
277 default = 1000;
278 description = lib.mdDoc ''
279 Retry sleep time connecting to the ZooKeeper config
280 '';
281 };
282 retryQuantity = mkOption {
283 type = types.int;
284 default = 3;
285 description = lib.mdDoc ''
286 Retries connecting to the ZooKeeper config
287 '';
288 };
289 };
290 zkConfigZPath = mkOption {
291 type = types.str;
292 description = lib.mdDoc ''
293 The base ZPath that Exhibitor should use
294 '';
295 example = "/exhibitor/config";
296 };
297
298 # Config options for s3 configType
299 s3Config = {
300 bucketName = mkOption {
301 type = types.str;
302 description = lib.mdDoc ''
303 Bucket name to store config
304 '';
305 };
306 objectKey = mkOption {
307 type = types.str;
308 description = lib.mdDoc ''
309 S3 key name to store the config
310 '';
311 };
312 configPrefix = mkOption {
313 type = types.str;
314 description = lib.mdDoc ''
315 When using AWS S3 shared config files, the prefix to use for values such as locks
316 '';
317 default = "exhibitor-";
318 };
319 };
320
321 # The next two are used for either s3backup or s3 configType
322 s3Credentials = mkOption {
323 type = types.nullOr types.path;
324 description = lib.mdDoc ''
325 Optional credentials to use for s3backup or s3config. Argument is the path
326 to an AWS credential properties file with two properties:
327 com.netflix.exhibitor.s3.access-key-id and com.netflix.exhibitor.s3.access-secret-key
328 '';
329 default = null;
330 };
331 s3Region = mkOption {
332 type = types.nullOr types.str;
333 description = lib.mdDoc ''
334 Optional region for S3 calls
335 '';
336 default = null;
337 };
338
339 # Config options for file config type
340 fsConfigDir = mkOption {
341 type = types.path;
342 description = lib.mdDoc ''
343 Directory to store Exhibitor properties (cannot be used with s3config).
344 Exhibitor uses file system locks so you can specify a shared location
345 so as to enable complete ensemble management.
346 '';
347 };
348 fsConfigLockPrefix = mkOption {
349 type = types.str;
350 description = lib.mdDoc ''
351 A prefix for a locking mechanism used in conjunction with fsconfigdir
352 '';
353 default = "exhibitor-lock-";
354 };
355 fsConfigName = mkOption {
356 type = types.str;
357 description = lib.mdDoc ''
358 The name of the file to store config in
359 '';
360 default = "exhibitor.properties";
361 };
362 };
363 };
364
365 config = mkIf cfg.enable {
366 systemd.services.exhibitor = {
367 description = "Exhibitor Daemon";
368 wantedBy = [ "multi-user.target" ];
369 after = [ "network.target" ];
370 environment = {
371 ZOO_LOG_DIR = cfg.baseDir;
372 };
373 serviceConfig = {
374 /***
375 Exhibitor is a bit un-nixy. It wants to present to you a user interface in order to
376 mutate the configuration of both itself and ZooKeeper, and to coordinate changes
377 among the members of the Zookeeper ensemble. I'm going for a different approach here,
378 which is to manage all the configuration via nix and have it write out the configuration
379 files that exhibitor will use, and to reduce the amount of inter-exhibitor orchestration.
380 ***/
381 ExecStart = ''
382 ${pkgs.exhibitor}/bin/startExhibitor.sh ${cliOptions}
383 '';
384 User = "zookeeper";
385 PermissionsStartOnly = true;
386 };
387 # This is a bit wonky, but the reason for this is that Exhibitor tries to write to
388 # ${cfg.baseDir}/zookeeper/bin/../conf/zoo.cfg
389 # I want everything but the conf directory to be in the immutable nix store, and I want defaults
390 # from the nix store
391 # If I symlink the bin directory in, then bin/../ will resolve to the parent of the symlink in the
392 # immutable nix store. Bind mounting a writable conf over the existing conf might work, but it gets very
393 # messy with trying to copy the existing out into a mutable store.
394 # Another option is to try to patch upstream exhibitor, but the current package just pulls down the
395 # prebuild JARs off of Maven, rather than building them ourselves, as Maven support in Nix isn't
396 # very mature. So, it seems like a reasonable compromise is to just copy out of the immutable store
397 # just before starting the service, so we're running binaries from the immutable store, but we work around
398 # Exhibitor's desire to mutate its current installation.
399 preStart = ''
400 mkdir -m 0700 -p ${cfg.baseDir}/zookeeper
401 # Not doing a chown -R to keep the base ZK files owned by root
402 chown zookeeper ${cfg.baseDir} ${cfg.baseDir}/zookeeper
403 cp -Rf ${pkgs.zookeeper}/* ${cfg.baseDir}/zookeeper
404 chown -R zookeeper ${cfg.baseDir}/zookeeper/conf
405 chmod -R u+w ${cfg.baseDir}/zookeeper/conf
406 replace_what=$(echo ${pkgs.zookeeper} | sed 's/[\/&]/\\&/g')
407 replace_with=$(echo ${cfg.baseDir}/zookeeper | sed 's/[\/&]/\\&/g')
408 sed -i 's/'"$replace_what"'/'"$replace_with"'/g' ${cfg.baseDir}/zookeeper/bin/zk*.sh
409 '';
410 };
411 users.users.zookeeper = {
412 uid = config.ids.uids.zookeeper;
413 description = "Zookeeper daemon user";
414 home = cfg.baseDir;
415 };
416 };
417}