pyrox.dev
1{ config, lib, pkgs, ... }:
2
3with lib;
4
5let
6 cfg = config.services.exhibitor;
7 exhibitorConfig = ''
8 zookeeper-install-directory=${cfg.baseDir}/zookeeper
9 zookeeper-data-directory=${cfg.zkDataDir}
10 zookeeper-log-directory=${cfg.zkLogDir}
11 zoo-cfg-extra=${cfg.zkExtraCfg}
12 client-port=${toString cfg.zkClientPort}
13 connect-port=${toString cfg.zkConnectPort}
14 election-port=${toString cfg.zkElectionPort}
15 cleanup-period-ms=${toString cfg.zkCleanupPeriod}
16 servers-spec=${concatStringsSep "," cfg.zkServersSpec}
17 auto-manage-instances=${toString cfg.autoManageInstances}
18 ${cfg.extraConf}
19 '';
20 # NB: toString rather than lib.boolToString on cfg.autoManageInstances is intended.
21 # Exhibitor tests if it's an integer not equal to 0, so the empty string (toString false)
22 # will operate in the same fashion as a 0.
23 configDir = pkgs.writeTextDir "exhibitor.properties" exhibitorConfig;
24 cliOptionsCommon = {
25 configtype = cfg.configType;
26 defaultconfig = "${configDir}/exhibitor.properties";
27 port = toString cfg.port;
28 hostname = cfg.hostname;
29 headingtext = if (cfg.headingText != null) then (lib.escapeShellArg cfg.headingText) else null;
30 nodemodification = lib.boolToString cfg.nodeModification;
31 configcheckms = toString cfg.configCheckMs;
32 jquerystyle = cfg.jqueryStyle;
33 loglines = toString cfg.logLines;
34 servo = lib.boolToString cfg.servo;
35 timeout = toString cfg.timeout;
36 };
37 s3CommonOptions = { s3region = cfg.s3Region; s3credentials = cfg.s3Credentials; };
38 cliOptionsPerConfig = {
39 s3 = {
40 s3config = "${cfg.s3Config.bucketName}:${cfg.s3Config.objectKey}";
41 s3configprefix = cfg.s3Config.configPrefix;
42 };
43 zookeeper = {
44 zkconfigconnect = concatStringsSep "," cfg.zkConfigConnect;
45 zkconfigexhibitorpath = cfg.zkConfigExhibitorPath;
46 zkconfigpollms = toString cfg.zkConfigPollMs;
47 zkconfigretry = "${toString cfg.zkConfigRetry.sleepMs}:${toString cfg.zkConfigRetry.retryQuantity}";
48 zkconfigzpath = cfg.zkConfigZPath;
49 zkconfigexhibitorport = toString cfg.zkConfigExhibitorPort; # NB: This might be null
50 };
51 file = {
52 fsconfigdir = cfg.fsConfigDir;
53 fsconfiglockprefix = cfg.fsConfigLockPrefix;
54 fsConfigName = fsConfigName;
55 };
56 none = {
57 noneconfigdir = configDir;
58 };
59 };
60 cliOptions = concatStringsSep " " (mapAttrsToList (k: v: "--${k} ${v}") (filterAttrs (k: v: v != null && v != "") (cliOptionsCommon //
61 cliOptionsPerConfig.${cfg.configType} //
62 s3CommonOptions //
63 optionalAttrs cfg.s3Backup { s3backup = "true"; } //
64 optionalAttrs cfg.fileSystemBackup { filesystembackup = "true"; }
65 )));
66in
67{
68 options = {
69 services.exhibitor = {
70 enable = mkOption {
71 type = types.bool;
72 default = false;
73 description = "
74 Whether to enable the exhibitor server.
75 ";
76 };
77 # See https://github.com/soabase/exhibitor/wiki/Running-Exhibitor for what these mean
78 # General options for any type of config
79 port = mkOption {
80 type = types.int;
81 default = 8080;
82 description = ''
83 The port for exhibitor to listen on and communicate with other exhibitors.
84 '';
85 };
86 baseDir = mkOption {
87 type = types.str;
88 default = "/var/exhibitor";
89 description = ''
90 Baseline directory for exhibitor runtime config.
91 '';
92 };
93 configType = mkOption {
94 type = types.enum [ "file" "s3" "zookeeper" "none" ];
95 description = ''
96 Which configuration type you want to use. Additional config will be
97 required depending on which type you are using.
98 '';
99 };
100 hostname = mkOption {
101 type = types.nullOr types.str;
102 description = ''
103 Hostname to use and advertise
104 '';
105 default = null;
106 };
107 nodeModification = mkOption {
108 type = types.bool;
109 description = ''
110 Whether the Explorer UI will allow nodes to be modified (use with caution).
111 '';
112 default = true;
113 };
114 configCheckMs = mkOption {
115 type = types.int;
116 description = ''
117 Period (ms) to check for shared config updates.
118 '';
119 default = 30000;
120 };
121 headingText = mkOption {
122 type = types.nullOr types.str;
123 description = ''
124 Extra text to display in UI header
125 '';
126 default = null;
127 };
128 jqueryStyle = mkOption {
129 type = types.enum [ "red" "black" "custom" ];
130 description = ''
131 Styling used for the JQuery-based UI.
132 '';
133 default = "red";
134 };
135 logLines = mkOption {
136 type = types.int;
137 description = ''
138 Max lines of logging to keep in memory for display.
139 '';
140 default = 1000;
141 };
142 servo = mkOption {
143 type = types.bool;
144 description = ''
145 ZooKeeper will be queried once a minute for its state via the 'mntr' four
146 letter word (this requires ZooKeeper 3.4.x+). Servo will be used to publish
147 this data via JMX.
148 '';
149 default = false;
150 };
151 timeout = mkOption {
152 type = types.int;
153 description = ''
154 Connection timeout (ms) for ZK connections.
155 '';
156 default = 30000;
157 };
158 autoManageInstances = mkOption {
159 type = types.bool;
160 description = ''
161 Automatically manage ZooKeeper instances in the ensemble
162 '';
163 default = false;
164 };
165 zkDataDir = mkOption {
166 type = types.str;
167 default = "${cfg.baseDir}/zkData";
168 description = ''
169 The Zookeeper data directory
170 '';
171 };
172 zkLogDir = mkOption {
173 type = types.path;
174 default = "${cfg.baseDir}/zkLogs";
175 description = ''
176 The Zookeeper logs directory
177 '';
178 };
179 extraConf = mkOption {
180 type = types.str;
181 default = "";
182 description = ''
183 Extra Exhibitor configuration to put in the ZooKeeper config file.
184 '';
185 };
186 zkExtraCfg = mkOption {
187 type = types.str;
188 default = "initLimit=5&syncLimit=2&tickTime=2000";
189 description = ''
190 Extra options to pass into Zookeeper
191 '';
192 };
193 zkClientPort = mkOption {
194 type = types.int;
195 default = 2181;
196 description = ''
197 Zookeeper client port
198 '';
199 };
200 zkConnectPort = mkOption {
201 type = types.int;
202 default = 2888;
203 description = ''
204 The port to use for followers to talk to each other.
205 '';
206 };
207 zkElectionPort = mkOption {
208 type = types.int;
209 default = 3888;
210 description = ''
211 The port for Zookeepers to use for leader election.
212 '';
213 };
214 zkCleanupPeriod = mkOption {
215 type = types.int;
216 default = 0;
217 description = ''
218 How often (in milliseconds) to run the Zookeeper log cleanup task.
219 '';
220 };
221 zkServersSpec = mkOption {
222 type = types.listOf types.str;
223 default = [];
224 description = ''
225 Zookeeper server spec for all servers in the ensemble.
226 '';
227 example = [ "S:1:zk1.example.com" "S:2:zk2.example.com" "S:3:zk3.example.com" "O:4:zk-observer.example.com" ];
228 };
229
230 # Backup options
231 s3Backup = mkOption {
232 type = types.bool;
233 default = false;
234 description = ''
235 Whether to enable backups to S3
236 '';
237 };
238 fileSystemBackup = mkOption {
239 type = types.bool;
240 default = false;
241 description = ''
242 Enables file system backup of ZooKeeper log files
243 '';
244 };
245
246 # Options for using zookeeper configType
247 zkConfigConnect = mkOption {
248 type = types.listOf types.str;
249 description = ''
250 The initial connection string for ZooKeeper shared config storage
251 '';
252 example = ["host1:2181" "host2:2181"];
253 };
254 zkConfigExhibitorPath = mkOption {
255 type = types.str;
256 description = ''
257 If the ZooKeeper shared config is also running Exhibitor, the URI path for the REST call
258 '';
259 default = "/";
260 };
261 zkConfigExhibitorPort = mkOption {
262 type = types.nullOr types.int;
263 description = ''
264 If the ZooKeeper shared config is also running Exhibitor, the port that
265 Exhibitor is listening on. IMPORTANT: if this value is not set it implies
266 that Exhibitor is not being used on the ZooKeeper shared config.
267 '';
268 };
269 zkConfigPollMs = mkOption {
270 type = types.int;
271 description = ''
272 The period in ms to check for changes in the config ensemble
273 '';
274 default = 10000;
275 };
276 zkConfigRetry = {
277 sleepMs = mkOption {
278 type = types.int;
279 default = 1000;
280 description = ''
281 Retry sleep time connecting to the ZooKeeper config
282 '';
283 };
284 retryQuantity = mkOption {
285 type = types.int;
286 default = 3;
287 description = ''
288 Retries connecting to the ZooKeeper config
289 '';
290 };
291 };
292 zkConfigZPath = mkOption {
293 type = types.str;
294 description = ''
295 The base ZPath that Exhibitor should use
296 '';
297 example = "/exhibitor/config";
298 };
299
300 # Config options for s3 configType
301 s3Config = {
302 bucketName = mkOption {
303 type = types.str;
304 description = ''
305 Bucket name to store config
306 '';
307 };
308 objectKey = mkOption {
309 type = types.str;
310 description = ''
311 S3 key name to store the config
312 '';
313 };
314 configPrefix = mkOption {
315 type = types.str;
316 description = ''
317 When using AWS S3 shared config files, the prefix to use for values such as locks
318 '';
319 default = "exhibitor-";
320 };
321 };
322
323 # The next two are used for either s3backup or s3 configType
324 s3Credentials = mkOption {
325 type = types.nullOr types.path;
326 description = ''
327 Optional credentials to use for s3backup or s3config. Argument is the path
328 to an AWS credential properties file with two properties:
329 com.netflix.exhibitor.s3.access-key-id and com.netflix.exhibitor.s3.access-secret-key
330 '';
331 default = null;
332 };
333 s3Region = mkOption {
334 type = types.nullOr types.str;
335 description = ''
336 Optional region for S3 calls
337 '';
338 default = null;
339 };
340
341 # Config options for file config type
342 fsConfigDir = mkOption {
343 type = types.path;
344 description = ''
345 Directory to store Exhibitor properties (cannot be used with s3config).
346 Exhibitor uses file system locks so you can specify a shared location
347 so as to enable complete ensemble management.
348 '';
349 };
350 fsConfigLockPrefix = mkOption {
351 type = types.str;
352 description = ''
353 A prefix for a locking mechanism used in conjunction with fsconfigdir
354 '';
355 default = "exhibitor-lock-";
356 };
357 fsConfigName = mkOption {
358 type = types.str;
359 description = ''
360 The name of the file to store config in
361 '';
362 default = "exhibitor.properties";
363 };
364 };
365 };
366
367 config = mkIf cfg.enable {
368 systemd.services.exhibitor = {
369 description = "Exhibitor Daemon";
370 wantedBy = [ "multi-user.target" ];
371 after = [ "network.target" ];
372 environment = {
373 ZOO_LOG_DIR = cfg.baseDir;
374 };
375 serviceConfig = {
376 /***
377 Exhibitor is a bit un-nixy. It wants to present to you a user interface in order to
378 mutate the configuration of both itself and ZooKeeper, and to coordinate changes
379 among the members of the Zookeeper ensemble. I'm going for a different approach here,
380 which is to manage all the configuration via nix and have it write out the configuration
381 files that exhibitor will use, and to reduce the amount of inter-exhibitor orchestration.
382 ***/
383 ExecStart = ''
384 ${pkgs.exhibitor}/bin/startExhibitor.sh ${cliOptions}
385 '';
386 User = "zookeeper";
387 PermissionsStartOnly = true;
388 };
389 # This is a bit wonky, but the reason for this is that Exhibitor tries to write to
390 # ${cfg.baseDir}/zookeeper/bin/../conf/zoo.cfg
391 # I want everything but the conf directory to be in the immutable nix store, and I want defaults
392 # from the nix store
393 # If I symlink the bin directory in, then bin/../ will resolve to the parent of the symlink in the
394 # immutable nix store. Bind mounting a writable conf over the existing conf might work, but it gets very
395 # messy with trying to copy the existing out into a mutable store.
396 # Another option is to try to patch upstream exhibitor, but the current package just pulls down the
397 # prebuild JARs off of Maven, rather than building them ourselves, as Maven support in Nix isn't
398 # very mature. So, it seems like a reasonable compromise is to just copy out of the immutable store
399 # just before starting the service, so we're running binaries from the immutable store, but we work around
400 # Exhibitor's desire to mutate its current installation.
401 preStart = ''
402 mkdir -m 0700 -p ${cfg.baseDir}/zookeeper
403 # Not doing a chown -R to keep the base ZK files owned by root
404 chown zookeeper ${cfg.baseDir} ${cfg.baseDir}/zookeeper
405 cp -Rf ${pkgs.zookeeper}/* ${cfg.baseDir}/zookeeper
406 chown -R zookeeper ${cfg.baseDir}/zookeeper/conf
407 chmod -R u+w ${cfg.baseDir}/zookeeper/conf
408 replace_what=$(echo ${pkgs.zookeeper} | sed 's/[\/&]/\\&/g')
409 replace_with=$(echo ${cfg.baseDir}/zookeeper | sed 's/[\/&]/\\&/g')
410 sed -i 's/'"$replace_what"'/'"$replace_with"'/g' ${cfg.baseDir}/zookeeper/bin/zk*.sh
411 '';
412 };
413 users.users.zookeeper = {
414 uid = config.ids.uids.zookeeper;
415 description = "Zookeeper daemon user";
416 home = cfg.baseDir;
417 };
418 };
419}