pyrox.dev
1{ config, pkgs, lib, ... }:
2
3with lib;
4
5let
6
7 cfg = config.services.hydra;
8
9 baseDir = "/var/lib/hydra";
10
11 hydraConf = pkgs.writeScript "hydra.conf" cfg.extraConfig;
12
13 hydraEnv =
14 { HYDRA_DBI = cfg.dbi;
15 HYDRA_CONFIG = "${baseDir}/hydra.conf";
16 HYDRA_DATA = "${baseDir}";
17 };
18
19 env =
20 { NIX_REMOTE = "daemon";
21 SSL_CERT_FILE = "/etc/ssl/certs/ca-certificates.crt"; # Remove in 16.03
22 PGPASSFILE = "${baseDir}/pgpass";
23 NIX_REMOTE_SYSTEMS = concatStringsSep ":" cfg.buildMachinesFiles;
24 } // optionalAttrs (cfg.smtpHost != null) {
25 EMAIL_SENDER_TRANSPORT = "SMTP";
26 EMAIL_SENDER_TRANSPORT_host = cfg.smtpHost;
27 } // hydraEnv // cfg.extraEnv;
28
29 serverEnv = env //
30 { HYDRA_TRACKER = cfg.tracker;
31 XDG_CACHE_HOME = "${baseDir}/www/.cache";
32 COLUMNS = "80";
33 PGPASSFILE = "${baseDir}/pgpass-www"; # grrr
34 } // (optionalAttrs cfg.debugServer { DBIC_TRACE = "1"; });
35
36 localDB = "dbi:Pg:dbname=hydra;user=hydra;";
37
38 haveLocalDB = cfg.dbi == localDB;
39
40 hydra-package =
41 let
42 makeWrapperArgs = concatStringsSep " " (mapAttrsToList (key: value: "--set \"${key}\" \"${value}\"") hydraEnv);
43 in pkgs.buildEnv rec {
44 name = "hydra-env";
45 buildInputs = [ pkgs.makeWrapper ];
46 paths = [ cfg.package ];
47
48 postBuild = ''
49 if [ -L "$out/bin" ]; then
50 unlink "$out/bin"
51 fi
52 mkdir -p "$out/bin"
53
54 for path in ${concatStringsSep " " paths}; do
55 if [ -d "$path/bin" ]; then
56 cd "$path/bin"
57 for prg in *; do
58 if [ -f "$prg" ]; then
59 rm -f "$out/bin/$prg"
60 if [ -x "$prg" ]; then
61 makeWrapper "$path/bin/$prg" "$out/bin/$prg" ${makeWrapperArgs}
62 fi
63 fi
64 done
65 fi
66 done
67 '';
68 };
69
70in
71
72{
73 ###### interface
74 options = {
75
76 services.hydra = {
77
78 enable = mkOption {
79 type = types.bool;
80 default = false;
81 description = ''
82 Whether to run Hydra services.
83 '';
84 };
85
86 dbi = mkOption {
87 type = types.str;
88 default = localDB;
89 example = "dbi:Pg:dbname=hydra;host=postgres.example.org;user=foo;";
90 description = ''
91 The DBI string for Hydra database connection.
92
93 NOTE: Attempts to set `application_name` will be overridden by
94 `hydra-TYPE` (where TYPE is e.g. `evaluator`, `queue-runner`,
95 etc.) in all hydra services to more easily distinguish where
96 queries are coming from.
97 '';
98 };
99
100 package = mkOption {
101 type = types.package;
102 default = pkgs.hydra-unstable;
103 defaultText = "pkgs.hydra-unstable";
104 description = "The Hydra package.";
105 };
106
107 hydraURL = mkOption {
108 type = types.str;
109 description = ''
110 The base URL for the Hydra webserver instance. Used for links in emails.
111 '';
112 };
113
114 listenHost = mkOption {
115 type = types.str;
116 default = "*";
117 example = "localhost";
118 description = ''
119 The hostname or address to listen on or <literal>*</literal> to listen
120 on all interfaces.
121 '';
122 };
123
124 port = mkOption {
125 type = types.int;
126 default = 3000;
127 description = ''
128 TCP port the web server should listen to.
129 '';
130 };
131
132 minimumDiskFree = mkOption {
133 type = types.int;
134 default = 0;
135 description = ''
136 Threshold of minimum disk space (GiB) to determine if the queue runner should run or not.
137 '';
138 };
139
140 minimumDiskFreeEvaluator = mkOption {
141 type = types.int;
142 default = 0;
143 description = ''
144 Threshold of minimum disk space (GiB) to determine if the evaluator should run or not.
145 '';
146 };
147
148 notificationSender = mkOption {
149 type = types.str;
150 description = ''
151 Sender email address used for email notifications.
152 '';
153 };
154
155 smtpHost = mkOption {
156 type = types.nullOr types.str;
157 default = null;
158 example = ["localhost"];
159 description = ''
160 Hostname of the SMTP server to use to send email.
161 '';
162 };
163
164 tracker = mkOption {
165 type = types.str;
166 default = "";
167 description = ''
168 Piece of HTML that is included on all pages.
169 '';
170 };
171
172 logo = mkOption {
173 type = types.nullOr types.path;
174 default = null;
175 description = ''
176 Path to a file containing the logo of your Hydra instance.
177 '';
178 };
179
180 debugServer = mkOption {
181 type = types.bool;
182 default = false;
183 description = "Whether to run the server in debug mode.";
184 };
185
186 extraConfig = mkOption {
187 type = types.lines;
188 description = "Extra lines for the Hydra configuration.";
189 };
190
191 extraEnv = mkOption {
192 type = types.attrsOf types.str;
193 default = {};
194 description = "Extra environment variables for Hydra.";
195 };
196
197 gcRootsDir = mkOption {
198 type = types.path;
199 default = "/nix/var/nix/gcroots/hydra";
200 description = "Directory that holds Hydra garbage collector roots.";
201 };
202
203 buildMachinesFiles = mkOption {
204 type = types.listOf types.path;
205 default = optional (config.nix.buildMachines != []) "/etc/nix/machines";
206 example = [ "/etc/nix/machines" "/var/lib/hydra/provisioner/machines" ];
207 description = "List of files containing build machines.";
208 };
209
210 useSubstitutes = mkOption {
211 type = types.bool;
212 default = false;
213 description = ''
214 Whether to use binary caches for downloading store paths. Note that
215 binary substitutions trigger (a potentially large number of) additional
216 HTTP requests that slow down the queue monitor thread significantly.
217 Also, this Hydra instance will serve those downloaded store paths to
218 its users with its own signature attached as if it had built them
219 itself, so don't enable this feature unless your active binary caches
220 are absolute trustworthy.
221 '';
222 };
223 };
224
225 };
226
227
228 ###### implementation
229
230 config = mkIf cfg.enable {
231
232 users.groups.hydra = {
233 gid = config.ids.gids.hydra;
234 };
235
236 users.users.hydra =
237 { description = "Hydra";
238 group = "hydra";
239 # We don't enable `createHome` here because the creation of the home directory is handled by the hydra-init service below.
240 home = baseDir;
241 useDefaultShell = true;
242 uid = config.ids.uids.hydra;
243 };
244
245 users.users.hydra-queue-runner =
246 { description = "Hydra queue runner";
247 group = "hydra";
248 useDefaultShell = true;
249 home = "${baseDir}/queue-runner"; # really only to keep SSH happy
250 uid = config.ids.uids.hydra-queue-runner;
251 };
252
253 users.users.hydra-www =
254 { description = "Hydra web server";
255 group = "hydra";
256 useDefaultShell = true;
257 uid = config.ids.uids.hydra-www;
258 };
259
260 nix.trustedUsers = [ "hydra-queue-runner" ];
261
262 services.hydra.extraConfig =
263 ''
264 using_frontend_proxy = 1
265 base_uri = ${cfg.hydraURL}
266 notification_sender = ${cfg.notificationSender}
267 max_servers = 25
268 ${optionalString (cfg.logo != null) ''
269 hydra_logo = ${cfg.logo}
270 ''}
271 gc_roots_dir = ${cfg.gcRootsDir}
272 use-substitutes = ${if cfg.useSubstitutes then "1" else "0"}
273 '';
274
275 environment.systemPackages = [ hydra-package ];
276
277 environment.variables = hydraEnv;
278
279 nix.extraOptions = ''
280 keep-outputs = true
281 keep-derivations = true
282
283
284 '' + optionalString (versionOlder (getVersion config.nix.package.out) "2.4pre") ''
285 # The default (`true') slows Nix down a lot since the build farm
286 # has so many GC roots.
287 gc-check-reachability = false
288 '';
289
290 systemd.services.hydra-init =
291 { wantedBy = [ "multi-user.target" ];
292 requires = optional haveLocalDB "postgresql.service";
293 after = optional haveLocalDB "postgresql.service";
294 environment = env // {
295 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-init";
296 };
297 preStart = ''
298 mkdir -p ${baseDir}
299 chown hydra.hydra ${baseDir}
300 chmod 0750 ${baseDir}
301
302 ln -sf ${hydraConf} ${baseDir}/hydra.conf
303
304 mkdir -m 0700 -p ${baseDir}/www
305 chown hydra-www.hydra ${baseDir}/www
306
307 mkdir -m 0700 -p ${baseDir}/queue-runner
308 mkdir -m 0750 -p ${baseDir}/build-logs
309 chown hydra-queue-runner.hydra ${baseDir}/queue-runner ${baseDir}/build-logs
310
311 ${optionalString haveLocalDB ''
312 if ! [ -e ${baseDir}/.db-created ]; then
313 ${pkgs.sudo}/bin/sudo -u ${config.services.postgresql.superUser} ${config.services.postgresql.package}/bin/createuser hydra
314 ${pkgs.sudo}/bin/sudo -u ${config.services.postgresql.superUser} ${config.services.postgresql.package}/bin/createdb -O hydra hydra
315 touch ${baseDir}/.db-created
316 fi
317 echo "create extension if not exists pg_trgm" | ${pkgs.sudo}/bin/sudo -u ${config.services.postgresql.superUser} -- ${config.services.postgresql.package}/bin/psql hydra
318 ''}
319
320 if [ ! -e ${cfg.gcRootsDir} ]; then
321
322 # Move legacy roots directory.
323 if [ -e /nix/var/nix/gcroots/per-user/hydra/hydra-roots ]; then
324 mv /nix/var/nix/gcroots/per-user/hydra/hydra-roots ${cfg.gcRootsDir}
325 fi
326
327 mkdir -p ${cfg.gcRootsDir}
328 fi
329
330 # Move legacy hydra-www roots.
331 if [ -e /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots ]; then
332 find /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots/ -type f \
333 | xargs -r mv -f -t ${cfg.gcRootsDir}/
334 rmdir /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots
335 fi
336
337 chown hydra.hydra ${cfg.gcRootsDir}
338 chmod 2775 ${cfg.gcRootsDir}
339 '';
340 serviceConfig.ExecStart = "${hydra-package}/bin/hydra-init";
341 serviceConfig.PermissionsStartOnly = true;
342 serviceConfig.User = "hydra";
343 serviceConfig.Type = "oneshot";
344 serviceConfig.RemainAfterExit = true;
345 };
346
347 systemd.services.hydra-server =
348 { wantedBy = [ "multi-user.target" ];
349 requires = [ "hydra-init.service" ];
350 after = [ "hydra-init.service" ];
351 environment = serverEnv // {
352 HYDRA_DBI = "${serverEnv.HYDRA_DBI};application_name=hydra-server";
353 };
354 restartTriggers = [ hydraConf ];
355 serviceConfig =
356 { ExecStart =
357 "@${hydra-package}/bin/hydra-server hydra-server -f -h '${cfg.listenHost}' "
358 + "-p ${toString cfg.port} --max_spare_servers 5 --max_servers 25 "
359 + "--max_requests 100 ${optionalString cfg.debugServer "-d"}";
360 User = "hydra-www";
361 PermissionsStartOnly = true;
362 Restart = "always";
363 };
364 };
365
366 systemd.services.hydra-queue-runner =
367 { wantedBy = [ "multi-user.target" ];
368 requires = [ "hydra-init.service" ];
369 after = [ "hydra-init.service" "network.target" ];
370 path = [ hydra-package pkgs.nettools pkgs.openssh pkgs.bzip2 config.nix.package ];
371 restartTriggers = [ hydraConf ];
372 environment = env // {
373 PGPASSFILE = "${baseDir}/pgpass-queue-runner"; # grrr
374 IN_SYSTEMD = "1"; # to get log severity levels
375 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-queue-runner";
376 };
377 serviceConfig =
378 { ExecStart = "@${hydra-package}/bin/hydra-queue-runner hydra-queue-runner -v";
379 ExecStopPost = "${hydra-package}/bin/hydra-queue-runner --unlock";
380 User = "hydra-queue-runner";
381 Restart = "always";
382
383 # Ensure we can get core dumps.
384 LimitCORE = "infinity";
385 WorkingDirectory = "${baseDir}/queue-runner";
386 };
387 };
388
389 systemd.services.hydra-evaluator =
390 { wantedBy = [ "multi-user.target" ];
391 requires = [ "hydra-init.service" ];
392 after = [ "hydra-init.service" "network.target" ];
393 path = with pkgs; [ hydra-package nettools jq ];
394 restartTriggers = [ hydraConf ];
395 environment = env // {
396 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-evaluator";
397 };
398 serviceConfig =
399 { ExecStart = "@${hydra-package}/bin/hydra-evaluator hydra-evaluator";
400 User = "hydra";
401 Restart = "always";
402 WorkingDirectory = baseDir;
403 };
404 };
405
406 systemd.services.hydra-update-gc-roots =
407 { requires = [ "hydra-init.service" ];
408 after = [ "hydra-init.service" ];
409 environment = env // {
410 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-update-gc-roots";
411 };
412 serviceConfig =
413 { ExecStart = "@${hydra-package}/bin/hydra-update-gc-roots hydra-update-gc-roots";
414 User = "hydra";
415 };
416 startAt = "2,14:15";
417 };
418
419 systemd.services.hydra-send-stats =
420 { wantedBy = [ "multi-user.target" ];
421 after = [ "hydra-init.service" ];
422 environment = env // {
423 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-send-stats";
424 };
425 serviceConfig =
426 { ExecStart = "@${hydra-package}/bin/hydra-send-stats hydra-send-stats";
427 User = "hydra";
428 };
429 };
430
431 systemd.services.hydra-notify =
432 { wantedBy = [ "multi-user.target" ];
433 requires = [ "hydra-init.service" ];
434 after = [ "hydra-init.service" ];
435 restartTriggers = [ hydraConf ];
436 environment = env // {
437 PGPASSFILE = "${baseDir}/pgpass-queue-runner";
438 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-notify";
439 };
440 serviceConfig =
441 { ExecStart = "@${hydra-package}/bin/hydra-notify hydra-notify";
442 # FIXME: run this under a less privileged user?
443 User = "hydra-queue-runner";
444 Restart = "always";
445 RestartSec = 5;
446 };
447 };
448
449 # If there is less than a certain amount of free disk space, stop
450 # the queue/evaluator to prevent builds from failing or aborting.
451 systemd.services.hydra-check-space =
452 { script =
453 ''
454 if [ $(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store))) -lt $((${toString cfg.minimumDiskFree} * 1024**3)) ]; then
455 echo "stopping Hydra queue runner due to lack of free space..."
456 systemctl stop hydra-queue-runner
457 fi
458 if [ $(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store))) -lt $((${toString cfg.minimumDiskFreeEvaluator} * 1024**3)) ]; then
459 echo "stopping Hydra evaluator due to lack of free space..."
460 systemctl stop hydra-evaluator
461 fi
462 '';
463 startAt = "*:0/5";
464 };
465
466 # Periodically compress build logs. The queue runner compresses
467 # logs automatically after a step finishes, but this doesn't work
468 # if the queue runner is stopped prematurely.
469 systemd.services.hydra-compress-logs =
470 { path = [ pkgs.bzip2 ];
471 script =
472 ''
473 find /var/lib/hydra/build-logs -type f -name "*.drv" -mtime +3 -size +0c | xargs -r bzip2 -v -f
474 '';
475 startAt = "Sun 01:45";
476 };
477
478 services.postgresql.enable = mkIf haveLocalDB true;
479
480 services.postgresql.identMap = optionalString haveLocalDB
481 ''
482 hydra-users hydra hydra
483 hydra-users hydra-queue-runner hydra
484 hydra-users hydra-www hydra
485 hydra-users root hydra
486 # The postgres user is used to create the pg_trgm extension for the hydra database
487 hydra-users postgres postgres
488 '';
489
490 services.postgresql.authentication = optionalString haveLocalDB
491 ''
492 local hydra all ident map=hydra-users
493 '';
494
495 };
496
497}