pyrox.dev
1{
2 config,
3 pkgs,
4 lib,
5 ...
6}:
7let
8
9 cfg = config.services.hydra;
10
11 baseDir = "/var/lib/hydra";
12
13 hydraConf = pkgs.writeScript "hydra.conf" cfg.extraConfig;
14
15 hydraEnv = {
16 HYDRA_DBI = cfg.dbi;
17 HYDRA_CONFIG = "${baseDir}/hydra.conf";
18 HYDRA_DATA = "${baseDir}";
19 };
20
21 env = {
22 NIX_REMOTE = "daemon";
23 PGPASSFILE = "${baseDir}/pgpass";
24 NIX_REMOTE_SYSTEMS = lib.concatStringsSep ":" cfg.buildMachinesFiles;
25 }
26 // lib.optionalAttrs (cfg.smtpHost != null) {
27 EMAIL_SENDER_TRANSPORT = "SMTP";
28 EMAIL_SENDER_TRANSPORT_host = cfg.smtpHost;
29 }
30 // hydraEnv
31 // cfg.extraEnv;
32
33 serverEnv =
34 env
35 // {
36 HYDRA_TRACKER = cfg.tracker;
37 XDG_CACHE_HOME = "${baseDir}/www/.cache";
38 COLUMNS = "80";
39 PGPASSFILE = "${baseDir}/pgpass-www"; # grrr
40 }
41 // (lib.optionalAttrs cfg.debugServer { DBIC_TRACE = "1"; });
42
43 localDB = "dbi:Pg:dbname=hydra;user=hydra;";
44
45 haveLocalDB = cfg.dbi == localDB;
46
47 hydra-package =
48 let
49 makeWrapperArgs = lib.concatStringsSep " " (
50 lib.mapAttrsToList (key: value: "--set-default \"${key}\" \"${value}\"") hydraEnv
51 );
52 in
53 pkgs.buildEnv rec {
54 name = "hydra-env";
55 nativeBuildInputs = [ pkgs.makeWrapper ];
56 paths = [ cfg.package ];
57
58 postBuild = ''
59 if [ -L "$out/bin" ]; then
60 unlink "$out/bin"
61 fi
62 mkdir -p "$out/bin"
63
64 for path in ${lib.concatStringsSep " " paths}; do
65 if [ -d "$path/bin" ]; then
66 cd "$path/bin"
67 for prg in *; do
68 if [ -f "$prg" ]; then
69 rm -f "$out/bin/$prg"
70 if [ -x "$prg" ]; then
71 makeWrapper "$path/bin/$prg" "$out/bin/$prg" ${makeWrapperArgs}
72 fi
73 fi
74 done
75 fi
76 done
77 '';
78 };
79
80in
81
82{
83 ###### interface
84 options = {
85
86 services.hydra = {
87
88 enable = lib.mkOption {
89 type = lib.types.bool;
90 default = false;
91 description = ''
92 Whether to run Hydra services.
93 '';
94 };
95
96 dbi = lib.mkOption {
97 type = lib.types.str;
98 default = localDB;
99 example = "dbi:Pg:dbname=hydra;host=postgres.example.org;user=foo;";
100 description = ''
101 The DBI string for Hydra database connection.
102
103 NOTE: Attempts to set `application_name` will be overridden by
104 `hydra-TYPE` (where TYPE is e.g. `evaluator`, `queue-runner`,
105 etc.) in all hydra services to more easily distinguish where
106 queries are coming from.
107 '';
108 };
109
110 package = lib.mkPackageOption pkgs "hydra" { };
111
112 hydraURL = lib.mkOption {
113 type = lib.types.str;
114 description = ''
115 The base URL for the Hydra webserver instance. Used for links in emails.
116 '';
117 };
118
119 listenHost = lib.mkOption {
120 type = lib.types.str;
121 default = "*";
122 example = "localhost";
123 description = ''
124 The hostname or address to listen on or `*` to listen
125 on all interfaces.
126 '';
127 };
128
129 port = lib.mkOption {
130 type = lib.types.port;
131 default = 3000;
132 description = ''
133 TCP port the web server should listen to.
134 '';
135 };
136
137 minimumDiskFree = lib.mkOption {
138 type = lib.types.int;
139 default = 0;
140 description = ''
141 Threshold of minimum disk space (GiB) to determine if the queue runner should run or not.
142 '';
143 };
144
145 minimumDiskFreeEvaluator = lib.mkOption {
146 type = lib.types.int;
147 default = 0;
148 description = ''
149 Threshold of minimum disk space (GiB) to determine if the evaluator should run or not.
150 '';
151 };
152
153 notificationSender = lib.mkOption {
154 type = lib.types.str;
155 description = ''
156 Sender email address used for email notifications.
157 '';
158 };
159
160 smtpHost = lib.mkOption {
161 type = lib.types.nullOr lib.types.str;
162 default = null;
163 example = "localhost";
164 description = ''
165 Hostname of the SMTP server to use to send email.
166 '';
167 };
168
169 tracker = lib.mkOption {
170 type = lib.types.str;
171 default = "";
172 description = ''
173 Piece of HTML that is included on all pages.
174 '';
175 };
176
177 logo = lib.mkOption {
178 type = lib.types.nullOr lib.types.path;
179 default = null;
180 description = ''
181 Path to a file containing the logo of your Hydra instance.
182 '';
183 };
184
185 debugServer = lib.mkOption {
186 type = lib.types.bool;
187 default = false;
188 description = "Whether to run the server in debug mode.";
189 };
190
191 maxServers = lib.mkOption {
192 type = lib.types.int;
193 default = 25;
194 description = "Maximum number of starman workers to spawn.";
195 };
196
197 minSpareServers = lib.mkOption {
198 type = lib.types.int;
199 default = 4;
200 description = "Minimum number of spare starman workers to keep.";
201 };
202
203 maxSpareServers = lib.mkOption {
204 type = lib.types.int;
205 default = 5;
206 description = "Maximum number of spare starman workers to keep.";
207 };
208
209 extraConfig = lib.mkOption {
210 type = lib.types.lines;
211 description = "Extra lines for the Hydra configuration.";
212 };
213
214 extraEnv = lib.mkOption {
215 type = lib.types.attrsOf lib.types.str;
216 default = { };
217 description = "Extra environment variables for Hydra.";
218 };
219
220 gcRootsDir = lib.mkOption {
221 type = lib.types.path;
222 default = "/nix/var/nix/gcroots/hydra";
223 description = "Directory that holds Hydra garbage collector roots.";
224 };
225
226 buildMachinesFiles = lib.mkOption {
227 type = lib.types.listOf lib.types.path;
228 default = lib.optional (config.nix.buildMachines != [ ]) "/etc/nix/machines";
229 defaultText = lib.literalExpression ''lib.optional (config.nix.buildMachines != []) "/etc/nix/machines"'';
230 example = [
231 "/etc/nix/machines"
232 "/var/lib/hydra/provisioner/machines"
233 ];
234 description = "List of files containing build machines.";
235 };
236
237 useSubstitutes = lib.mkOption {
238 type = lib.types.bool;
239 default = false;
240 description = ''
241 Whether to use binary caches for downloading store paths. Note that
242 binary substitutions trigger (a potentially large number of) additional
243 HTTP requests that slow down the queue monitor thread significantly.
244 Also, this Hydra instance will serve those downloaded store paths to
245 its users with its own signature attached as if it had built them
246 itself, so don't enable this feature unless your active binary caches
247 are absolute trustworthy.
248 '';
249 };
250 };
251
252 };
253
254 ###### implementation
255
256 config = lib.mkIf cfg.enable {
257 assertions = [
258 {
259 assertion = cfg.maxServers != 0 && cfg.maxSpareServers != 0 && cfg.minSpareServers != 0;
260 message = "services.hydra.{minSpareServers,maxSpareServers,minSpareServers} cannot be 0";
261 }
262 {
263 assertion = cfg.minSpareServers < cfg.maxSpareServers;
264 message = "services.hydra.minSpareServers cannot be bigger than services.hydra.maxSpareServers";
265 }
266 ];
267
268 users.groups.hydra = {
269 gid = config.ids.gids.hydra;
270 };
271
272 users.users.hydra = {
273 description = "Hydra";
274 group = "hydra";
275 # We don't enable `createHome` here because the creation of the home directory is handled by the hydra-init service below.
276 home = baseDir;
277 useDefaultShell = true;
278 uid = config.ids.uids.hydra;
279 };
280
281 users.users.hydra-queue-runner = {
282 description = "Hydra queue runner";
283 group = "hydra";
284 useDefaultShell = true;
285 home = "${baseDir}/queue-runner"; # really only to keep SSH happy
286 uid = config.ids.uids.hydra-queue-runner;
287 };
288
289 users.users.hydra-www = {
290 description = "Hydra web server";
291 group = "hydra";
292 useDefaultShell = true;
293 uid = config.ids.uids.hydra-www;
294 };
295
296 services.hydra.extraConfig = ''
297 using_frontend_proxy = 1
298 base_uri = ${cfg.hydraURL}
299 notification_sender = ${cfg.notificationSender}
300 max_servers = ${toString cfg.maxServers}
301 ${lib.optionalString (cfg.logo != null) ''
302 hydra_logo = ${cfg.logo}
303 ''}
304 gc_roots_dir = ${cfg.gcRootsDir}
305 use-substitutes = ${if cfg.useSubstitutes then "1" else "0"}
306 '';
307
308 environment.systemPackages = [ hydra-package ];
309
310 environment.variables = hydraEnv;
311
312 nix.settings = lib.mkMerge [
313 {
314 keep-outputs = true;
315 keep-derivations = true;
316 trusted-users = [ "hydra-queue-runner" ];
317 }
318
319 (lib.mkIf (lib.versionOlder (lib.getVersion config.nix.package.out) "2.4pre") {
320 # The default (`true') slows Nix down a lot since the build farm
321 # has so many GC roots.
322 gc-check-reachability = false;
323 })
324 ];
325
326 systemd.slices.system-hydra = {
327 description = "Hydra CI Server Slice";
328 documentation = [
329 "file://${cfg.package}/share/doc/hydra/index.html"
330 "https://nixos.org/hydra/manual/"
331 ];
332 };
333
334 systemd.services.hydra-init = {
335 wantedBy = [ "multi-user.target" ];
336 requires = lib.optional haveLocalDB "postgresql.target";
337 after = lib.optional haveLocalDB "postgresql.target";
338 environment = env // {
339 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-init";
340 };
341 path = [ pkgs.util-linux ];
342 preStart = ''
343 mkdir -p ${baseDir}
344 chown hydra:hydra ${baseDir}
345 chmod 0750 ${baseDir}
346
347 ln -sf ${hydraConf} ${baseDir}/hydra.conf
348
349 mkdir -m 0700 ${baseDir}/www || true
350 chown hydra-www:hydra ${baseDir}/www
351
352 mkdir -m 0700 ${baseDir}/queue-runner || true
353 mkdir -m 0750 ${baseDir}/build-logs || true
354 mkdir -m 0750 ${baseDir}/runcommand-logs || true
355 chown hydra-queue-runner:hydra \
356 ${baseDir}/queue-runner \
357 ${baseDir}/build-logs \
358 ${baseDir}/runcommand-logs
359
360 ${lib.optionalString haveLocalDB ''
361 if ! [ -e ${baseDir}/.db-created ]; then
362 runuser -u ${config.services.postgresql.superUser} ${config.services.postgresql.package}/bin/createuser hydra
363 runuser -u ${config.services.postgresql.superUser} ${config.services.postgresql.package}/bin/createdb -- -O hydra hydra
364 touch ${baseDir}/.db-created
365 fi
366 echo "create extension if not exists pg_trgm" | runuser -u ${config.services.postgresql.superUser} -- ${config.services.postgresql.package}/bin/psql hydra
367 ''}
368
369 if [ ! -e ${cfg.gcRootsDir} ]; then
370
371 # Move legacy roots directory.
372 if [ -e /nix/var/nix/gcroots/per-user/hydra/hydra-roots ]; then
373 mv /nix/var/nix/gcroots/per-user/hydra/hydra-roots ${cfg.gcRootsDir}
374 fi
375
376 mkdir -p ${cfg.gcRootsDir}
377 fi
378
379 # Move legacy hydra-www roots.
380 if [ -e /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots ]; then
381 find /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots/ -type f -print0 \
382 | xargs -0 -r mv -f -t ${cfg.gcRootsDir}/
383 rmdir /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots
384 fi
385
386 chown hydra:hydra ${cfg.gcRootsDir}
387 chmod 2775 ${cfg.gcRootsDir}
388 '';
389 serviceConfig.ExecStart = "${hydra-package}/bin/hydra-init";
390 serviceConfig.PermissionsStartOnly = true;
391 serviceConfig.User = "hydra";
392 serviceConfig.Type = "oneshot";
393 serviceConfig.RemainAfterExit = true;
394 serviceConfig.Slice = "system-hydra.slice";
395 };
396
397 systemd.services.hydra-server = {
398 wantedBy = [ "multi-user.target" ];
399 requires = [ "hydra-init.service" ];
400 after = [ "hydra-init.service" ];
401 environment = serverEnv // {
402 HYDRA_DBI = "${serverEnv.HYDRA_DBI};application_name=hydra-server";
403 };
404 restartTriggers = [ hydraConf ];
405 serviceConfig = {
406 ExecStart =
407 "@${hydra-package}/bin/hydra-server hydra-server -f -h '${cfg.listenHost}' "
408 + "-p ${toString cfg.port} --min_spare_servers ${toString cfg.minSpareServers} --max_spare_servers ${toString cfg.maxSpareServers} "
409 + "--max_servers ${toString cfg.maxServers} --max_requests 100 ${lib.optionalString cfg.debugServer "-d"}";
410 User = "hydra-www";
411 PermissionsStartOnly = true;
412 Restart = "always";
413 Slice = "system-hydra.slice";
414 };
415 };
416
417 systemd.services.hydra-queue-runner = {
418 wantedBy = [ "multi-user.target" ];
419 requires = [ "hydra-init.service" ];
420 after = [
421 "hydra-init.service"
422 "network.target"
423 ];
424 path = [
425 config.nix.package
426 hydra-package
427 pkgs.bzip2
428 pkgs.hostname-debian
429 pkgs.openssh
430 ];
431 restartTriggers = [ hydraConf ];
432 environment = env // {
433 PGPASSFILE = "${baseDir}/pgpass-queue-runner"; # grrr
434 IN_SYSTEMD = "1"; # to get log severity levels
435 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-queue-runner";
436 };
437 serviceConfig = {
438 ExecStart = "@${hydra-package}/bin/hydra-queue-runner hydra-queue-runner -v";
439 ExecStopPost = "${hydra-package}/bin/hydra-queue-runner --unlock";
440 User = "hydra-queue-runner";
441 Restart = "always";
442 Slice = "system-hydra.slice";
443
444 # Ensure we can get core dumps.
445 LimitCORE = "infinity";
446 WorkingDirectory = "${baseDir}/queue-runner";
447 };
448 };
449
450 systemd.services.hydra-evaluator = {
451 wantedBy = [ "multi-user.target" ];
452 requires = [ "hydra-init.service" ];
453 wants = [ "network-online.target" ];
454 after = [
455 "hydra-init.service"
456 "network.target"
457 "network-online.target"
458 ];
459 path = with pkgs; [
460 hostname-debian
461 hydra-package
462 jq
463 ];
464 restartTriggers = [ hydraConf ];
465 environment = env // {
466 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-evaluator";
467 };
468 serviceConfig = {
469 ExecStart = "@${hydra-package}/bin/hydra-evaluator hydra-evaluator";
470 User = "hydra";
471 Restart = "always";
472 WorkingDirectory = baseDir;
473 Slice = "system-hydra.slice";
474 };
475 };
476
477 systemd.services.hydra-update-gc-roots = {
478 requires = [ "hydra-init.service" ];
479 after = [ "hydra-init.service" ];
480 environment = env // {
481 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-update-gc-roots";
482 };
483 serviceConfig = {
484 ExecStart = "@${hydra-package}/bin/hydra-update-gc-roots hydra-update-gc-roots";
485 User = "hydra";
486 Slice = "system-hydra.slice";
487 };
488 startAt = "2,14:15";
489 };
490
491 systemd.services.hydra-send-stats = {
492 wantedBy = [ "multi-user.target" ];
493 after = [ "hydra-init.service" ];
494 environment = env // {
495 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-send-stats";
496 };
497 serviceConfig = {
498 ExecStart = "@${hydra-package}/bin/hydra-send-stats hydra-send-stats";
499 User = "hydra";
500 Slice = "system-hydra.slice";
501 };
502 };
503
504 systemd.services.hydra-notify = {
505 wantedBy = [ "multi-user.target" ];
506 requires = [ "hydra-init.service" ];
507 after = [ "hydra-init.service" ];
508 restartTriggers = [ hydraConf ];
509 path = [ pkgs.zstd ];
510 environment = env // {
511 PGPASSFILE = "${baseDir}/pgpass-queue-runner";
512 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-notify";
513 };
514 serviceConfig = {
515 ExecStart = "@${hydra-package}/bin/hydra-notify hydra-notify";
516 # FIXME: run this under a less privileged user?
517 User = "hydra-queue-runner";
518 Restart = "always";
519 RestartSec = 5;
520 Slice = "system-hydra.slice";
521 };
522 };
523
524 # If there is less than a certain amount of free disk space, stop
525 # the queue/evaluator to prevent builds from failing or aborting.
526 systemd.services.hydra-check-space = {
527 script = ''
528 if [ $(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store))) -lt $((${toString cfg.minimumDiskFree} * 1024**3)) ]; then
529 echo "stopping Hydra queue runner due to lack of free space..."
530 systemctl stop hydra-queue-runner
531 fi
532 if [ $(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store))) -lt $((${toString cfg.minimumDiskFreeEvaluator} * 1024**3)) ]; then
533 echo "stopping Hydra evaluator due to lack of free space..."
534 systemctl stop hydra-evaluator
535 fi
536 '';
537 startAt = "*:0/5";
538 serviceConfig.Slice = "system-hydra.slice";
539 };
540
541 # Periodically compress build logs. The queue runner compresses
542 # logs automatically after a step finishes, but this doesn't work
543 # if the queue runner is stopped prematurely.
544 systemd.services.hydra-compress-logs = {
545 path = [
546 pkgs.bzip2
547 pkgs.zstd
548 ];
549 script = ''
550 set -eou pipefail
551 compression=$(sed -nr 's/compress_build_logs_compression = ()/\1/p' ${baseDir}/hydra.conf)
552 if [[ $compression == "" || $compression == bzip2 ]]; then
553 compressionCmd=(bzip2)
554 elif [[ $compression == zstd ]]; then
555 compressionCmd=(zstd --rm)
556 fi
557 find ${baseDir}/build-logs -ignore_readdir_race -type f -name "*.drv" -mtime +3 -size +0c -print0 | xargs -0 -r "''${compressionCmd[@]}" --force --quiet
558 '';
559 startAt = "Sun 01:45";
560 serviceConfig.Slice = "system-hydra.slice";
561 };
562
563 services.postgresql.enable = lib.mkIf haveLocalDB true;
564
565 services.postgresql.identMap = lib.optionalString haveLocalDB ''
566 hydra hydra hydra
567 hydra hydra-queue-runner hydra
568 hydra hydra-www hydra
569 hydra root hydra
570 '';
571
572 services.postgresql.authentication = lib.optionalString haveLocalDB ''
573 local all hydra peer map=hydra
574 '';
575
576 };
577
578}