pyrox.dev
1{
2 config,
3 pkgs,
4 lib,
5 ...
6}:
7let
8
9 cfg = config.services.hydra;
10
11 baseDir = "/var/lib/hydra";
12
13 hydraConf = pkgs.writeScript "hydra.conf" cfg.extraConfig;
14
15 hydraEnv = {
16 HYDRA_DBI = cfg.dbi;
17 HYDRA_CONFIG = "${baseDir}/hydra.conf";
18 HYDRA_DATA = "${baseDir}";
19 };
20
21 env =
22 {
23 NIX_REMOTE = "daemon";
24 PGPASSFILE = "${baseDir}/pgpass";
25 NIX_REMOTE_SYSTEMS = lib.concatStringsSep ":" cfg.buildMachinesFiles;
26 }
27 // lib.optionalAttrs (cfg.smtpHost != null) {
28 EMAIL_SENDER_TRANSPORT = "SMTP";
29 EMAIL_SENDER_TRANSPORT_host = cfg.smtpHost;
30 }
31 // hydraEnv
32 // cfg.extraEnv;
33
34 serverEnv =
35 env
36 // {
37 HYDRA_TRACKER = cfg.tracker;
38 XDG_CACHE_HOME = "${baseDir}/www/.cache";
39 COLUMNS = "80";
40 PGPASSFILE = "${baseDir}/pgpass-www"; # grrr
41 }
42 // (lib.optionalAttrs cfg.debugServer { DBIC_TRACE = "1"; });
43
44 localDB = "dbi:Pg:dbname=hydra;user=hydra;";
45
46 haveLocalDB = cfg.dbi == localDB;
47
48 hydra-package =
49 let
50 makeWrapperArgs = lib.concatStringsSep " " (
51 lib.mapAttrsToList (key: value: "--set-default \"${key}\" \"${value}\"") hydraEnv
52 );
53 in
54 pkgs.buildEnv rec {
55 name = "hydra-env";
56 nativeBuildInputs = [ pkgs.makeWrapper ];
57 paths = [ cfg.package ];
58
59 postBuild = ''
60 if [ -L "$out/bin" ]; then
61 unlink "$out/bin"
62 fi
63 mkdir -p "$out/bin"
64
65 for path in ${lib.concatStringsSep " " paths}; do
66 if [ -d "$path/bin" ]; then
67 cd "$path/bin"
68 for prg in *; do
69 if [ -f "$prg" ]; then
70 rm -f "$out/bin/$prg"
71 if [ -x "$prg" ]; then
72 makeWrapper "$path/bin/$prg" "$out/bin/$prg" ${makeWrapperArgs}
73 fi
74 fi
75 done
76 fi
77 done
78 '';
79 };
80
81in
82
83{
84 ###### interface
85 options = {
86
87 services.hydra = {
88
89 enable = lib.mkOption {
90 type = lib.types.bool;
91 default = false;
92 description = ''
93 Whether to run Hydra services.
94 '';
95 };
96
97 dbi = lib.mkOption {
98 type = lib.types.str;
99 default = localDB;
100 example = "dbi:Pg:dbname=hydra;host=postgres.example.org;user=foo;";
101 description = ''
102 The DBI string for Hydra database connection.
103
104 NOTE: Attempts to set `application_name` will be overridden by
105 `hydra-TYPE` (where TYPE is e.g. `evaluator`, `queue-runner`,
106 etc.) in all hydra services to more easily distinguish where
107 queries are coming from.
108 '';
109 };
110
111 package = lib.mkPackageOption pkgs "hydra" { };
112
113 hydraURL = lib.mkOption {
114 type = lib.types.str;
115 description = ''
116 The base URL for the Hydra webserver instance. Used for links in emails.
117 '';
118 };
119
120 listenHost = lib.mkOption {
121 type = lib.types.str;
122 default = "*";
123 example = "localhost";
124 description = ''
125 The hostname or address to listen on or `*` to listen
126 on all interfaces.
127 '';
128 };
129
130 port = lib.mkOption {
131 type = lib.types.port;
132 default = 3000;
133 description = ''
134 TCP port the web server should listen to.
135 '';
136 };
137
138 minimumDiskFree = lib.mkOption {
139 type = lib.types.int;
140 default = 0;
141 description = ''
142 Threshold of minimum disk space (GiB) to determine if the queue runner should run or not.
143 '';
144 };
145
146 minimumDiskFreeEvaluator = lib.mkOption {
147 type = lib.types.int;
148 default = 0;
149 description = ''
150 Threshold of minimum disk space (GiB) to determine if the evaluator should run or not.
151 '';
152 };
153
154 notificationSender = lib.mkOption {
155 type = lib.types.str;
156 description = ''
157 Sender email address used for email notifications.
158 '';
159 };
160
161 smtpHost = lib.mkOption {
162 type = lib.types.nullOr lib.types.str;
163 default = null;
164 example = "localhost";
165 description = ''
166 Hostname of the SMTP server to use to send email.
167 '';
168 };
169
170 tracker = lib.mkOption {
171 type = lib.types.str;
172 default = "";
173 description = ''
174 Piece of HTML that is included on all pages.
175 '';
176 };
177
178 logo = lib.mkOption {
179 type = lib.types.nullOr lib.types.path;
180 default = null;
181 description = ''
182 Path to a file containing the logo of your Hydra instance.
183 '';
184 };
185
186 debugServer = lib.mkOption {
187 type = lib.types.bool;
188 default = false;
189 description = "Whether to run the server in debug mode.";
190 };
191
192 maxServers = lib.mkOption {
193 type = lib.types.int;
194 default = 25;
195 description = "Maximum number of starman workers to spawn.";
196 };
197
198 minSpareServers = lib.mkOption {
199 type = lib.types.int;
200 default = 4;
201 description = "Minimum number of spare starman workers to keep.";
202 };
203
204 maxSpareServers = lib.mkOption {
205 type = lib.types.int;
206 default = 5;
207 description = "Maximum number of spare starman workers to keep.";
208 };
209
210 extraConfig = lib.mkOption {
211 type = lib.types.lines;
212 description = "Extra lines for the Hydra configuration.";
213 };
214
215 extraEnv = lib.mkOption {
216 type = lib.types.attrsOf lib.types.str;
217 default = { };
218 description = "Extra environment variables for Hydra.";
219 };
220
221 gcRootsDir = lib.mkOption {
222 type = lib.types.path;
223 default = "/nix/var/nix/gcroots/hydra";
224 description = "Directory that holds Hydra garbage collector roots.";
225 };
226
227 buildMachinesFiles = lib.mkOption {
228 type = lib.types.listOf lib.types.path;
229 default = lib.optional (config.nix.buildMachines != [ ]) "/etc/nix/machines";
230 defaultText = lib.literalExpression ''lib.optional (config.nix.buildMachines != []) "/etc/nix/machines"'';
231 example = [
232 "/etc/nix/machines"
233 "/var/lib/hydra/provisioner/machines"
234 ];
235 description = "List of files containing build machines.";
236 };
237
238 useSubstitutes = lib.mkOption {
239 type = lib.types.bool;
240 default = false;
241 description = ''
242 Whether to use binary caches for downloading store paths. Note that
243 binary substitutions trigger (a potentially large number of) additional
244 HTTP requests that slow down the queue monitor thread significantly.
245 Also, this Hydra instance will serve those downloaded store paths to
246 its users with its own signature attached as if it had built them
247 itself, so don't enable this feature unless your active binary caches
248 are absolute trustworthy.
249 '';
250 };
251 };
252
253 };
254
255 ###### implementation
256
257 config = lib.mkIf cfg.enable {
258 assertions = [
259 {
260 assertion = cfg.maxServers != 0 && cfg.maxSpareServers != 0 && cfg.minSpareServers != 0;
261 message = "services.hydra.{minSpareServers,maxSpareServers,minSpareServers} cannot be 0";
262 }
263 {
264 assertion = cfg.minSpareServers < cfg.maxSpareServers;
265 message = "services.hydra.minSpareServers cannot be bigger than services.hydra.maxSpareServers";
266 }
267 ];
268
269 users.groups.hydra = {
270 gid = config.ids.gids.hydra;
271 };
272
273 users.users.hydra = {
274 description = "Hydra";
275 group = "hydra";
276 # We don't enable `createHome` here because the creation of the home directory is handled by the hydra-init service below.
277 home = baseDir;
278 useDefaultShell = true;
279 uid = config.ids.uids.hydra;
280 };
281
282 users.users.hydra-queue-runner = {
283 description = "Hydra queue runner";
284 group = "hydra";
285 useDefaultShell = true;
286 home = "${baseDir}/queue-runner"; # really only to keep SSH happy
287 uid = config.ids.uids.hydra-queue-runner;
288 };
289
290 users.users.hydra-www = {
291 description = "Hydra web server";
292 group = "hydra";
293 useDefaultShell = true;
294 uid = config.ids.uids.hydra-www;
295 };
296
297 services.hydra.extraConfig = ''
298 using_frontend_proxy = 1
299 base_uri = ${cfg.hydraURL}
300 notification_sender = ${cfg.notificationSender}
301 max_servers = ${toString cfg.maxServers}
302 ${lib.optionalString (cfg.logo != null) ''
303 hydra_logo = ${cfg.logo}
304 ''}
305 gc_roots_dir = ${cfg.gcRootsDir}
306 use-substitutes = ${if cfg.useSubstitutes then "1" else "0"}
307 '';
308
309 environment.systemPackages = [ hydra-package ];
310
311 environment.variables = hydraEnv;
312
313 nix.settings = lib.mkMerge [
314 {
315 keep-outputs = true;
316 keep-derivations = true;
317 trusted-users = [ "hydra-queue-runner" ];
318 }
319
320 (lib.mkIf (lib.versionOlder (lib.getVersion config.nix.package.out) "2.4pre") {
321 # The default (`true') slows Nix down a lot since the build farm
322 # has so many GC roots.
323 gc-check-reachability = false;
324 })
325 ];
326
327 systemd.slices.system-hydra = {
328 description = "Hydra CI Server Slice";
329 documentation = [
330 "file://${cfg.package}/share/doc/hydra/index.html"
331 "https://nixos.org/hydra/manual/"
332 ];
333 };
334
335 systemd.services.hydra-init = {
336 wantedBy = [ "multi-user.target" ];
337 requires = lib.optional haveLocalDB "postgresql.service";
338 after = lib.optional haveLocalDB "postgresql.service";
339 environment = env // {
340 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-init";
341 };
342 path = [ pkgs.util-linux ];
343 preStart = ''
344 mkdir -p ${baseDir}
345 chown hydra:hydra ${baseDir}
346 chmod 0750 ${baseDir}
347
348 ln -sf ${hydraConf} ${baseDir}/hydra.conf
349
350 mkdir -m 0700 ${baseDir}/www || true
351 chown hydra-www:hydra ${baseDir}/www
352
353 mkdir -m 0700 ${baseDir}/queue-runner || true
354 mkdir -m 0750 ${baseDir}/build-logs || true
355 mkdir -m 0750 ${baseDir}/runcommand-logs || true
356 chown hydra-queue-runner:hydra \
357 ${baseDir}/queue-runner \
358 ${baseDir}/build-logs \
359 ${baseDir}/runcommand-logs
360
361 ${lib.optionalString haveLocalDB ''
362 if ! [ -e ${baseDir}/.db-created ]; then
363 runuser -u ${config.services.postgresql.superUser} ${config.services.postgresql.package}/bin/createuser hydra
364 runuser -u ${config.services.postgresql.superUser} ${config.services.postgresql.package}/bin/createdb -- -O hydra hydra
365 touch ${baseDir}/.db-created
366 fi
367 echo "create extension if not exists pg_trgm" | runuser -u ${config.services.postgresql.superUser} -- ${config.services.postgresql.package}/bin/psql hydra
368 ''}
369
370 if [ ! -e ${cfg.gcRootsDir} ]; then
371
372 # Move legacy roots directory.
373 if [ -e /nix/var/nix/gcroots/per-user/hydra/hydra-roots ]; then
374 mv /nix/var/nix/gcroots/per-user/hydra/hydra-roots ${cfg.gcRootsDir}
375 fi
376
377 mkdir -p ${cfg.gcRootsDir}
378 fi
379
380 # Move legacy hydra-www roots.
381 if [ -e /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots ]; then
382 find /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots/ -type f -print0 \
383 | xargs -0 -r mv -f -t ${cfg.gcRootsDir}/
384 rmdir /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots
385 fi
386
387 chown hydra:hydra ${cfg.gcRootsDir}
388 chmod 2775 ${cfg.gcRootsDir}
389 '';
390 serviceConfig.ExecStart = "${hydra-package}/bin/hydra-init";
391 serviceConfig.PermissionsStartOnly = true;
392 serviceConfig.User = "hydra";
393 serviceConfig.Type = "oneshot";
394 serviceConfig.RemainAfterExit = true;
395 serviceConfig.Slice = "system-hydra.slice";
396 };
397
398 systemd.services.hydra-server = {
399 wantedBy = [ "multi-user.target" ];
400 requires = [ "hydra-init.service" ];
401 after = [ "hydra-init.service" ];
402 environment = serverEnv // {
403 HYDRA_DBI = "${serverEnv.HYDRA_DBI};application_name=hydra-server";
404 };
405 restartTriggers = [ hydraConf ];
406 serviceConfig = {
407 ExecStart =
408 "@${hydra-package}/bin/hydra-server hydra-server -f -h '${cfg.listenHost}' "
409 + "-p ${toString cfg.port} --min_spare_servers ${toString cfg.minSpareServers} --max_spare_servers ${toString cfg.maxSpareServers} "
410 + "--max_servers ${toString cfg.maxServers} --max_requests 100 ${lib.optionalString cfg.debugServer "-d"}";
411 User = "hydra-www";
412 PermissionsStartOnly = true;
413 Restart = "always";
414 Slice = "system-hydra.slice";
415 };
416 };
417
418 systemd.services.hydra-queue-runner = {
419 wantedBy = [ "multi-user.target" ];
420 requires = [ "hydra-init.service" ];
421 after = [
422 "hydra-init.service"
423 "network.target"
424 ];
425 path = [
426 hydra-package
427 pkgs.nettools
428 pkgs.openssh
429 pkgs.bzip2
430 config.nix.package
431 ];
432 restartTriggers = [ hydraConf ];
433 environment = env // {
434 PGPASSFILE = "${baseDir}/pgpass-queue-runner"; # grrr
435 IN_SYSTEMD = "1"; # to get log severity levels
436 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-queue-runner";
437 };
438 serviceConfig = {
439 ExecStart = "@${hydra-package}/bin/hydra-queue-runner hydra-queue-runner -v";
440 ExecStopPost = "${hydra-package}/bin/hydra-queue-runner --unlock";
441 User = "hydra-queue-runner";
442 Restart = "always";
443 Slice = "system-hydra.slice";
444
445 # Ensure we can get core dumps.
446 LimitCORE = "infinity";
447 WorkingDirectory = "${baseDir}/queue-runner";
448 };
449 };
450
451 systemd.services.hydra-evaluator = {
452 wantedBy = [ "multi-user.target" ];
453 requires = [ "hydra-init.service" ];
454 wants = [ "network-online.target" ];
455 after = [
456 "hydra-init.service"
457 "network.target"
458 "network-online.target"
459 ];
460 path = with pkgs; [
461 hydra-package
462 nettools
463 jq
464 ];
465 restartTriggers = [ hydraConf ];
466 environment = env // {
467 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-evaluator";
468 };
469 serviceConfig = {
470 ExecStart = "@${hydra-package}/bin/hydra-evaluator hydra-evaluator";
471 User = "hydra";
472 Restart = "always";
473 WorkingDirectory = baseDir;
474 Slice = "system-hydra.slice";
475 };
476 };
477
478 systemd.services.hydra-update-gc-roots = {
479 requires = [ "hydra-init.service" ];
480 after = [ "hydra-init.service" ];
481 environment = env // {
482 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-update-gc-roots";
483 };
484 serviceConfig = {
485 ExecStart = "@${hydra-package}/bin/hydra-update-gc-roots hydra-update-gc-roots";
486 User = "hydra";
487 Slice = "system-hydra.slice";
488 };
489 startAt = "2,14:15";
490 };
491
492 systemd.services.hydra-send-stats = {
493 wantedBy = [ "multi-user.target" ];
494 after = [ "hydra-init.service" ];
495 environment = env // {
496 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-send-stats";
497 };
498 serviceConfig = {
499 ExecStart = "@${hydra-package}/bin/hydra-send-stats hydra-send-stats";
500 User = "hydra";
501 Slice = "system-hydra.slice";
502 };
503 };
504
505 systemd.services.hydra-notify = {
506 wantedBy = [ "multi-user.target" ];
507 requires = [ "hydra-init.service" ];
508 after = [ "hydra-init.service" ];
509 restartTriggers = [ hydraConf ];
510 path = [ pkgs.zstd ];
511 environment = env // {
512 PGPASSFILE = "${baseDir}/pgpass-queue-runner";
513 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-notify";
514 };
515 serviceConfig = {
516 ExecStart = "@${hydra-package}/bin/hydra-notify hydra-notify";
517 # FIXME: run this under a less privileged user?
518 User = "hydra-queue-runner";
519 Restart = "always";
520 RestartSec = 5;
521 Slice = "system-hydra.slice";
522 };
523 };
524
525 # If there is less than a certain amount of free disk space, stop
526 # the queue/evaluator to prevent builds from failing or aborting.
527 systemd.services.hydra-check-space = {
528 script = ''
529 if [ $(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store))) -lt $((${toString cfg.minimumDiskFree} * 1024**3)) ]; then
530 echo "stopping Hydra queue runner due to lack of free space..."
531 systemctl stop hydra-queue-runner
532 fi
533 if [ $(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store))) -lt $((${toString cfg.minimumDiskFreeEvaluator} * 1024**3)) ]; then
534 echo "stopping Hydra evaluator due to lack of free space..."
535 systemctl stop hydra-evaluator
536 fi
537 '';
538 startAt = "*:0/5";
539 serviceConfig.Slice = "system-hydra.slice";
540 };
541
542 # Periodically compress build logs. The queue runner compresses
543 # logs automatically after a step finishes, but this doesn't work
544 # if the queue runner is stopped prematurely.
545 systemd.services.hydra-compress-logs = {
546 path = [
547 pkgs.bzip2
548 pkgs.zstd
549 ];
550 script = ''
551 set -eou pipefail
552 compression=$(sed -nr 's/compress_build_logs_compression = ()/\1/p' ${baseDir}/hydra.conf)
553 if [[ $compression == "" || $compression == bzip2 ]]; then
554 compressionCmd=(bzip2)
555 elif [[ $compression == zstd ]]; then
556 compressionCmd=(zstd --rm)
557 fi
558 find ${baseDir}/build-logs -ignore_readdir_race -type f -name "*.drv" -mtime +3 -size +0c -print0 | xargs -0 -r "''${compressionCmd[@]}" --force --quiet
559 '';
560 startAt = "Sun 01:45";
561 serviceConfig.Slice = "system-hydra.slice";
562 };
563
564 services.postgresql.enable = lib.mkIf haveLocalDB true;
565
566 services.postgresql.identMap = lib.optionalString haveLocalDB ''
567 hydra hydra hydra
568 hydra hydra-queue-runner hydra
569 hydra hydra-www hydra
570 hydra root hydra
571 '';
572
573 services.postgresql.authentication = lib.optionalString haveLocalDB ''
574 local all hydra peer map=hydra
575 '';
576
577 };
578
579}