pyrox.dev
1{ config, pkgs, lib, ... }:
2
3with lib;
4
5let
6
7 cfg = config.services.hydra;
8
9 baseDir = "/var/lib/hydra";
10
11 hydraConf = pkgs.writeScript "hydra.conf" cfg.extraConfig;
12
13 hydraEnv =
14 { HYDRA_DBI = cfg.dbi;
15 HYDRA_CONFIG = "${baseDir}/hydra.conf";
16 HYDRA_DATA = "${baseDir}";
17 };
18
19 env =
20 { NIX_REMOTE = "daemon";
21 SSL_CERT_FILE = "/etc/ssl/certs/ca-certificates.crt"; # Remove in 16.03
22 PGPASSFILE = "${baseDir}/pgpass";
23 NIX_REMOTE_SYSTEMS = concatStringsSep ":" cfg.buildMachinesFiles;
24 } // optionalAttrs (cfg.smtpHost != null) {
25 EMAIL_SENDER_TRANSPORT = "SMTP";
26 EMAIL_SENDER_TRANSPORT_host = cfg.smtpHost;
27 } // hydraEnv // cfg.extraEnv;
28
29 serverEnv = env //
30 { HYDRA_TRACKER = cfg.tracker;
31 XDG_CACHE_HOME = "${baseDir}/www/.cache";
32 COLUMNS = "80";
33 PGPASSFILE = "${baseDir}/pgpass-www"; # grrr
34 } // (optionalAttrs cfg.debugServer { DBIC_TRACE = "1"; });
35
36 localDB = "dbi:Pg:dbname=hydra;user=hydra;";
37
38 haveLocalDB = cfg.dbi == localDB;
39
40 hydra-package =
41 let
42 makeWrapperArgs = concatStringsSep " " (mapAttrsToList (key: value: "--set \"${key}\" \"${value}\"") hydraEnv);
43 in pkgs.buildEnv rec {
44 name = "hydra-env";
45 nativeBuildInputs = [ pkgs.makeWrapper ];
46 paths = [ cfg.package ];
47
48 postBuild = ''
49 if [ -L "$out/bin" ]; then
50 unlink "$out/bin"
51 fi
52 mkdir -p "$out/bin"
53
54 for path in ${concatStringsSep " " paths}; do
55 if [ -d "$path/bin" ]; then
56 cd "$path/bin"
57 for prg in *; do
58 if [ -f "$prg" ]; then
59 rm -f "$out/bin/$prg"
60 if [ -x "$prg" ]; then
61 makeWrapper "$path/bin/$prg" "$out/bin/$prg" ${makeWrapperArgs}
62 fi
63 fi
64 done
65 fi
66 done
67 '';
68 };
69
70in
71
72{
73 ###### interface
74 options = {
75
76 services.hydra = {
77
78 enable = mkOption {
79 type = types.bool;
80 default = false;
81 description = lib.mdDoc ''
82 Whether to run Hydra services.
83 '';
84 };
85
86 dbi = mkOption {
87 type = types.str;
88 default = localDB;
89 example = "dbi:Pg:dbname=hydra;host=postgres.example.org;user=foo;";
90 description = lib.mdDoc ''
91 The DBI string for Hydra database connection.
92
93 NOTE: Attempts to set `application_name` will be overridden by
94 `hydra-TYPE` (where TYPE is e.g. `evaluator`, `queue-runner`,
95 etc.) in all hydra services to more easily distinguish where
96 queries are coming from.
97 '';
98 };
99
100 package = mkOption {
101 type = types.package;
102 default = pkgs.hydra_unstable;
103 defaultText = literalExpression "pkgs.hydra_unstable";
104 description = lib.mdDoc "The Hydra package.";
105 };
106
107 hydraURL = mkOption {
108 type = types.str;
109 description = lib.mdDoc ''
110 The base URL for the Hydra webserver instance. Used for links in emails.
111 '';
112 };
113
114 listenHost = mkOption {
115 type = types.str;
116 default = "*";
117 example = "localhost";
118 description = lib.mdDoc ''
119 The hostname or address to listen on or `*` to listen
120 on all interfaces.
121 '';
122 };
123
124 port = mkOption {
125 type = types.int;
126 default = 3000;
127 description = lib.mdDoc ''
128 TCP port the web server should listen to.
129 '';
130 };
131
132 minimumDiskFree = mkOption {
133 type = types.int;
134 default = 0;
135 description = lib.mdDoc ''
136 Threshold of minimum disk space (GiB) to determine if the queue runner should run or not.
137 '';
138 };
139
140 minimumDiskFreeEvaluator = mkOption {
141 type = types.int;
142 default = 0;
143 description = lib.mdDoc ''
144 Threshold of minimum disk space (GiB) to determine if the evaluator should run or not.
145 '';
146 };
147
148 notificationSender = mkOption {
149 type = types.str;
150 description = lib.mdDoc ''
151 Sender email address used for email notifications.
152 '';
153 };
154
155 smtpHost = mkOption {
156 type = types.nullOr types.str;
157 default = null;
158 example = "localhost";
159 description = lib.mdDoc ''
160 Hostname of the SMTP server to use to send email.
161 '';
162 };
163
164 tracker = mkOption {
165 type = types.str;
166 default = "";
167 description = lib.mdDoc ''
168 Piece of HTML that is included on all pages.
169 '';
170 };
171
172 logo = mkOption {
173 type = types.nullOr types.path;
174 default = null;
175 description = lib.mdDoc ''
176 Path to a file containing the logo of your Hydra instance.
177 '';
178 };
179
180 debugServer = mkOption {
181 type = types.bool;
182 default = false;
183 description = lib.mdDoc "Whether to run the server in debug mode.";
184 };
185
186 extraConfig = mkOption {
187 type = types.lines;
188 description = lib.mdDoc "Extra lines for the Hydra configuration.";
189 };
190
191 extraEnv = mkOption {
192 type = types.attrsOf types.str;
193 default = {};
194 description = lib.mdDoc "Extra environment variables for Hydra.";
195 };
196
197 gcRootsDir = mkOption {
198 type = types.path;
199 default = "/nix/var/nix/gcroots/hydra";
200 description = lib.mdDoc "Directory that holds Hydra garbage collector roots.";
201 };
202
203 buildMachinesFiles = mkOption {
204 type = types.listOf types.path;
205 default = optional (config.nix.buildMachines != []) "/etc/nix/machines";
206 defaultText = literalExpression ''optional (config.nix.buildMachines != []) "/etc/nix/machines"'';
207 example = [ "/etc/nix/machines" "/var/lib/hydra/provisioner/machines" ];
208 description = lib.mdDoc "List of files containing build machines.";
209 };
210
211 useSubstitutes = mkOption {
212 type = types.bool;
213 default = false;
214 description = lib.mdDoc ''
215 Whether to use binary caches for downloading store paths. Note that
216 binary substitutions trigger (a potentially large number of) additional
217 HTTP requests that slow down the queue monitor thread significantly.
218 Also, this Hydra instance will serve those downloaded store paths to
219 its users with its own signature attached as if it had built them
220 itself, so don't enable this feature unless your active binary caches
221 are absolute trustworthy.
222 '';
223 };
224 };
225
226 };
227
228
229 ###### implementation
230
231 config = mkIf cfg.enable {
232
233 users.groups.hydra = {
234 gid = config.ids.gids.hydra;
235 };
236
237 users.users.hydra =
238 { description = "Hydra";
239 group = "hydra";
240 # We don't enable `createHome` here because the creation of the home directory is handled by the hydra-init service below.
241 home = baseDir;
242 useDefaultShell = true;
243 uid = config.ids.uids.hydra;
244 };
245
246 users.users.hydra-queue-runner =
247 { description = "Hydra queue runner";
248 group = "hydra";
249 useDefaultShell = true;
250 home = "${baseDir}/queue-runner"; # really only to keep SSH happy
251 uid = config.ids.uids.hydra-queue-runner;
252 };
253
254 users.users.hydra-www =
255 { description = "Hydra web server";
256 group = "hydra";
257 useDefaultShell = true;
258 uid = config.ids.uids.hydra-www;
259 };
260
261 services.hydra.extraConfig =
262 ''
263 using_frontend_proxy = 1
264 base_uri = ${cfg.hydraURL}
265 notification_sender = ${cfg.notificationSender}
266 max_servers = 25
267 ${optionalString (cfg.logo != null) ''
268 hydra_logo = ${cfg.logo}
269 ''}
270 gc_roots_dir = ${cfg.gcRootsDir}
271 use-substitutes = ${if cfg.useSubstitutes then "1" else "0"}
272 '';
273
274 environment.systemPackages = [ hydra-package ];
275
276 environment.variables = hydraEnv;
277
278 nix.settings = mkMerge [
279 {
280 keep-outputs = true;
281 keep-derivations = true;
282 trusted-users = [ "hydra-queue-runner" ];
283 }
284
285 (mkIf (versionOlder (getVersion config.nix.package.out) "2.4pre")
286 {
287 # The default (`true') slows Nix down a lot since the build farm
288 # has so many GC roots.
289 gc-check-reachability = false;
290 }
291 )
292 ];
293
294 systemd.services.hydra-init =
295 { wantedBy = [ "multi-user.target" ];
296 requires = optional haveLocalDB "postgresql.service";
297 after = optional haveLocalDB "postgresql.service";
298 environment = env // {
299 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-init";
300 };
301 path = [ pkgs.util-linux ];
302 preStart = ''
303 mkdir -p ${baseDir}
304 chown hydra:hydra ${baseDir}
305 chmod 0750 ${baseDir}
306
307 ln -sf ${hydraConf} ${baseDir}/hydra.conf
308
309 mkdir -m 0700 -p ${baseDir}/www
310 chown hydra-www:hydra ${baseDir}/www
311
312 mkdir -m 0700 -p ${baseDir}/queue-runner
313 mkdir -m 0750 -p ${baseDir}/build-logs
314 mkdir -m 0750 -p ${baseDir}/runcommand-logs
315 chown hydra-queue-runner.hydra \
316 ${baseDir}/queue-runner \
317 ${baseDir}/build-logs \
318 ${baseDir}/runcommand-logs
319
320 ${optionalString haveLocalDB ''
321 if ! [ -e ${baseDir}/.db-created ]; then
322 runuser -u ${config.services.postgresql.superUser} ${config.services.postgresql.package}/bin/createuser hydra
323 runuser -u ${config.services.postgresql.superUser} ${config.services.postgresql.package}/bin/createdb -- -O hydra hydra
324 touch ${baseDir}/.db-created
325 fi
326 echo "create extension if not exists pg_trgm" | runuser -u ${config.services.postgresql.superUser} -- ${config.services.postgresql.package}/bin/psql hydra
327 ''}
328
329 if [ ! -e ${cfg.gcRootsDir} ]; then
330
331 # Move legacy roots directory.
332 if [ -e /nix/var/nix/gcroots/per-user/hydra/hydra-roots ]; then
333 mv /nix/var/nix/gcroots/per-user/hydra/hydra-roots ${cfg.gcRootsDir}
334 fi
335
336 mkdir -p ${cfg.gcRootsDir}
337 fi
338
339 # Move legacy hydra-www roots.
340 if [ -e /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots ]; then
341 find /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots/ -type f \
342 | xargs -r mv -f -t ${cfg.gcRootsDir}/
343 rmdir /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots
344 fi
345
346 chown hydra:hydra ${cfg.gcRootsDir}
347 chmod 2775 ${cfg.gcRootsDir}
348 '';
349 serviceConfig.ExecStart = "${hydra-package}/bin/hydra-init";
350 serviceConfig.PermissionsStartOnly = true;
351 serviceConfig.User = "hydra";
352 serviceConfig.Type = "oneshot";
353 serviceConfig.RemainAfterExit = true;
354 };
355
356 systemd.services.hydra-server =
357 { wantedBy = [ "multi-user.target" ];
358 requires = [ "hydra-init.service" ];
359 after = [ "hydra-init.service" ];
360 environment = serverEnv // {
361 HYDRA_DBI = "${serverEnv.HYDRA_DBI};application_name=hydra-server";
362 };
363 restartTriggers = [ hydraConf ];
364 serviceConfig =
365 { ExecStart =
366 "@${hydra-package}/bin/hydra-server hydra-server -f -h '${cfg.listenHost}' "
367 + "-p ${toString cfg.port} --max_spare_servers 5 --max_servers 25 "
368 + "--max_requests 100 ${optionalString cfg.debugServer "-d"}";
369 User = "hydra-www";
370 PermissionsStartOnly = true;
371 Restart = "always";
372 };
373 };
374
375 systemd.services.hydra-queue-runner =
376 { wantedBy = [ "multi-user.target" ];
377 requires = [ "hydra-init.service" ];
378 after = [ "hydra-init.service" "network.target" ];
379 path = [ hydra-package pkgs.nettools pkgs.openssh pkgs.bzip2 config.nix.package ];
380 restartTriggers = [ hydraConf ];
381 environment = env // {
382 PGPASSFILE = "${baseDir}/pgpass-queue-runner"; # grrr
383 IN_SYSTEMD = "1"; # to get log severity levels
384 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-queue-runner";
385 };
386 serviceConfig =
387 { ExecStart = "@${hydra-package}/bin/hydra-queue-runner hydra-queue-runner -v";
388 ExecStopPost = "${hydra-package}/bin/hydra-queue-runner --unlock";
389 User = "hydra-queue-runner";
390 Restart = "always";
391
392 # Ensure we can get core dumps.
393 LimitCORE = "infinity";
394 WorkingDirectory = "${baseDir}/queue-runner";
395 };
396 };
397
398 systemd.services.hydra-evaluator =
399 { wantedBy = [ "multi-user.target" ];
400 requires = [ "hydra-init.service" ];
401 after = [ "hydra-init.service" "network.target" ];
402 path = with pkgs; [ hydra-package nettools jq ];
403 restartTriggers = [ hydraConf ];
404 environment = env // {
405 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-evaluator";
406 };
407 serviceConfig =
408 { ExecStart = "@${hydra-package}/bin/hydra-evaluator hydra-evaluator";
409 User = "hydra";
410 Restart = "always";
411 WorkingDirectory = baseDir;
412 };
413 };
414
415 systemd.services.hydra-update-gc-roots =
416 { requires = [ "hydra-init.service" ];
417 after = [ "hydra-init.service" ];
418 environment = env // {
419 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-update-gc-roots";
420 };
421 serviceConfig =
422 { ExecStart = "@${hydra-package}/bin/hydra-update-gc-roots hydra-update-gc-roots";
423 User = "hydra";
424 };
425 startAt = "2,14:15";
426 };
427
428 systemd.services.hydra-send-stats =
429 { wantedBy = [ "multi-user.target" ];
430 after = [ "hydra-init.service" ];
431 environment = env // {
432 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-send-stats";
433 };
434 serviceConfig =
435 { ExecStart = "@${hydra-package}/bin/hydra-send-stats hydra-send-stats";
436 User = "hydra";
437 };
438 };
439
440 systemd.services.hydra-notify =
441 { wantedBy = [ "multi-user.target" ];
442 requires = [ "hydra-init.service" ];
443 after = [ "hydra-init.service" ];
444 restartTriggers = [ hydraConf ];
445 environment = env // {
446 PGPASSFILE = "${baseDir}/pgpass-queue-runner";
447 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-notify";
448 };
449 serviceConfig =
450 { ExecStart = "@${hydra-package}/bin/hydra-notify hydra-notify";
451 # FIXME: run this under a less privileged user?
452 User = "hydra-queue-runner";
453 Restart = "always";
454 RestartSec = 5;
455 };
456 };
457
458 # If there is less than a certain amount of free disk space, stop
459 # the queue/evaluator to prevent builds from failing or aborting.
460 systemd.services.hydra-check-space =
461 { script =
462 ''
463 if [ $(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store))) -lt $((${toString cfg.minimumDiskFree} * 1024**3)) ]; then
464 echo "stopping Hydra queue runner due to lack of free space..."
465 systemctl stop hydra-queue-runner
466 fi
467 if [ $(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store))) -lt $((${toString cfg.minimumDiskFreeEvaluator} * 1024**3)) ]; then
468 echo "stopping Hydra evaluator due to lack of free space..."
469 systemctl stop hydra-evaluator
470 fi
471 '';
472 startAt = "*:0/5";
473 };
474
475 # Periodically compress build logs. The queue runner compresses
476 # logs automatically after a step finishes, but this doesn't work
477 # if the queue runner is stopped prematurely.
478 systemd.services.hydra-compress-logs =
479 { path = [ pkgs.bzip2 ];
480 script =
481 ''
482 find /var/lib/hydra/build-logs -type f -name "*.drv" -mtime +3 -size +0c | xargs -r bzip2 -v -f
483 '';
484 startAt = "Sun 01:45";
485 };
486
487 services.postgresql.enable = mkIf haveLocalDB true;
488
489 services.postgresql.identMap = optionalString haveLocalDB
490 ''
491 hydra-users hydra hydra
492 hydra-users hydra-queue-runner hydra
493 hydra-users hydra-www hydra
494 hydra-users root hydra
495 # The postgres user is used to create the pg_trgm extension for the hydra database
496 hydra-users postgres postgres
497 '';
498
499 services.postgresql.authentication = optionalString haveLocalDB
500 ''
501 local hydra all ident map=hydra-users
502 '';
503
504 };
505
506}