pyrox.dev
1{ config, pkgs, lib, ... }:
2
3with lib;
4
5let
6
7 cfg = config.services.hydra;
8
9 baseDir = "/var/lib/hydra";
10
11 hydraConf = pkgs.writeScript "hydra.conf" cfg.extraConfig;
12
13 hydraEnv =
14 { HYDRA_DBI = cfg.dbi;
15 HYDRA_CONFIG = "${baseDir}/hydra.conf";
16 HYDRA_DATA = "${baseDir}";
17 };
18
19 env =
20 { NIX_REMOTE = "daemon";
21 SSL_CERT_FILE = "/etc/ssl/certs/ca-certificates.crt"; # Remove in 16.03
22 PGPASSFILE = "${baseDir}/pgpass";
23 NIX_REMOTE_SYSTEMS = concatStringsSep ":" cfg.buildMachinesFiles;
24 } // optionalAttrs (cfg.smtpHost != null) {
25 EMAIL_SENDER_TRANSPORT = "SMTP";
26 EMAIL_SENDER_TRANSPORT_host = cfg.smtpHost;
27 } // hydraEnv // cfg.extraEnv;
28
29 serverEnv = env //
30 { HYDRA_TRACKER = cfg.tracker;
31 XDG_CACHE_HOME = "${baseDir}/www/.cache";
32 COLUMNS = "80";
33 PGPASSFILE = "${baseDir}/pgpass-www"; # grrr
34 } // (optionalAttrs cfg.debugServer { DBIC_TRACE = "1"; });
35
36 localDB = "dbi:Pg:dbname=hydra;user=hydra;";
37
38 haveLocalDB = cfg.dbi == localDB;
39
40 hydra-package =
41 let
42 makeWrapperArgs = concatStringsSep " " (mapAttrsToList (key: value: "--set-default \"${key}\" \"${value}\"") hydraEnv);
43 in pkgs.buildEnv rec {
44 name = "hydra-env";
45 nativeBuildInputs = [ pkgs.makeWrapper ];
46 paths = [ cfg.package ];
47
48 postBuild = ''
49 if [ -L "$out/bin" ]; then
50 unlink "$out/bin"
51 fi
52 mkdir -p "$out/bin"
53
54 for path in ${concatStringsSep " " paths}; do
55 if [ -d "$path/bin" ]; then
56 cd "$path/bin"
57 for prg in *; do
58 if [ -f "$prg" ]; then
59 rm -f "$out/bin/$prg"
60 if [ -x "$prg" ]; then
61 makeWrapper "$path/bin/$prg" "$out/bin/$prg" ${makeWrapperArgs}
62 fi
63 fi
64 done
65 fi
66 done
67 '';
68 };
69
70in
71
72{
73 ###### interface
74 options = {
75
76 services.hydra = {
77
78 enable = mkOption {
79 type = types.bool;
80 default = false;
81 description = ''
82 Whether to run Hydra services.
83 '';
84 };
85
86 dbi = mkOption {
87 type = types.str;
88 default = localDB;
89 example = "dbi:Pg:dbname=hydra;host=postgres.example.org;user=foo;";
90 description = ''
91 The DBI string for Hydra database connection.
92
93 NOTE: Attempts to set `application_name` will be overridden by
94 `hydra-TYPE` (where TYPE is e.g. `evaluator`, `queue-runner`,
95 etc.) in all hydra services to more easily distinguish where
96 queries are coming from.
97 '';
98 };
99
100 package = mkPackageOption pkgs "hydra_unstable" { };
101
102 hydraURL = mkOption {
103 type = types.str;
104 description = ''
105 The base URL for the Hydra webserver instance. Used for links in emails.
106 '';
107 };
108
109 listenHost = mkOption {
110 type = types.str;
111 default = "*";
112 example = "localhost";
113 description = ''
114 The hostname or address to listen on or `*` to listen
115 on all interfaces.
116 '';
117 };
118
119 port = mkOption {
120 type = types.port;
121 default = 3000;
122 description = ''
123 TCP port the web server should listen to.
124 '';
125 };
126
127 minimumDiskFree = mkOption {
128 type = types.int;
129 default = 0;
130 description = ''
131 Threshold of minimum disk space (GiB) to determine if the queue runner should run or not.
132 '';
133 };
134
135 minimumDiskFreeEvaluator = mkOption {
136 type = types.int;
137 default = 0;
138 description = ''
139 Threshold of minimum disk space (GiB) to determine if the evaluator should run or not.
140 '';
141 };
142
143 notificationSender = mkOption {
144 type = types.str;
145 description = ''
146 Sender email address used for email notifications.
147 '';
148 };
149
150 smtpHost = mkOption {
151 type = types.nullOr types.str;
152 default = null;
153 example = "localhost";
154 description = ''
155 Hostname of the SMTP server to use to send email.
156 '';
157 };
158
159 tracker = mkOption {
160 type = types.str;
161 default = "";
162 description = ''
163 Piece of HTML that is included on all pages.
164 '';
165 };
166
167 logo = mkOption {
168 type = types.nullOr types.path;
169 default = null;
170 description = ''
171 Path to a file containing the logo of your Hydra instance.
172 '';
173 };
174
175 debugServer = mkOption {
176 type = types.bool;
177 default = false;
178 description = "Whether to run the server in debug mode.";
179 };
180
181 maxServers = mkOption {
182 type = types.int;
183 default = 25;
184 description = "Maximum number of starman workers to spawn.";
185 };
186
187 minSpareServers = mkOption {
188 type = types.int;
189 default = 4;
190 description = "Minimum number of spare starman workers to keep.";
191 };
192
193 maxSpareServers = mkOption {
194 type = types.int;
195 default = 5;
196 description = "Maximum number of spare starman workers to keep.";
197 };
198
199 extraConfig = mkOption {
200 type = types.lines;
201 description = "Extra lines for the Hydra configuration.";
202 };
203
204 extraEnv = mkOption {
205 type = types.attrsOf types.str;
206 default = {};
207 description = "Extra environment variables for Hydra.";
208 };
209
210 gcRootsDir = mkOption {
211 type = types.path;
212 default = "/nix/var/nix/gcroots/hydra";
213 description = "Directory that holds Hydra garbage collector roots.";
214 };
215
216 buildMachinesFiles = mkOption {
217 type = types.listOf types.path;
218 default = optional (config.nix.buildMachines != []) "/etc/nix/machines";
219 defaultText = literalExpression ''optional (config.nix.buildMachines != []) "/etc/nix/machines"'';
220 example = [ "/etc/nix/machines" "/var/lib/hydra/provisioner/machines" ];
221 description = "List of files containing build machines.";
222 };
223
224 useSubstitutes = mkOption {
225 type = types.bool;
226 default = false;
227 description = ''
228 Whether to use binary caches for downloading store paths. Note that
229 binary substitutions trigger (a potentially large number of) additional
230 HTTP requests that slow down the queue monitor thread significantly.
231 Also, this Hydra instance will serve those downloaded store paths to
232 its users with its own signature attached as if it had built them
233 itself, so don't enable this feature unless your active binary caches
234 are absolute trustworthy.
235 '';
236 };
237 };
238
239 };
240
241
242 ###### implementation
243
244 config = mkIf cfg.enable {
245 assertions = [
246 {
247 assertion = cfg.maxServers != 0 && cfg.maxSpareServers != 0 && cfg.minSpareServers != 0;
248 message = "services.hydra.{minSpareServers,maxSpareServers,minSpareServers} cannot be 0";
249 }
250 {
251 assertion = cfg.minSpareServers < cfg.maxSpareServers;
252 message = "services.hydra.minSpareServers cannot be bigger than servives.hydra.maxSpareServers";
253 }
254 ];
255
256 users.groups.hydra = {
257 gid = config.ids.gids.hydra;
258 };
259
260 users.users.hydra =
261 { description = "Hydra";
262 group = "hydra";
263 # We don't enable `createHome` here because the creation of the home directory is handled by the hydra-init service below.
264 home = baseDir;
265 useDefaultShell = true;
266 uid = config.ids.uids.hydra;
267 };
268
269 users.users.hydra-queue-runner =
270 { description = "Hydra queue runner";
271 group = "hydra";
272 useDefaultShell = true;
273 home = "${baseDir}/queue-runner"; # really only to keep SSH happy
274 uid = config.ids.uids.hydra-queue-runner;
275 };
276
277 users.users.hydra-www =
278 { description = "Hydra web server";
279 group = "hydra";
280 useDefaultShell = true;
281 uid = config.ids.uids.hydra-www;
282 };
283
284 services.hydra.extraConfig =
285 ''
286 using_frontend_proxy = 1
287 base_uri = ${cfg.hydraURL}
288 notification_sender = ${cfg.notificationSender}
289 max_servers = ${toString cfg.maxServers}
290 ${optionalString (cfg.logo != null) ''
291 hydra_logo = ${cfg.logo}
292 ''}
293 gc_roots_dir = ${cfg.gcRootsDir}
294 use-substitutes = ${if cfg.useSubstitutes then "1" else "0"}
295 '';
296
297 environment.systemPackages = [ hydra-package ];
298
299 environment.variables = hydraEnv;
300
301 nix.settings = mkMerge [
302 {
303 keep-outputs = true;
304 keep-derivations = true;
305 trusted-users = [ "hydra-queue-runner" ];
306 }
307
308 (mkIf (versionOlder (getVersion config.nix.package.out) "2.4pre")
309 {
310 # The default (`true') slows Nix down a lot since the build farm
311 # has so many GC roots.
312 gc-check-reachability = false;
313 }
314 )
315 ];
316
317 systemd.services.hydra-init =
318 { wantedBy = [ "multi-user.target" ];
319 requires = optional haveLocalDB "postgresql.service";
320 after = optional haveLocalDB "postgresql.service";
321 environment = env // {
322 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-init";
323 };
324 path = [ pkgs.util-linux ];
325 preStart = ''
326 mkdir -p ${baseDir}
327 chown hydra:hydra ${baseDir}
328 chmod 0750 ${baseDir}
329
330 ln -sf ${hydraConf} ${baseDir}/hydra.conf
331
332 mkdir -m 0700 -p ${baseDir}/www
333 chown hydra-www:hydra ${baseDir}/www
334
335 mkdir -m 0700 -p ${baseDir}/queue-runner
336 mkdir -m 0750 -p ${baseDir}/build-logs
337 mkdir -m 0750 -p ${baseDir}/runcommand-logs
338 chown hydra-queue-runner.hydra \
339 ${baseDir}/queue-runner \
340 ${baseDir}/build-logs \
341 ${baseDir}/runcommand-logs
342
343 ${optionalString haveLocalDB ''
344 if ! [ -e ${baseDir}/.db-created ]; then
345 runuser -u ${config.services.postgresql.superUser} ${config.services.postgresql.package}/bin/createuser hydra
346 runuser -u ${config.services.postgresql.superUser} ${config.services.postgresql.package}/bin/createdb -- -O hydra hydra
347 touch ${baseDir}/.db-created
348 fi
349 echo "create extension if not exists pg_trgm" | runuser -u ${config.services.postgresql.superUser} -- ${config.services.postgresql.package}/bin/psql hydra
350 ''}
351
352 if [ ! -e ${cfg.gcRootsDir} ]; then
353
354 # Move legacy roots directory.
355 if [ -e /nix/var/nix/gcroots/per-user/hydra/hydra-roots ]; then
356 mv /nix/var/nix/gcroots/per-user/hydra/hydra-roots ${cfg.gcRootsDir}
357 fi
358
359 mkdir -p ${cfg.gcRootsDir}
360 fi
361
362 # Move legacy hydra-www roots.
363 if [ -e /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots ]; then
364 find /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots/ -type f \
365 | xargs -r mv -f -t ${cfg.gcRootsDir}/
366 rmdir /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots
367 fi
368
369 chown hydra:hydra ${cfg.gcRootsDir}
370 chmod 2775 ${cfg.gcRootsDir}
371 '';
372 serviceConfig.ExecStart = "${hydra-package}/bin/hydra-init";
373 serviceConfig.PermissionsStartOnly = true;
374 serviceConfig.User = "hydra";
375 serviceConfig.Type = "oneshot";
376 serviceConfig.RemainAfterExit = true;
377 };
378
379 systemd.services.hydra-server =
380 { wantedBy = [ "multi-user.target" ];
381 requires = [ "hydra-init.service" ];
382 after = [ "hydra-init.service" ];
383 environment = serverEnv // {
384 HYDRA_DBI = "${serverEnv.HYDRA_DBI};application_name=hydra-server";
385 };
386 restartTriggers = [ hydraConf ];
387 serviceConfig =
388 { ExecStart =
389 "@${hydra-package}/bin/hydra-server hydra-server -f -h '${cfg.listenHost}' "
390 + "-p ${toString cfg.port} --min_spare_servers ${toString cfg.minSpareServers} --max_spare_servers ${toString cfg.maxSpareServers} "
391 + "--max_servers ${toString cfg.maxServers} --max_requests 100 ${optionalString cfg.debugServer "-d"}";
392 User = "hydra-www";
393 PermissionsStartOnly = true;
394 Restart = "always";
395 };
396 };
397
398 systemd.services.hydra-queue-runner =
399 { wantedBy = [ "multi-user.target" ];
400 requires = [ "hydra-init.service" ];
401 after = [ "hydra-init.service" "network.target" ];
402 path = [ hydra-package pkgs.nettools pkgs.openssh pkgs.bzip2 config.nix.package ];
403 restartTriggers = [ hydraConf ];
404 environment = env // {
405 PGPASSFILE = "${baseDir}/pgpass-queue-runner"; # grrr
406 IN_SYSTEMD = "1"; # to get log severity levels
407 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-queue-runner";
408 };
409 serviceConfig =
410 { ExecStart = "@${hydra-package}/bin/hydra-queue-runner hydra-queue-runner -v";
411 ExecStopPost = "${hydra-package}/bin/hydra-queue-runner --unlock";
412 User = "hydra-queue-runner";
413 Restart = "always";
414
415 # Ensure we can get core dumps.
416 LimitCORE = "infinity";
417 WorkingDirectory = "${baseDir}/queue-runner";
418 };
419 };
420
421 systemd.services.hydra-evaluator =
422 { wantedBy = [ "multi-user.target" ];
423 requires = [ "hydra-init.service" ];
424 wants = [ "network-online.target" ];
425 after = [ "hydra-init.service" "network.target" "network-online.target" ];
426 path = with pkgs; [ hydra-package nettools jq ];
427 restartTriggers = [ hydraConf ];
428 environment = env // {
429 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-evaluator";
430 };
431 serviceConfig =
432 { ExecStart = "@${hydra-package}/bin/hydra-evaluator hydra-evaluator";
433 User = "hydra";
434 Restart = "always";
435 WorkingDirectory = baseDir;
436 };
437 };
438
439 systemd.services.hydra-update-gc-roots =
440 { requires = [ "hydra-init.service" ];
441 after = [ "hydra-init.service" ];
442 environment = env // {
443 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-update-gc-roots";
444 };
445 serviceConfig =
446 { ExecStart = "@${hydra-package}/bin/hydra-update-gc-roots hydra-update-gc-roots";
447 User = "hydra";
448 };
449 startAt = "2,14:15";
450 };
451
452 systemd.services.hydra-send-stats =
453 { wantedBy = [ "multi-user.target" ];
454 after = [ "hydra-init.service" ];
455 environment = env // {
456 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-send-stats";
457 };
458 serviceConfig =
459 { ExecStart = "@${hydra-package}/bin/hydra-send-stats hydra-send-stats";
460 User = "hydra";
461 };
462 };
463
464 systemd.services.hydra-notify =
465 { wantedBy = [ "multi-user.target" ];
466 requires = [ "hydra-init.service" ];
467 after = [ "hydra-init.service" ];
468 restartTriggers = [ hydraConf ];
469 environment = env // {
470 PGPASSFILE = "${baseDir}/pgpass-queue-runner";
471 HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-notify";
472 };
473 serviceConfig =
474 { ExecStart = "@${hydra-package}/bin/hydra-notify hydra-notify";
475 # FIXME: run this under a less privileged user?
476 User = "hydra-queue-runner";
477 Restart = "always";
478 RestartSec = 5;
479 };
480 };
481
482 # If there is less than a certain amount of free disk space, stop
483 # the queue/evaluator to prevent builds from failing or aborting.
484 systemd.services.hydra-check-space =
485 { script =
486 ''
487 if [ $(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store))) -lt $((${toString cfg.minimumDiskFree} * 1024**3)) ]; then
488 echo "stopping Hydra queue runner due to lack of free space..."
489 systemctl stop hydra-queue-runner
490 fi
491 if [ $(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store))) -lt $((${toString cfg.minimumDiskFreeEvaluator} * 1024**3)) ]; then
492 echo "stopping Hydra evaluator due to lack of free space..."
493 systemctl stop hydra-evaluator
494 fi
495 '';
496 startAt = "*:0/5";
497 };
498
499 # Periodically compress build logs. The queue runner compresses
500 # logs automatically after a step finishes, but this doesn't work
501 # if the queue runner is stopped prematurely.
502 systemd.services.hydra-compress-logs =
503 { path = [ pkgs.bzip2 ];
504 script =
505 ''
506 find /var/lib/hydra/build-logs -type f -name "*.drv" -mtime +3 -size +0c | xargs -r bzip2 -v -f
507 '';
508 startAt = "Sun 01:45";
509 };
510
511 services.postgresql.enable = mkIf haveLocalDB true;
512
513 services.postgresql.identMap = optionalString haveLocalDB
514 ''
515 hydra-users hydra hydra
516 hydra-users hydra-queue-runner hydra
517 hydra-users hydra-www hydra
518 hydra-users root hydra
519 # The postgres user is used to create the pg_trgm extension for the hydra database
520 hydra-users postgres postgres
521 '';
522
523 services.postgresql.authentication = optionalString haveLocalDB
524 ''
525 local hydra all ident map=hydra-users
526 '';
527
528 };
529
530}