pyrox.dev
1{ config, pkgs, lib, ... }:
2
3with lib;
4
5let
6
7 cfg = config.services.hydra;
8
9 baseDir = "/var/lib/hydra";
10
11 hydraConf = pkgs.writeScript "hydra.conf" cfg.extraConfig;
12
13 hydraEnv =
14 { HYDRA_DBI = cfg.dbi;
15 HYDRA_CONFIG = "${baseDir}/hydra.conf";
16 HYDRA_DATA = "${baseDir}";
17 };
18
19 env =
20 { NIX_REMOTE = "daemon";
21 SSL_CERT_FILE = "/etc/ssl/certs/ca-certificates.crt"; # Remove in 16.03
22 PGPASSFILE = "${baseDir}/pgpass";
23 NIX_REMOTE_SYSTEMS = concatStringsSep ":" cfg.buildMachinesFiles;
24 } // optionalAttrs (cfg.smtpHost != null) {
25 EMAIL_SENDER_TRANSPORT = "SMTP";
26 EMAIL_SENDER_TRANSPORT_host = cfg.smtpHost;
27 } // hydraEnv // cfg.extraEnv;
28
29 serverEnv = env //
30 { HYDRA_TRACKER = cfg.tracker;
31 COLUMNS = "80";
32 PGPASSFILE = "${baseDir}/pgpass-www"; # grrr
33 } // (optionalAttrs cfg.debugServer { DBIC_TRACE = "1"; });
34
35 localDB = "dbi:Pg:dbname=hydra;user=hydra;";
36
37 haveLocalDB = cfg.dbi == localDB;
38
39in
40
41{
42 ###### interface
43 options = {
44
45 services.hydra = rec {
46
47 enable = mkOption {
48 type = types.bool;
49 default = false;
50 description = ''
51 Whether to run Hydra services.
52 '';
53 };
54
55 dbi = mkOption {
56 type = types.str;
57 default = localDB;
58 example = "dbi:Pg:dbname=hydra;host=postgres.example.org;user=foo;";
59 description = ''
60 The DBI string for Hydra database connection.
61 '';
62 };
63
64 package = mkOption {
65 type = types.path;
66 default = pkgs.hydra;
67 defaultText = "pkgs.hydra";
68 description = "The Hydra package.";
69 };
70
71 hydraURL = mkOption {
72 type = types.str;
73 description = ''
74 The base URL for the Hydra webserver instance. Used for links in emails.
75 '';
76 };
77
78 listenHost = mkOption {
79 type = types.str;
80 default = "*";
81 example = "localhost";
82 description = ''
83 The hostname or address to listen on or <literal>*</literal> to listen
84 on all interfaces.
85 '';
86 };
87
88 port = mkOption {
89 type = types.int;
90 default = 3000;
91 description = ''
92 TCP port the web server should listen to.
93 '';
94 };
95
96 minimumDiskFree = mkOption {
97 type = types.int;
98 default = 0;
99 description = ''
100 Threshold of minimum disk space (GiB) to determine if the queue runner should run or not.
101 '';
102 };
103
104 minimumDiskFreeEvaluator = mkOption {
105 type = types.int;
106 default = 0;
107 description = ''
108 Threshold of minimum disk space (GiB) to determine if the evaluator should run or not.
109 '';
110 };
111
112 notificationSender = mkOption {
113 type = types.str;
114 description = ''
115 Sender email address used for email notifications.
116 '';
117 };
118
119 smtpHost = mkOption {
120 type = types.nullOr types.str;
121 default = null;
122 example = ["localhost"];
123 description = ''
124 Hostname of the SMTP server to use to send email.
125 '';
126 };
127
128 tracker = mkOption {
129 type = types.str;
130 default = "";
131 description = ''
132 Piece of HTML that is included on all pages.
133 '';
134 };
135
136 logo = mkOption {
137 type = types.nullOr types.path;
138 default = null;
139 description = ''
140 Path to a file containing the logo of your Hydra instance.
141 '';
142 };
143
144 debugServer = mkOption {
145 type = types.bool;
146 default = false;
147 description = "Whether to run the server in debug mode.";
148 };
149
150 extraConfig = mkOption {
151 type = types.lines;
152 description = "Extra lines for the Hydra configuration.";
153 };
154
155 extraEnv = mkOption {
156 type = types.attrsOf types.str;
157 default = {};
158 description = "Extra environment variables for Hydra.";
159 };
160
161 gcRootsDir = mkOption {
162 type = types.path;
163 default = "/nix/var/nix/gcroots/hydra";
164 description = "Directory that holds Hydra garbage collector roots.";
165 };
166
167 buildMachinesFiles = mkOption {
168 type = types.listOf types.path;
169 default = [ "/etc/nix/machines" ];
170 example = [ "/etc/nix/machines" "/var/lib/hydra/provisioner/machines" ];
171 description = "List of files containing build machines.";
172 };
173
174 useSubstitutes = mkOption {
175 type = types.bool;
176 default = false;
177 description = ''
178 Whether to use binary caches for downloading store paths. Note that
179 binary substitutions trigger (a potentially large number of) additional
180 HTTP requests that slow down the queue monitor thread significantly.
181 Also, this Hydra instance will serve those downloaded store paths to
182 its users with its own signature attached as if it had built them
183 itself, so don't enable this feature unless your active binary caches
184 are absolute trustworthy.
185 '';
186 };
187 };
188
189 };
190
191
192 ###### implementation
193
194 config = mkIf cfg.enable {
195
196 users.extraGroups.hydra = {
197 gid = config.ids.gids.hydra;
198 };
199
200 users.extraUsers.hydra =
201 { description = "Hydra";
202 group = "hydra";
203 createHome = true;
204 home = baseDir;
205 useDefaultShell = true;
206 uid = config.ids.uids.hydra;
207 };
208
209 users.extraUsers.hydra-queue-runner =
210 { description = "Hydra queue runner";
211 group = "hydra";
212 useDefaultShell = true;
213 home = "${baseDir}/queue-runner"; # really only to keep SSH happy
214 uid = config.ids.uids.hydra-queue-runner;
215 };
216
217 users.extraUsers.hydra-www =
218 { description = "Hydra web server";
219 group = "hydra";
220 useDefaultShell = true;
221 uid = config.ids.uids.hydra-www;
222 };
223
224 nix.trustedUsers = [ "hydra-queue-runner" ];
225
226 services.hydra.extraConfig =
227 ''
228 using_frontend_proxy 1
229 base_uri ${cfg.hydraURL}
230 notification_sender ${cfg.notificationSender}
231 max_servers 25
232 ${optionalString (cfg.logo != null) ''
233 hydra_logo ${cfg.logo}
234 ''}
235 gc_roots_dir ${cfg.gcRootsDir}
236 use-substitutes = ${if cfg.useSubstitutes then "1" else "0"}
237 '';
238
239 environment.systemPackages = [ cfg.package ];
240
241 environment.variables = hydraEnv;
242
243 nix.extraOptions = ''
244 gc-keep-outputs = true
245 gc-keep-derivations = true
246
247 # The default (`true') slows Nix down a lot since the build farm
248 # has so many GC roots.
249 gc-check-reachability = false
250 '';
251
252 systemd.services.hydra-init =
253 { wantedBy = [ "multi-user.target" ];
254 requires = optional haveLocalDB "postgresql.service";
255 after = optional haveLocalDB "postgresql.service";
256 environment = env;
257 preStart = ''
258 mkdir -p ${baseDir}
259 chown hydra.hydra ${baseDir}
260 chmod 0750 ${baseDir}
261
262 ln -sf ${hydraConf} ${baseDir}/hydra.conf
263
264 mkdir -m 0700 -p ${baseDir}/www
265 chown hydra-www.hydra ${baseDir}/www
266
267 mkdir -m 0700 -p ${baseDir}/queue-runner
268 mkdir -m 0750 -p ${baseDir}/build-logs
269 chown hydra-queue-runner.hydra ${baseDir}/queue-runner ${baseDir}/build-logs
270
271 ${optionalString haveLocalDB ''
272 if ! [ -e ${baseDir}/.db-created ]; then
273 ${config.services.postgresql.package}/bin/createuser hydra
274 ${config.services.postgresql.package}/bin/createdb -O hydra hydra
275 touch ${baseDir}/.db-created
276 fi
277 ''}
278
279 if [ ! -e ${cfg.gcRootsDir} ]; then
280
281 # Move legacy roots directory.
282 if [ -e /nix/var/nix/gcroots/per-user/hydra/hydra-roots ]; then
283 mv /nix/var/nix/gcroots/per-user/hydra/hydra-roots ${cfg.gcRootsDir}
284 fi
285
286 mkdir -p ${cfg.gcRootsDir}
287 fi
288
289 # Move legacy hydra-www roots.
290 if [ -e /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots ]; then
291 find /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots/ -type f \
292 | xargs -r mv -f -t ${cfg.gcRootsDir}/
293 rmdir /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots
294 fi
295
296 chown hydra.hydra ${cfg.gcRootsDir}
297 chmod 2775 ${cfg.gcRootsDir}
298 '';
299 serviceConfig.ExecStart = "${cfg.package}/bin/hydra-init";
300 serviceConfig.PermissionsStartOnly = true;
301 serviceConfig.User = "hydra";
302 serviceConfig.Type = "oneshot";
303 serviceConfig.RemainAfterExit = true;
304 };
305
306 systemd.services.hydra-server =
307 { wantedBy = [ "multi-user.target" ];
308 requires = [ "hydra-init.service" ];
309 after = [ "hydra-init.service" ];
310 environment = serverEnv;
311 restartTriggers = [ hydraConf ];
312 serviceConfig =
313 { ExecStart =
314 "@${cfg.package}/bin/hydra-server hydra-server -f -h '${cfg.listenHost}' "
315 + "-p ${toString cfg.port} --max_spare_servers 5 --max_servers 25 "
316 + "--max_requests 100 ${optionalString cfg.debugServer "-d"}";
317 User = "hydra-www";
318 PermissionsStartOnly = true;
319 Restart = "always";
320 };
321 };
322
323 systemd.services.hydra-queue-runner =
324 { wantedBy = [ "multi-user.target" ];
325 requires = [ "hydra-init.service" ];
326 after = [ "hydra-init.service" "network.target" ];
327 path = [ cfg.package pkgs.nettools pkgs.openssh pkgs.bzip2 config.nix.package ];
328 restartTriggers = [ hydraConf ];
329 environment = env // {
330 PGPASSFILE = "${baseDir}/pgpass-queue-runner"; # grrr
331 IN_SYSTEMD = "1"; # to get log severity levels
332 };
333 serviceConfig =
334 { ExecStart = "@${cfg.package}/bin/hydra-queue-runner hydra-queue-runner -v --option build-use-substitutes ${boolToString cfg.useSubstitutes}";
335 ExecStopPost = "${cfg.package}/bin/hydra-queue-runner --unlock";
336 User = "hydra-queue-runner";
337 Restart = "always";
338
339 # Ensure we can get core dumps.
340 LimitCORE = "infinity";
341 WorkingDirectory = "${baseDir}/queue-runner";
342 };
343 };
344
345 systemd.services.hydra-evaluator =
346 { wantedBy = [ "multi-user.target" ];
347 requires = [ "hydra-init.service" ];
348 after = [ "hydra-init.service" "network.target" ];
349 path = with pkgs; [ cfg.package nettools jq ];
350 restartTriggers = [ hydraConf ];
351 environment = env;
352 serviceConfig =
353 { ExecStart = "@${cfg.package}/bin/hydra-evaluator hydra-evaluator";
354 User = "hydra";
355 Restart = "always";
356 WorkingDirectory = baseDir;
357 };
358 };
359
360 systemd.services.hydra-update-gc-roots =
361 { requires = [ "hydra-init.service" ];
362 after = [ "hydra-init.service" ];
363 environment = env;
364 serviceConfig =
365 { ExecStart = "@${cfg.package}/bin/hydra-update-gc-roots hydra-update-gc-roots";
366 User = "hydra";
367 };
368 startAt = "2,14:15";
369 };
370
371 systemd.services.hydra-send-stats =
372 { wantedBy = [ "multi-user.target" ];
373 after = [ "hydra-init.service" ];
374 environment = env;
375 serviceConfig =
376 { ExecStart = "@${cfg.package}/bin/hydra-send-stats hydra-send-stats";
377 User = "hydra";
378 };
379 };
380
381 # If there is less than a certain amount of free disk space, stop
382 # the queue/evaluator to prevent builds from failing or aborting.
383 systemd.services.hydra-check-space =
384 { script =
385 ''
386 if [ $(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store))) -lt $((${toString cfg.minimumDiskFree} * 1024**3)) ]; then
387 echo "stopping Hydra queue runner due to lack of free space..."
388 systemctl stop hydra-queue-runner
389 fi
390 if [ $(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store))) -lt $((${toString cfg.minimumDiskFreeEvaluator} * 1024**3)) ]; then
391 echo "stopping Hydra evaluator due to lack of free space..."
392 systemctl stop hydra-evaluator
393 fi
394 '';
395 startAt = "*:0/5";
396 };
397
398 # Periodically compress build logs. The queue runner compresses
399 # logs automatically after a step finishes, but this doesn't work
400 # if the queue runner is stopped prematurely.
401 systemd.services.hydra-compress-logs =
402 { path = [ pkgs.bzip2 ];
403 script =
404 ''
405 find /var/lib/hydra/build-logs -type f -name "*.drv" -mtime +3 -size +0c | xargs -r bzip2 -v -f
406 '';
407 startAt = "Sun 01:45";
408 };
409
410 services.postgresql.enable = mkIf haveLocalDB true;
411
412 services.postgresql.identMap = optionalString haveLocalDB
413 ''
414 hydra-users hydra hydra
415 hydra-users hydra-queue-runner hydra
416 hydra-users hydra-www hydra
417 hydra-users root hydra
418 '';
419
420 services.postgresql.authentication = optionalString haveLocalDB
421 ''
422 local hydra all ident map=hydra-users
423 '';
424
425 };
426
427}