pyrox.dev
1{ config, pkgs, lib, ... }:
2
3with lib;
4
5let
6
7 cfg = config.services.hydra;
8
9 baseDir = "/var/lib/hydra";
10
11 hydraConf = pkgs.writeScript "hydra.conf" cfg.extraConfig;
12
13 hydraEnv =
14 { HYDRA_DBI = cfg.dbi;
15 HYDRA_CONFIG = "${baseDir}/hydra.conf";
16 HYDRA_DATA = "${baseDir}";
17 };
18
19 env =
20 { NIX_REMOTE = "daemon";
21 SSL_CERT_FILE = "/etc/ssl/certs/ca-certificates.crt"; # Remove in 16.03
22 PGPASSFILE = "${baseDir}/pgpass";
23 NIX_REMOTE_SYSTEMS = concatStringsSep ":" cfg.buildMachinesFiles;
24 } // optionalAttrs (cfg.smtpHost != null) {
25 EMAIL_SENDER_TRANSPORT = "SMTP";
26 EMAIL_SENDER_TRANSPORT_host = cfg.smtpHost;
27 } // hydraEnv // cfg.extraEnv;
28
29 serverEnv = env //
30 { HYDRA_TRACKER = cfg.tracker;
31 COLUMNS = "80";
32 PGPASSFILE = "${baseDir}/pgpass-www"; # grrr
33 } // (optionalAttrs cfg.debugServer { DBIC_TRACE = "1"; });
34
35 localDB = "dbi:Pg:dbname=hydra;user=hydra;";
36
37 haveLocalDB = cfg.dbi == localDB;
38
39in
40
41{
42 ###### interface
43 options = {
44
45 services.hydra = rec {
46
47 enable = mkOption {
48 type = types.bool;
49 default = false;
50 description = ''
51 Whether to run Hydra services.
52 '';
53 };
54
55 dbi = mkOption {
56 type = types.str;
57 default = localDB;
58 example = "dbi:Pg:dbname=hydra;host=postgres.example.org;user=foo;";
59 description = ''
60 The DBI string for Hydra database connection.
61 '';
62 };
63
64 package = mkOption {
65 type = types.path;
66 default = pkgs.hydra;
67 defaultText = "pkgs.hydra";
68 description = "The Hydra package.";
69 };
70
71 hydraURL = mkOption {
72 type = types.str;
73 description = ''
74 The base URL for the Hydra webserver instance. Used for links in emails.
75 '';
76 };
77
78 listenHost = mkOption {
79 type = types.str;
80 default = "*";
81 example = "localhost";
82 description = ''
83 The hostname or address to listen on or <literal>*</literal> to listen
84 on all interfaces.
85 '';
86 };
87
88 port = mkOption {
89 type = types.int;
90 default = 3000;
91 description = ''
92 TCP port the web server should listen to.
93 '';
94 };
95
96 minimumDiskFree = mkOption {
97 type = types.int;
98 default = 0;
99 description = ''
100 Threshold of minimum disk space (GiB) to determine if the queue runner should run or not.
101 '';
102 };
103
104 minimumDiskFreeEvaluator = mkOption {
105 type = types.int;
106 default = 0;
107 description = ''
108 Threshold of minimum disk space (GiB) to determine if the evaluator should run or not.
109 '';
110 };
111
112 notificationSender = mkOption {
113 type = types.str;
114 description = ''
115 Sender email address used for email notifications.
116 '';
117 };
118
119 smtpHost = mkOption {
120 type = types.nullOr types.str;
121 default = null;
122 example = ["localhost"];
123 description = ''
124 Hostname of the SMTP server to use to send email.
125 '';
126 };
127
128 tracker = mkOption {
129 type = types.str;
130 default = "";
131 description = ''
132 Piece of HTML that is included on all pages.
133 '';
134 };
135
136 logo = mkOption {
137 type = types.nullOr types.path;
138 default = null;
139 description = ''
140 Path to a file containing the logo of your Hydra instance.
141 '';
142 };
143
144 debugServer = mkOption {
145 type = types.bool;
146 default = false;
147 description = "Whether to run the server in debug mode.";
148 };
149
150 extraConfig = mkOption {
151 type = types.lines;
152 description = "Extra lines for the Hydra configuration.";
153 };
154
155 extraEnv = mkOption {
156 type = types.attrsOf types.str;
157 default = {};
158 description = "Extra environment variables for Hydra.";
159 };
160
161 gcRootsDir = mkOption {
162 type = types.path;
163 default = "/nix/var/nix/gcroots/hydra";
164 description = "Directory that holds Hydra garbage collector roots.";
165 };
166
167 buildMachinesFiles = mkOption {
168 type = types.listOf types.path;
169 default = [ "/etc/nix/machines" ];
170 example = [ "/etc/nix/machines" "/var/lib/hydra/provisioner/machines" ];
171 description = "List of files containing build machines.";
172 };
173
174 useSubstitutes = mkOption {
175 type = types.bool;
176 default = false;
177 description = ''
178 Whether to use binary caches for downloading store paths. Note that
179 binary substitutions trigger (a potentially large number of) additional
180 HTTP requests that slow down the queue monitor thread significantly.
181 Also, this Hydra instance will serve those downloaded store paths to
182 its users with its own signature attached as if it had built them
183 itself, so don't enable this feature unless your active binary caches
184 are absolute trustworthy.
185 '';
186 };
187 };
188
189 };
190
191
192 ###### implementation
193
194 config = mkIf cfg.enable {
195
196 users.extraGroups.hydra = {
197 gid = config.ids.gids.hydra;
198 };
199
200 users.extraUsers.hydra =
201 { description = "Hydra";
202 group = "hydra";
203 createHome = true;
204 home = baseDir;
205 useDefaultShell = true;
206 uid = config.ids.uids.hydra;
207 };
208
209 users.extraUsers.hydra-queue-runner =
210 { description = "Hydra queue runner";
211 group = "hydra";
212 useDefaultShell = true;
213 home = "${baseDir}/queue-runner"; # really only to keep SSH happy
214 uid = config.ids.uids.hydra-queue-runner;
215 };
216
217 users.extraUsers.hydra-www =
218 { description = "Hydra web server";
219 group = "hydra";
220 useDefaultShell = true;
221 uid = config.ids.uids.hydra-www;
222 };
223
224 nix.trustedUsers = [ "hydra-queue-runner" ];
225
226 services.hydra.extraConfig =
227 ''
228 using_frontend_proxy 1
229 base_uri ${cfg.hydraURL}
230 notification_sender ${cfg.notificationSender}
231 max_servers 25
232 ${optionalString (cfg.logo != null) ''
233 hydra_logo ${cfg.logo}
234 ''}
235 gc_roots_dir ${cfg.gcRootsDir}
236 '';
237
238 environment.systemPackages = [ cfg.package ];
239
240 environment.variables = hydraEnv;
241
242 nix.extraOptions = ''
243 gc-keep-outputs = true
244 gc-keep-derivations = true
245
246 # The default (`true') slows Nix down a lot since the build farm
247 # has so many GC roots.
248 gc-check-reachability = false
249 '';
250
251 systemd.services.hydra-init =
252 { wantedBy = [ "multi-user.target" ];
253 requires = optional haveLocalDB "postgresql.service";
254 after = optional haveLocalDB "postgresql.service";
255 environment = env;
256 preStart = ''
257 mkdir -p ${baseDir}
258 chown hydra.hydra ${baseDir}
259 chmod 0750 ${baseDir}
260
261 ln -sf ${hydraConf} ${baseDir}/hydra.conf
262
263 mkdir -m 0700 -p ${baseDir}/www
264 chown hydra-www.hydra ${baseDir}/www
265
266 mkdir -m 0700 -p ${baseDir}/queue-runner
267 mkdir -m 0750 -p ${baseDir}/build-logs
268 chown hydra-queue-runner.hydra ${baseDir}/queue-runner ${baseDir}/build-logs
269
270 ${optionalString haveLocalDB ''
271 if ! [ -e ${baseDir}/.db-created ]; then
272 ${config.services.postgresql.package}/bin/createuser hydra
273 ${config.services.postgresql.package}/bin/createdb -O hydra hydra
274 touch ${baseDir}/.db-created
275 fi
276 ''}
277
278 if [ ! -e ${cfg.gcRootsDir} ]; then
279
280 # Move legacy roots directory.
281 if [ -e /nix/var/nix/gcroots/per-user/hydra/hydra-roots ]; then
282 mv /nix/var/nix/gcroots/per-user/hydra/hydra-roots ${cfg.gcRootsDir}
283 fi
284
285 mkdir -p ${cfg.gcRootsDir}
286 fi
287
288 # Move legacy hydra-www roots.
289 if [ -e /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots ]; then
290 find /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots/ -type f \
291 | xargs -r mv -f -t ${cfg.gcRootsDir}/
292 rmdir /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots
293 fi
294
295 chown hydra.hydra ${cfg.gcRootsDir}
296 chmod 2775 ${cfg.gcRootsDir}
297 '';
298 serviceConfig.ExecStart = "${cfg.package}/bin/hydra-init";
299 serviceConfig.PermissionsStartOnly = true;
300 serviceConfig.User = "hydra";
301 serviceConfig.Type = "oneshot";
302 serviceConfig.RemainAfterExit = true;
303 };
304
305 systemd.services.hydra-server =
306 { wantedBy = [ "multi-user.target" ];
307 requires = [ "hydra-init.service" ];
308 after = [ "hydra-init.service" ];
309 environment = serverEnv;
310 serviceConfig =
311 { ExecStart =
312 "@${cfg.package}/bin/hydra-server hydra-server -f -h '${cfg.listenHost}' "
313 + "-p ${toString cfg.port} --max_spare_servers 5 --max_servers 25 "
314 + "--max_requests 100 ${optionalString cfg.debugServer "-d"}";
315 User = "hydra-www";
316 PermissionsStartOnly = true;
317 Restart = "always";
318 };
319 };
320
321 systemd.services.hydra-queue-runner =
322 { wantedBy = [ "multi-user.target" ];
323 requires = [ "hydra-init.service" ];
324 after = [ "hydra-init.service" "network.target" ];
325 path = [ cfg.package pkgs.nettools pkgs.openssh pkgs.bzip2 config.nix.package ];
326 environment = env // {
327 PGPASSFILE = "${baseDir}/pgpass-queue-runner"; # grrr
328 IN_SYSTEMD = "1"; # to get log severity levels
329 };
330 serviceConfig =
331 { ExecStart = "@${cfg.package}/bin/hydra-queue-runner hydra-queue-runner -v --option build-use-substitutes ${if cfg.useSubstitutes then "true" else "false"}";
332 ExecStopPost = "${cfg.package}/bin/hydra-queue-runner --unlock";
333 User = "hydra-queue-runner";
334 Restart = "always";
335
336 # Ensure we can get core dumps.
337 LimitCORE = "infinity";
338 WorkingDirectory = "${baseDir}/queue-runner";
339 };
340 };
341
342 systemd.services.hydra-evaluator =
343 { wantedBy = [ "multi-user.target" ];
344 requires = [ "hydra-init.service" ];
345 after = [ "hydra-init.service" "network.target" ];
346 path = [ pkgs.nettools ];
347 environment = env;
348 serviceConfig =
349 { ExecStart = "@${cfg.package}/bin/hydra-evaluator hydra-evaluator";
350 User = "hydra";
351 Restart = "always";
352 WorkingDirectory = baseDir;
353 };
354 };
355
356 systemd.services.hydra-update-gc-roots =
357 { requires = [ "hydra-init.service" ];
358 after = [ "hydra-init.service" ];
359 environment = env;
360 serviceConfig =
361 { ExecStart = "@${cfg.package}/bin/hydra-update-gc-roots hydra-update-gc-roots";
362 User = "hydra";
363 };
364 startAt = "2,14:15";
365 };
366
367 systemd.services.hydra-send-stats =
368 { wantedBy = [ "multi-user.target" ];
369 after = [ "hydra-init.service" ];
370 environment = env;
371 serviceConfig =
372 { ExecStart = "@${cfg.package}/bin/hydra-send-stats hydra-send-stats";
373 User = "hydra";
374 };
375 };
376
377 # If there is less than a certain amount of free disk space, stop
378 # the queue/evaluator to prevent builds from failing or aborting.
379 systemd.services.hydra-check-space =
380 { script =
381 ''
382 if [ $(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store))) -lt $((${toString cfg.minimumDiskFree} * 1024**3)) ]; then
383 echo "stopping Hydra queue runner due to lack of free space..."
384 systemctl stop hydra-queue-runner
385 fi
386 if [ $(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store))) -lt $((${toString cfg.minimumDiskFreeEvaluator} * 1024**3)) ]; then
387 echo "stopping Hydra evaluator due to lack of free space..."
388 systemctl stop hydra-evaluator
389 fi
390 '';
391 startAt = "*:0/5";
392 };
393
394 # Periodically compress build logs. The queue runner compresses
395 # logs automatically after a step finishes, but this doesn't work
396 # if the queue runner is stopped prematurely.
397 systemd.services.hydra-compress-logs =
398 { path = [ pkgs.bzip2 ];
399 script =
400 ''
401 find /var/lib/hydra/build-logs -type f -name "*.drv" -mtime +3 -size +0c | xargs -r bzip2 -v -f
402 '';
403 startAt = "Sun 01:45";
404 };
405
406 services.postgresql.enable = mkIf haveLocalDB true;
407
408 services.postgresql.identMap = optionalString haveLocalDB
409 ''
410 hydra-users hydra hydra
411 hydra-users hydra-queue-runner hydra
412 hydra-users hydra-www hydra
413 hydra-users root hydra
414 '';
415
416 services.postgresql.authentication = optionalString haveLocalDB
417 ''
418 local hydra all ident map=hydra-users
419 '';
420
421 };
422
423}