pyrox.dev
1{ config, pkgs, lib, ... }:
2
3with lib;
4
5let
6
7 cfg = config.services.hydra;
8
9 baseDir = "/var/lib/hydra";
10
11 hydraConf = pkgs.writeScript "hydra.conf" cfg.extraConfig;
12
13 hydraEnv =
14 { HYDRA_DBI = cfg.dbi;
15 HYDRA_CONFIG = "${baseDir}/hydra.conf";
16 HYDRA_DATA = "${baseDir}";
17 };
18
19 env =
20 { NIX_REMOTE = "daemon";
21 SSL_CERT_FILE = "/etc/ssl/certs/ca-certificates.crt"; # Remove in 16.03
22 PGPASSFILE = "${baseDir}/pgpass";
23 NIX_REMOTE_SYSTEMS = concatStringsSep ":" cfg.buildMachinesFiles;
24 } // optionalAttrs (cfg.smtpHost != null) {
25 EMAIL_SENDER_TRANSPORT = "SMTP";
26 EMAIL_SENDER_TRANSPORT_host = cfg.smtpHost;
27 } // hydraEnv // cfg.extraEnv;
28
29 serverEnv = env //
30 { HYDRA_TRACKER = cfg.tracker;
31 XDG_CACHE_HOME = "${baseDir}/www/.cache";
32 COLUMNS = "80";
33 PGPASSFILE = "${baseDir}/pgpass-www"; # grrr
34 } // (optionalAttrs cfg.debugServer { DBIC_TRACE = "1"; });
35
36 localDB = "dbi:Pg:dbname=hydra;user=hydra;";
37
38 haveLocalDB = cfg.dbi == localDB;
39
40in
41
42{
43 ###### interface
44 options = {
45
46 services.hydra = rec {
47
48 enable = mkOption {
49 type = types.bool;
50 default = false;
51 description = ''
52 Whether to run Hydra services.
53 '';
54 };
55
56 dbi = mkOption {
57 type = types.str;
58 default = localDB;
59 example = "dbi:Pg:dbname=hydra;host=postgres.example.org;user=foo;";
60 description = ''
61 The DBI string for Hydra database connection.
62 '';
63 };
64
65 package = mkOption {
66 type = types.path;
67 default = pkgs.hydra;
68 defaultText = "pkgs.hydra";
69 description = "The Hydra package.";
70 };
71
72 hydraURL = mkOption {
73 type = types.str;
74 description = ''
75 The base URL for the Hydra webserver instance. Used for links in emails.
76 '';
77 };
78
79 listenHost = mkOption {
80 type = types.str;
81 default = "*";
82 example = "localhost";
83 description = ''
84 The hostname or address to listen on or <literal>*</literal> to listen
85 on all interfaces.
86 '';
87 };
88
89 port = mkOption {
90 type = types.int;
91 default = 3000;
92 description = ''
93 TCP port the web server should listen to.
94 '';
95 };
96
97 minimumDiskFree = mkOption {
98 type = types.int;
99 default = 0;
100 description = ''
101 Threshold of minimum disk space (GiB) to determine if the queue runner should run or not.
102 '';
103 };
104
105 minimumDiskFreeEvaluator = mkOption {
106 type = types.int;
107 default = 0;
108 description = ''
109 Threshold of minimum disk space (GiB) to determine if the evaluator should run or not.
110 '';
111 };
112
113 notificationSender = mkOption {
114 type = types.str;
115 description = ''
116 Sender email address used for email notifications.
117 '';
118 };
119
120 smtpHost = mkOption {
121 type = types.nullOr types.str;
122 default = null;
123 example = ["localhost"];
124 description = ''
125 Hostname of the SMTP server to use to send email.
126 '';
127 };
128
129 tracker = mkOption {
130 type = types.str;
131 default = "";
132 description = ''
133 Piece of HTML that is included on all pages.
134 '';
135 };
136
137 logo = mkOption {
138 type = types.nullOr types.path;
139 default = null;
140 description = ''
141 Path to a file containing the logo of your Hydra instance.
142 '';
143 };
144
145 debugServer = mkOption {
146 type = types.bool;
147 default = false;
148 description = "Whether to run the server in debug mode.";
149 };
150
151 extraConfig = mkOption {
152 type = types.lines;
153 description = "Extra lines for the Hydra configuration.";
154 };
155
156 extraEnv = mkOption {
157 type = types.attrsOf types.str;
158 default = {};
159 description = "Extra environment variables for Hydra.";
160 };
161
162 gcRootsDir = mkOption {
163 type = types.path;
164 default = "/nix/var/nix/gcroots/hydra";
165 description = "Directory that holds Hydra garbage collector roots.";
166 };
167
168 buildMachinesFiles = mkOption {
169 type = types.listOf types.path;
170 default = [ "/etc/nix/machines" ];
171 example = [ "/etc/nix/machines" "/var/lib/hydra/provisioner/machines" ];
172 description = "List of files containing build machines.";
173 };
174
175 useSubstitutes = mkOption {
176 type = types.bool;
177 default = false;
178 description = ''
179 Whether to use binary caches for downloading store paths. Note that
180 binary substitutions trigger (a potentially large number of) additional
181 HTTP requests that slow down the queue monitor thread significantly.
182 Also, this Hydra instance will serve those downloaded store paths to
183 its users with its own signature attached as if it had built them
184 itself, so don't enable this feature unless your active binary caches
185 are absolute trustworthy.
186 '';
187 };
188 };
189
190 };
191
192
193 ###### implementation
194
195 config = mkIf cfg.enable {
196
197 users.groups.hydra = {
198 gid = config.ids.gids.hydra;
199 };
200
201 users.users.hydra =
202 { description = "Hydra";
203 group = "hydra";
204 createHome = true;
205 home = baseDir;
206 useDefaultShell = true;
207 uid = config.ids.uids.hydra;
208 };
209
210 users.users.hydra-queue-runner =
211 { description = "Hydra queue runner";
212 group = "hydra";
213 useDefaultShell = true;
214 home = "${baseDir}/queue-runner"; # really only to keep SSH happy
215 uid = config.ids.uids.hydra-queue-runner;
216 };
217
218 users.users.hydra-www =
219 { description = "Hydra web server";
220 group = "hydra";
221 useDefaultShell = true;
222 uid = config.ids.uids.hydra-www;
223 };
224
225 nix.trustedUsers = [ "hydra-queue-runner" ];
226
227 services.hydra.extraConfig =
228 ''
229 using_frontend_proxy = 1
230 base_uri = ${cfg.hydraURL}
231 notification_sender = ${cfg.notificationSender}
232 max_servers = 25
233 ${optionalString (cfg.logo != null) ''
234 hydra_logo = ${cfg.logo}
235 ''}
236 gc_roots_dir = ${cfg.gcRootsDir}
237 use-substitutes = ${if cfg.useSubstitutes then "1" else "0"}
238 '';
239
240 environment.systemPackages = [ cfg.package ];
241
242 environment.variables = hydraEnv;
243
244 nix.extraOptions = ''
245 gc-keep-outputs = true
246 gc-keep-derivations = true
247
248 # The default (`true') slows Nix down a lot since the build farm
249 # has so many GC roots.
250 gc-check-reachability = false
251 '';
252
253 systemd.services.hydra-init =
254 { wantedBy = [ "multi-user.target" ];
255 requires = optional haveLocalDB "postgresql.service";
256 after = optional haveLocalDB "postgresql.service";
257 environment = env;
258 preStart = ''
259 mkdir -p ${baseDir}
260 chown hydra.hydra ${baseDir}
261 chmod 0750 ${baseDir}
262
263 ln -sf ${hydraConf} ${baseDir}/hydra.conf
264
265 mkdir -m 0700 -p ${baseDir}/www
266 chown hydra-www.hydra ${baseDir}/www
267
268 mkdir -m 0700 -p ${baseDir}/queue-runner
269 mkdir -m 0750 -p ${baseDir}/build-logs
270 chown hydra-queue-runner.hydra ${baseDir}/queue-runner ${baseDir}/build-logs
271
272 ${optionalString haveLocalDB ''
273 if ! [ -e ${baseDir}/.db-created ]; then
274 ${pkgs.sudo}/bin/sudo -u ${config.services.postgresql.superUser} ${config.services.postgresql.package}/bin/createuser hydra
275 ${pkgs.sudo}/bin/sudo -u ${config.services.postgresql.superUser} ${config.services.postgresql.package}/bin/createdb -O hydra hydra
276 touch ${baseDir}/.db-created
277 fi
278 ''}
279
280 if [ ! -e ${cfg.gcRootsDir} ]; then
281
282 # Move legacy roots directory.
283 if [ -e /nix/var/nix/gcroots/per-user/hydra/hydra-roots ]; then
284 mv /nix/var/nix/gcroots/per-user/hydra/hydra-roots ${cfg.gcRootsDir}
285 fi
286
287 mkdir -p ${cfg.gcRootsDir}
288 fi
289
290 # Move legacy hydra-www roots.
291 if [ -e /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots ]; then
292 find /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots/ -type f \
293 | xargs -r mv -f -t ${cfg.gcRootsDir}/
294 rmdir /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots
295 fi
296
297 chown hydra.hydra ${cfg.gcRootsDir}
298 chmod 2775 ${cfg.gcRootsDir}
299 '';
300 serviceConfig.ExecStart = "${cfg.package}/bin/hydra-init";
301 serviceConfig.PermissionsStartOnly = true;
302 serviceConfig.User = "hydra";
303 serviceConfig.Type = "oneshot";
304 serviceConfig.RemainAfterExit = true;
305 };
306
307 systemd.services.hydra-server =
308 { wantedBy = [ "multi-user.target" ];
309 requires = [ "hydra-init.service" ];
310 after = [ "hydra-init.service" ];
311 environment = serverEnv;
312 restartTriggers = [ hydraConf ];
313 serviceConfig =
314 { ExecStart =
315 "@${cfg.package}/bin/hydra-server hydra-server -f -h '${cfg.listenHost}' "
316 + "-p ${toString cfg.port} --max_spare_servers 5 --max_servers 25 "
317 + "--max_requests 100 ${optionalString cfg.debugServer "-d"}";
318 User = "hydra-www";
319 PermissionsStartOnly = true;
320 Restart = "always";
321 };
322 };
323
324 systemd.services.hydra-queue-runner =
325 { wantedBy = [ "multi-user.target" ];
326 requires = [ "hydra-init.service" ];
327 after = [ "hydra-init.service" "network.target" ];
328 path = [ cfg.package pkgs.nettools pkgs.openssh pkgs.bzip2 config.nix.package ];
329 restartTriggers = [ hydraConf ];
330 environment = env // {
331 PGPASSFILE = "${baseDir}/pgpass-queue-runner"; # grrr
332 IN_SYSTEMD = "1"; # to get log severity levels
333 };
334 serviceConfig =
335 { ExecStart = "@${cfg.package}/bin/hydra-queue-runner hydra-queue-runner -v --option build-use-substitutes ${boolToString cfg.useSubstitutes}";
336 ExecStopPost = "${cfg.package}/bin/hydra-queue-runner --unlock";
337 User = "hydra-queue-runner";
338 Restart = "always";
339
340 # Ensure we can get core dumps.
341 LimitCORE = "infinity";
342 WorkingDirectory = "${baseDir}/queue-runner";
343 };
344 };
345
346 systemd.services.hydra-evaluator =
347 { wantedBy = [ "multi-user.target" ];
348 requires = [ "hydra-init.service" ];
349 after = [ "hydra-init.service" "network.target" ];
350 path = with pkgs; [ cfg.package nettools jq ];
351 restartTriggers = [ hydraConf ];
352 environment = env;
353 serviceConfig =
354 { ExecStart = "@${cfg.package}/bin/hydra-evaluator hydra-evaluator";
355 User = "hydra";
356 Restart = "always";
357 WorkingDirectory = baseDir;
358 };
359 };
360
361 systemd.services.hydra-update-gc-roots =
362 { requires = [ "hydra-init.service" ];
363 after = [ "hydra-init.service" ];
364 environment = env;
365 serviceConfig =
366 { ExecStart = "@${cfg.package}/bin/hydra-update-gc-roots hydra-update-gc-roots";
367 User = "hydra";
368 };
369 startAt = "2,14:15";
370 };
371
372 systemd.services.hydra-send-stats =
373 { wantedBy = [ "multi-user.target" ];
374 after = [ "hydra-init.service" ];
375 environment = env;
376 serviceConfig =
377 { ExecStart = "@${cfg.package}/bin/hydra-send-stats hydra-send-stats";
378 User = "hydra";
379 };
380 };
381
382 # If there is less than a certain amount of free disk space, stop
383 # the queue/evaluator to prevent builds from failing or aborting.
384 systemd.services.hydra-check-space =
385 { script =
386 ''
387 if [ $(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store))) -lt $((${toString cfg.minimumDiskFree} * 1024**3)) ]; then
388 echo "stopping Hydra queue runner due to lack of free space..."
389 systemctl stop hydra-queue-runner
390 fi
391 if [ $(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store))) -lt $((${toString cfg.minimumDiskFreeEvaluator} * 1024**3)) ]; then
392 echo "stopping Hydra evaluator due to lack of free space..."
393 systemctl stop hydra-evaluator
394 fi
395 '';
396 startAt = "*:0/5";
397 };
398
399 # Periodically compress build logs. The queue runner compresses
400 # logs automatically after a step finishes, but this doesn't work
401 # if the queue runner is stopped prematurely.
402 systemd.services.hydra-compress-logs =
403 { path = [ pkgs.bzip2 ];
404 script =
405 ''
406 find /var/lib/hydra/build-logs -type f -name "*.drv" -mtime +3 -size +0c | xargs -r bzip2 -v -f
407 '';
408 startAt = "Sun 01:45";
409 };
410
411 services.postgresql.enable = mkIf haveLocalDB true;
412
413 services.postgresql.identMap = optionalString haveLocalDB
414 ''
415 hydra-users hydra hydra
416 hydra-users hydra-queue-runner hydra
417 hydra-users hydra-www hydra
418 hydra-users root hydra
419 '';
420
421 services.postgresql.authentication = optionalString haveLocalDB
422 ''
423 local hydra all ident map=hydra-users
424 '';
425
426 };
427
428}