pyrox.dev
1{ config, lib, pkgs, ... }:
2with builtins;
3with lib;
4let
5 cfg = config.services.gitlab-runner;
6 hasDocker = config.virtualisation.docker.enable;
7
8 /* The whole logic of this module is to diff the hashes of the desired vs existing runners
9 The hash is recorded in the runner's name because we can't do better yet
10 See https://gitlab.com/gitlab-org/gitlab-runner/-/issues/29350 for more details
11 */
12 genRunnerName = name: service: let
13 hash = substring 0 12 (hashString "md5" (unsafeDiscardStringContext (toJSON service)));
14 in if service ? description && service.description != null
15 then "${hash} ${service.description}"
16 else "${name}_${config.networking.hostName}_${hash}";
17
18 hashedServices = mapAttrs'
19 (name: service: nameValuePair (genRunnerName name service) service) cfg.services;
20 configPath = ''"$HOME"/.gitlab-runner/config.toml'';
21 configureScript = pkgs.writeShellApplication {
22 name = "gitlab-runner-configure";
23 runtimeInputs = with pkgs; [
24 bash
25 gawk
26 jq
27 moreutils
28 remarshal
29 util-linux
30 cfg.package
31 perl
32 python3
33 ];
34 text = if (cfg.configFile != null) then ''
35 cp ${cfg.configFile} ${configPath}
36 # make config file readable by service
37 chown -R --reference="$HOME" "$(dirname ${configPath})"
38 '' else ''
39 export CONFIG_FILE=${configPath}
40
41 mkdir -p "$(dirname ${configPath})"
42 touch ${configPath}
43
44 # update global options
45 remarshal --if toml --of json ${configPath} \
46 | jq -cM 'with_entries(select([.key] | inside(["runners"])))' \
47 | jq -scM '.[0] + .[1]' - <(echo ${escapeShellArg (toJSON cfg.settings)}) \
48 | remarshal --if json --of toml \
49 | sponge ${configPath}
50
51 # remove no longer existing services
52 gitlab-runner verify --delete
53
54 ${toShellVar "NEEDED_SERVICES" (lib.mapAttrs (name: value: 1) hashedServices)}
55
56 declare -A REGISTERED_SERVICES
57
58 while IFS="," read -r name token;
59 do
60 REGISTERED_SERVICES["$name"]="$token"
61 done < <(gitlab-runner --log-format json list 2>&1 | grep Token | jq -r '.msg +"," + .Token')
62
63 echo "NEEDED_SERVICES: " "''${!NEEDED_SERVICES[@]}"
64 echo "REGISTERED_SERVICES:" "''${!REGISTERED_SERVICES[@]}"
65
66 # difference between current and desired state
67 declare -A NEW_SERVICES
68 for name in "''${!NEEDED_SERVICES[@]}"; do
69 if [ ! -v 'REGISTERED_SERVICES[$name]' ]; then
70 NEW_SERVICES[$name]=1
71 fi
72 done
73
74 declare -A OLD_SERVICES
75 # shellcheck disable=SC2034
76 for name in "''${!REGISTERED_SERVICES[@]}"; do
77 if [ ! -v 'NEEDED_SERVICES[$name]' ]; then
78 OLD_SERVICES[$name]=1
79 fi
80 done
81
82 # register new services
83 ${concatStringsSep "\n" (mapAttrsToList (name: service: ''
84 # TODO so here we should mention NEW_SERVICES
85 if [ -v 'NEW_SERVICES["${name}"]' ] ; then
86 bash -c ${escapeShellArg (concatStringsSep " \\\n " ([
87 "set -a && source ${service.registrationConfigFile} &&"
88 "gitlab-runner register"
89 "--non-interactive"
90 "--name '${name}'"
91 "--executor ${service.executor}"
92 "--limit ${toString service.limit}"
93 "--request-concurrency ${toString service.requestConcurrency}"
94 "--maximum-timeout ${toString service.maximumTimeout}"
95 ] ++ service.registrationFlags
96 ++ optional (service.buildsDir != null)
97 "--builds-dir ${service.buildsDir}"
98 ++ optional (service.cloneUrl != null)
99 "--clone-url ${service.cloneUrl}"
100 ++ optional (service.preCloneScript != null)
101 "--pre-clone-script ${service.preCloneScript}"
102 ++ optional (service.preBuildScript != null)
103 "--pre-build-script ${service.preBuildScript}"
104 ++ optional (service.postBuildScript != null)
105 "--post-build-script ${service.postBuildScript}"
106 ++ optional (service.tagList != [ ])
107 "--tag-list ${concatStringsSep "," service.tagList}"
108 ++ optional service.runUntagged
109 "--run-untagged"
110 ++ optional service.protected
111 "--access-level ref_protected"
112 ++ optional service.debugTraceDisabled
113 "--debug-trace-disabled"
114 ++ map (e: "--env ${escapeShellArg e}") (mapAttrsToList (name: value: "${name}=${value}") service.environmentVariables)
115 ++ optionals (hasPrefix "docker" service.executor) (
116 assert (
117 assertMsg (service.dockerImage != null)
118 "dockerImage option is required for ${service.executor} executor (${name})");
119 [ "--docker-image ${service.dockerImage}" ]
120 ++ optional service.dockerDisableCache
121 "--docker-disable-cache"
122 ++ optional service.dockerPrivileged
123 "--docker-privileged"
124 ++ map (v: "--docker-volumes ${escapeShellArg v}") service.dockerVolumes
125 ++ map (v: "--docker-extra-hosts ${escapeShellArg v}") service.dockerExtraHosts
126 ++ map (v: "--docker-allowed-images ${escapeShellArg v}") service.dockerAllowedImages
127 ++ map (v: "--docker-allowed-services ${escapeShellArg v}") service.dockerAllowedServices
128 )
129 ))} && sleep 1 || exit 1
130 fi
131 '') hashedServices)}
132
133 # check key is in array https://stackoverflow.com/questions/30353951/how-to-check-if-dictionary-contains-a-key-in-bash
134
135 echo "NEW_SERVICES: ''${NEW_SERVICES[*]}"
136 echo "OLD_SERVICES: ''${OLD_SERVICES[*]}"
137 # unregister old services
138 for NAME in "''${!OLD_SERVICES[@]}"
139 do
140 [ -n "$NAME" ] && gitlab-runner unregister \
141 --name "$NAME" && sleep 1
142 done
143
144 # make config file readable by service
145 chown -R --reference="$HOME" "$(dirname ${configPath})"
146 '';
147 };
148 startScript = pkgs.writeShellScriptBin "gitlab-runner-start" ''
149 export CONFIG_FILE=${configPath}
150 exec gitlab-runner run --working-directory $HOME
151 '';
152in {
153 options.services.gitlab-runner = {
154 enable = mkEnableOption (lib.mdDoc "Gitlab Runner");
155 configFile = mkOption {
156 type = types.nullOr types.path;
157 default = null;
158 description = lib.mdDoc ''
159 Configuration file for gitlab-runner.
160
161 {option}`configFile` takes precedence over {option}`services`.
162 {option}`checkInterval` and {option}`concurrent` will be ignored too.
163
164 This option is deprecated, please use {option}`services` instead.
165 You can use {option}`registrationConfigFile` and
166 {option}`registrationFlags`
167 for settings not covered by this module.
168 '';
169 };
170 settings = mkOption {
171 type = types.submodule {
172 freeformType = (pkgs.formats.json { }).type;
173 };
174 default = { };
175 description = lib.mdDoc ''
176 Global gitlab-runner configuration. See
177 <https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-global-section>
178 for supported values.
179 '';
180 };
181 gracefulTermination = mkOption {
182 type = types.bool;
183 default = false;
184 description = lib.mdDoc ''
185 Finish all remaining jobs before stopping.
186 If not set gitlab-runner will stop immediately without waiting
187 for jobs to finish, which will lead to failed builds.
188 '';
189 };
190 gracefulTimeout = mkOption {
191 type = types.str;
192 default = "infinity";
193 example = "5min 20s";
194 description = lib.mdDoc ''
195 Time to wait until a graceful shutdown is turned into a forceful one.
196 '';
197 };
198 package = mkOption {
199 type = types.package;
200 default = pkgs.gitlab-runner;
201 defaultText = literalExpression "pkgs.gitlab-runner";
202 example = literalExpression "pkgs.gitlab-runner_1_11";
203 description = lib.mdDoc "Gitlab Runner package to use.";
204 };
205 extraPackages = mkOption {
206 type = types.listOf types.package;
207 default = [ ];
208 description = lib.mdDoc ''
209 Extra packages to add to PATH for the gitlab-runner process.
210 '';
211 };
212 services = mkOption {
213 description = lib.mdDoc "GitLab Runner services.";
214 default = { };
215 example = literalExpression ''
216 {
217 # runner for building in docker via host's nix-daemon
218 # nix store will be readable in runner, might be insecure
219 nix = {
220 # File should contain at least these two variables:
221 # `CI_SERVER_URL`
222 # `REGISTRATION_TOKEN`
223 registrationConfigFile = "/run/secrets/gitlab-runner-registration";
224 dockerImage = "alpine";
225 dockerVolumes = [
226 "/nix/store:/nix/store:ro"
227 "/nix/var/nix/db:/nix/var/nix/db:ro"
228 "/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket:ro"
229 ];
230 dockerDisableCache = true;
231 preBuildScript = pkgs.writeScript "setup-container" '''
232 mkdir -p -m 0755 /nix/var/log/nix/drvs
233 mkdir -p -m 0755 /nix/var/nix/gcroots
234 mkdir -p -m 0755 /nix/var/nix/profiles
235 mkdir -p -m 0755 /nix/var/nix/temproots
236 mkdir -p -m 0755 /nix/var/nix/userpool
237 mkdir -p -m 1777 /nix/var/nix/gcroots/per-user
238 mkdir -p -m 1777 /nix/var/nix/profiles/per-user
239 mkdir -p -m 0755 /nix/var/nix/profiles/per-user/root
240 mkdir -p -m 0700 "$HOME/.nix-defexpr"
241
242 . ''${pkgs.nix}/etc/profile.d/nix.sh
243
244 ''${pkgs.nix}/bin/nix-env -i ''${concatStringsSep " " (with pkgs; [ nix cacert git openssh ])}
245
246 ''${pkgs.nix}/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable
247 ''${pkgs.nix}/bin/nix-channel --update nixpkgs
248 ''';
249 environmentVariables = {
250 ENV = "/etc/profile";
251 USER = "root";
252 NIX_REMOTE = "daemon";
253 PATH = "/nix/var/nix/profiles/default/bin:/nix/var/nix/profiles/default/sbin:/bin:/sbin:/usr/bin:/usr/sbin";
254 NIX_SSL_CERT_FILE = "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt";
255 };
256 tagList = [ "nix" ];
257 };
258 # runner for building docker images
259 docker-images = {
260 # File should contain at least these two variables:
261 # `CI_SERVER_URL`
262 # `REGISTRATION_TOKEN`
263 registrationConfigFile = "/run/secrets/gitlab-runner-registration";
264 dockerImage = "docker:stable";
265 dockerVolumes = [
266 "/var/run/docker.sock:/var/run/docker.sock"
267 ];
268 tagList = [ "docker-images" ];
269 };
270 # runner for executing stuff on host system (very insecure!)
271 # make sure to add required packages (including git!)
272 # to `environment.systemPackages`
273 shell = {
274 # File should contain at least these two variables:
275 # `CI_SERVER_URL`
276 # `REGISTRATION_TOKEN`
277 registrationConfigFile = "/run/secrets/gitlab-runner-registration";
278 executor = "shell";
279 tagList = [ "shell" ];
280 };
281 # runner for everything else
282 default = {
283 # File should contain at least these two variables:
284 # `CI_SERVER_URL`
285 # `REGISTRATION_TOKEN`
286 registrationConfigFile = "/run/secrets/gitlab-runner-registration";
287 dockerImage = "debian:stable";
288 };
289 }
290 '';
291 type = types.attrsOf (types.submodule {
292 options = {
293 registrationConfigFile = mkOption {
294 type = types.path;
295 description = lib.mdDoc ''
296 Absolute path to a file with environment variables
297 used for gitlab-runner registration.
298 A list of all supported environment variables can be found in
299 `gitlab-runner register --help`.
300
301 Ones that you probably want to set is
302
303 `CI_SERVER_URL=<CI server URL>`
304
305 `REGISTRATION_TOKEN=<registration secret>`
306
307 WARNING: make sure to use quoted absolute path,
308 or it is going to be copied to Nix Store.
309 '';
310 };
311 registrationFlags = mkOption {
312 type = types.listOf types.str;
313 default = [ ];
314 example = [ "--docker-helper-image my/gitlab-runner-helper" ];
315 description = lib.mdDoc ''
316 Extra command-line flags passed to
317 `gitlab-runner register`.
318 Execute `gitlab-runner register --help`
319 for a list of supported flags.
320 '';
321 };
322 environmentVariables = mkOption {
323 type = types.attrsOf types.str;
324 default = { };
325 example = { NAME = "value"; };
326 description = lib.mdDoc ''
327 Custom environment variables injected to build environment.
328 For secrets you can use {option}`registrationConfigFile`
329 with `RUNNER_ENV` variable set.
330 '';
331 };
332 description = mkOption {
333 type = types.nullOr types.str;
334 default = null;
335 description = lib.mdDoc ''
336 Name/description of the runner.
337 '';
338 };
339 executor = mkOption {
340 type = types.str;
341 default = "docker";
342 description = lib.mdDoc ''
343 Select executor, eg. shell, docker, etc.
344 See [runner documentation](https://docs.gitlab.com/runner/executors/README.html) for more information.
345 '';
346 };
347 buildsDir = mkOption {
348 type = types.nullOr types.path;
349 default = null;
350 example = "/var/lib/gitlab-runner/builds";
351 description = lib.mdDoc ''
352 Absolute path to a directory where builds will be stored
353 in context of selected executor (Locally, Docker, SSH).
354 '';
355 };
356 cloneUrl = mkOption {
357 type = types.nullOr types.str;
358 default = null;
359 example = "http://gitlab.example.local";
360 description = lib.mdDoc ''
361 Overwrite the URL for the GitLab instance. Used if the Runner can’t connect to GitLab on the URL GitLab exposes itself.
362 '';
363 };
364 dockerImage = mkOption {
365 type = types.nullOr types.str;
366 default = null;
367 description = lib.mdDoc ''
368 Docker image to be used.
369 '';
370 };
371 dockerVolumes = mkOption {
372 type = types.listOf types.str;
373 default = [ ];
374 example = [ "/var/run/docker.sock:/var/run/docker.sock" ];
375 description = lib.mdDoc ''
376 Bind-mount a volume and create it
377 if it doesn't exist prior to mounting.
378 '';
379 };
380 dockerDisableCache = mkOption {
381 type = types.bool;
382 default = false;
383 description = lib.mdDoc ''
384 Disable all container caching.
385 '';
386 };
387 dockerPrivileged = mkOption {
388 type = types.bool;
389 default = false;
390 description = lib.mdDoc ''
391 Give extended privileges to container.
392 '';
393 };
394 dockerExtraHosts = mkOption {
395 type = types.listOf types.str;
396 default = [ ];
397 example = [ "other-host:127.0.0.1" ];
398 description = lib.mdDoc ''
399 Add a custom host-to-IP mapping.
400 '';
401 };
402 dockerAllowedImages = mkOption {
403 type = types.listOf types.str;
404 default = [ ];
405 example = [ "ruby:*" "python:*" "php:*" "my.registry.tld:5000/*:*" ];
406 description = lib.mdDoc ''
407 Whitelist allowed images.
408 '';
409 };
410 dockerAllowedServices = mkOption {
411 type = types.listOf types.str;
412 default = [ ];
413 example = [ "postgres:9" "redis:*" "mysql:*" ];
414 description = lib.mdDoc ''
415 Whitelist allowed services.
416 '';
417 };
418 preCloneScript = mkOption {
419 type = types.nullOr types.path;
420 default = null;
421 description = lib.mdDoc ''
422 Runner-specific command script executed before code is pulled.
423 '';
424 };
425 preBuildScript = mkOption {
426 type = types.nullOr types.path;
427 default = null;
428 description = lib.mdDoc ''
429 Runner-specific command script executed after code is pulled,
430 just before build executes.
431 '';
432 };
433 postBuildScript = mkOption {
434 type = types.nullOr types.path;
435 default = null;
436 description = lib.mdDoc ''
437 Runner-specific command script executed after code is pulled
438 and just after build executes.
439 '';
440 };
441 tagList = mkOption {
442 type = types.listOf types.str;
443 default = [ ];
444 description = lib.mdDoc ''
445 Tag list.
446 '';
447 };
448 runUntagged = mkOption {
449 type = types.bool;
450 default = false;
451 description = lib.mdDoc ''
452 Register to run untagged builds; defaults to
453 `true` when {option}`tagList` is empty.
454 '';
455 };
456 limit = mkOption {
457 type = types.int;
458 default = 0;
459 description = lib.mdDoc ''
460 Limit how many jobs can be handled concurrently by this service.
461 0 (default) simply means don't limit.
462 '';
463 };
464 requestConcurrency = mkOption {
465 type = types.int;
466 default = 0;
467 description = lib.mdDoc ''
468 Limit number of concurrent requests for new jobs from GitLab.
469 '';
470 };
471 maximumTimeout = mkOption {
472 type = types.int;
473 default = 0;
474 description = lib.mdDoc ''
475 What is the maximum timeout (in seconds) that will be set for
476 job when using this Runner. 0 (default) simply means don't limit.
477 '';
478 };
479 protected = mkOption {
480 type = types.bool;
481 default = false;
482 description = lib.mdDoc ''
483 When set to true Runner will only run on pipelines
484 triggered on protected branches.
485 '';
486 };
487 debugTraceDisabled = mkOption {
488 type = types.bool;
489 default = false;
490 description = lib.mdDoc ''
491 When set to true Runner will disable the possibility of
492 using the `CI_DEBUG_TRACE` feature.
493 '';
494 };
495 };
496 });
497 };
498 clear-docker-cache = {
499 enable = mkOption {
500 type = types.bool;
501 default = false;
502 description = lib.mdDoc ''
503 Whether to periodically prune gitlab runner's Docker resources. If
504 enabled, a systemd timer will run {command}`clear-docker-cache` as
505 specified by the `dates` option.
506 '';
507 };
508
509 flags = mkOption {
510 type = types.listOf types.str;
511 default = [ ];
512 example = [ "prune" ];
513 description = lib.mdDoc ''
514 Any additional flags passed to {command}`clear-docker-cache`.
515 '';
516 };
517
518 dates = mkOption {
519 default = "weekly";
520 type = types.str;
521 description = lib.mdDoc ''
522 Specification (in the format described by
523 {manpage}`systemd.time(7)`) of the time at
524 which the prune will occur.
525 '';
526 };
527
528 package = mkOption {
529 default = config.virtualisation.docker.package;
530 defaultText = literalExpression "config.virtualisation.docker.package";
531 example = literalExpression "pkgs.docker";
532 description = lib.mdDoc "Docker package to use for clearing up docker cache.";
533 };
534 };
535 };
536 config = mkIf cfg.enable {
537 warnings = mapAttrsToList
538 (n: v: "services.gitlab-runner.services.${n}.`registrationConfigFile` points to a file in Nix Store. You should use quoted absolute path to prevent this.")
539 (filterAttrs (n: v: isStorePath v.registrationConfigFile) cfg.services);
540
541 environment.systemPackages = [ cfg.package ];
542 systemd.services.gitlab-runner = {
543 description = "Gitlab Runner";
544 documentation = [ "https://docs.gitlab.com/runner/" ];
545 after = [ "network.target" ]
546 ++ optional hasDocker "docker.service";
547 requires = optional hasDocker "docker.service";
548 wantedBy = [ "multi-user.target" ];
549 environment = config.networking.proxy.envVars // {
550 HOME = "/var/lib/gitlab-runner";
551 };
552 path = with pkgs; [
553 bash
554 gawk
555 jq
556 moreutils
557 remarshal
558 util-linux
559 cfg.package
560 ] ++ cfg.extraPackages;
561 reloadIfChanged = true;
562 serviceConfig = {
563 # Set `DynamicUser` under `systemd.services.gitlab-runner.serviceConfig`
564 # to `lib.mkForce false` in your configuration to run this service as root.
565 # You can also set `User` and `Group` options to run this service as desired user.
566 # Make sure to restart service or changes won't apply.
567 DynamicUser = true;
568 StateDirectory = "gitlab-runner";
569 SupplementaryGroups = optional hasDocker "docker";
570 ExecStartPre = "!${configureScript}/bin/gitlab-runner-configure";
571 ExecStart = "${startScript}/bin/gitlab-runner-start";
572 ExecReload = "!${configureScript}/bin/gitlab-runner-configure";
573 } // optionalAttrs cfg.gracefulTermination {
574 TimeoutStopSec = "${cfg.gracefulTimeout}";
575 KillSignal = "SIGQUIT";
576 KillMode = "process";
577 };
578 };
579 # Enable periodic clear-docker-cache script
580 systemd.services.gitlab-runner-clear-docker-cache = mkIf (cfg.clear-docker-cache.enable && (any (s: s.executor == "docker") (attrValues cfg.services))) {
581 description = "Prune gitlab-runner docker resources";
582 restartIfChanged = false;
583 unitConfig.X-StopOnRemoval = false;
584
585 serviceConfig.Type = "oneshot";
586
587 path = [ cfg.clear-docker-cache.package pkgs.gawk ];
588
589 script = ''
590 ${pkgs.gitlab-runner}/bin/clear-docker-cache ${toString cfg.clear-docker-cache.flags}
591 '';
592
593 startAt = cfg.clear-docker-cache.dates;
594 };
595 # Enable docker if `docker` executor is used in any service
596 virtualisation.docker.enable = mkIf (
597 any (s: s.executor == "docker") (attrValues cfg.services)
598 ) (mkDefault true);
599 };
600 imports = [
601 (mkRenamedOptionModule [ "services" "gitlab-runner" "packages" ] [ "services" "gitlab-runner" "extraPackages" ] )
602 (mkRemovedOptionModule [ "services" "gitlab-runner" "configOptions" ] "Use services.gitlab-runner.services option instead" )
603 (mkRemovedOptionModule [ "services" "gitlab-runner" "workDir" ] "You should move contents of workDir (if any) to /var/lib/gitlab-runner" )
604
605 (mkRenamedOptionModule [ "services" "gitlab-runner" "checkInterval" ] [ "services" "gitlab-runner" "settings" "check_interval" ] )
606 (mkRenamedOptionModule [ "services" "gitlab-runner" "concurrent" ] [ "services" "gitlab-runner" "settings" "concurrent" ] )
607 (mkRenamedOptionModule [ "services" "gitlab-runner" "sentryDSN" ] [ "services" "gitlab-runner" "settings" "sentry_dsn" ] )
608 (mkRenamedOptionModule [ "services" "gitlab-runner" "prometheusListenAddress" ] [ "services" "gitlab-runner" "settings" "listen_address" ] )
609
610 (mkRenamedOptionModule [ "services" "gitlab-runner" "sessionServer" "listenAddress" ] [ "services" "gitlab-runner" "settings" "session_server" "listen_address" ] )
611 (mkRenamedOptionModule [ "services" "gitlab-runner" "sessionServer" "advertiseAddress" ] [ "services" "gitlab-runner" "settings" "session_server" "advertise_address" ] )
612 (mkRenamedOptionModule [ "services" "gitlab-runner" "sessionServer" "sessionTimeout" ] [ "services" "gitlab-runner" "settings" "session_server" "session_timeout" ] )
613 ];
614}