pyrox.dev
1{ config, lib, pkgs, ... }:
2with lib;
3let
4 cfg = config.services.nomad;
5 format = pkgs.formats.json { };
6in
7{
8 ##### interface
9 options = {
10 services.nomad = {
11 enable = mkEnableOption "Nomad, a distributed, highly available, datacenter-aware scheduler";
12
13 package = mkPackageOption pkgs "nomad" { };
14
15 extraPackages = mkOption {
16 type = types.listOf types.package;
17 default = [ ];
18 description = ''
19 Extra packages to add to {env}`PATH` for the Nomad agent process.
20 '';
21 example = literalExpression ''
22 with pkgs; [ cni-plugins ]
23 '';
24 };
25
26 dropPrivileges = mkOption {
27 type = types.bool;
28 default = true;
29 description = ''
30 Whether the nomad agent should be run as a non-root nomad user.
31 '';
32 };
33
34 enableDocker = mkOption {
35 type = types.bool;
36 default = true;
37 description = ''
38 Enable Docker support. Needed for Nomad's docker driver.
39
40 Note that the docker group membership is effectively equivalent
41 to being root, see https://github.com/moby/moby/issues/9976.
42 '';
43 };
44
45 extraSettingsPaths = mkOption {
46 type = types.listOf types.path;
47 default = [ ];
48 description = ''
49 Additional settings paths used to configure nomad. These can be files or directories.
50 '';
51 example = literalExpression ''
52 [ "/etc/nomad-mutable.json" "/run/keys/nomad-with-secrets.json" "/etc/nomad/config.d" ]
53 '';
54 };
55
56 extraSettingsPlugins = mkOption {
57 type = types.listOf (types.either types.package types.path);
58 default = [ ];
59 description = ''
60 Additional plugins dir used to configure nomad.
61 '';
62 example = literalExpression ''
63 [ "<pluginDir>" pkgs.nomad-driver-nix pkgs.nomad-driver-podman ]
64 '';
65 };
66
67 credentials = mkOption {
68 description = ''
69 Credentials envs used to configure nomad secrets.
70 '';
71 type = types.attrsOf types.str;
72 default = { };
73
74 example = {
75 logs_remote_write_password = "/run/keys/nomad_write_password";
76 };
77 };
78
79 settings = mkOption {
80 type = format.type;
81 default = { };
82 description = ''
83 Configuration for Nomad. See the [documentation](https://www.nomadproject.io/docs/configuration)
84 for supported values.
85
86 Notes about `data_dir`:
87
88 If `data_dir` is set to a value other than the
89 default value of `"/var/lib/nomad"` it is the Nomad
90 cluster manager's responsibility to make sure that this directory
91 exists and has the appropriate permissions.
92
93 Additionally, if `dropPrivileges` is
94 `true` then `data_dir`
95 *cannot* be customized. Setting
96 `dropPrivileges` to `true` enables
97 the `DynamicUser` feature of systemd which directly
98 manages and operates on `StateDirectory`.
99 '';
100 example = literalExpression ''
101 {
102 # A minimal config example:
103 server = {
104 enabled = true;
105 bootstrap_expect = 1; # for demo; no fault tolerance
106 };
107 client = {
108 enabled = true;
109 };
110 }
111 '';
112 };
113 };
114 };
115
116 ##### implementation
117 config = mkIf cfg.enable {
118 services.nomad.settings = {
119 # Agrees with `StateDirectory = "nomad"` set below.
120 data_dir = mkDefault "/var/lib/nomad";
121 };
122
123 environment = {
124 etc."nomad.json".source = format.generate "nomad.json" cfg.settings;
125 systemPackages = [ cfg.package ];
126 };
127
128 systemd.services.nomad = {
129 description = "Nomad";
130 wantedBy = [ "multi-user.target" ];
131 wants = [ "network-online.target" ];
132 after = [ "network-online.target" ];
133 restartTriggers = [ config.environment.etc."nomad.json".source ];
134
135 path = cfg.extraPackages ++ (with pkgs; [
136 # Client mode requires at least the following:
137 coreutils
138 iproute2
139 iptables
140 ]);
141
142 serviceConfig = mkMerge [
143 {
144 DynamicUser = cfg.dropPrivileges;
145 ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
146 ExecStart =
147 let
148 pluginsDir = pkgs.symlinkJoin
149 {
150 name = "nomad-plugins";
151 paths = cfg.extraSettingsPlugins;
152 };
153 in
154 "${cfg.package}/bin/nomad agent -config=/etc/nomad.json -plugin-dir=${pluginsDir}/bin" +
155 concatMapStrings (path: " -config=${path}") cfg.extraSettingsPaths +
156 concatMapStrings (key: " -config=\${CREDENTIALS_DIRECTORY}/${key}") (lib.attrNames cfg.credentials);
157 KillMode = "process";
158 KillSignal = "SIGINT";
159 LimitNOFILE = 65536;
160 LimitNPROC = "infinity";
161 OOMScoreAdjust = -1000;
162 Restart = "on-failure";
163 RestartSec = 2;
164 TasksMax = "infinity";
165 LoadCredential = lib.mapAttrsToList (key: value: "${key}:${value}") cfg.credentials;
166 }
167 (mkIf cfg.enableDocker {
168 SupplementaryGroups = "docker"; # space-separated string
169 })
170 (mkIf (cfg.settings.data_dir == "/var/lib/nomad") {
171 StateDirectory = "nomad";
172 })
173 ];
174
175 unitConfig = {
176 StartLimitIntervalSec = 10;
177 StartLimitBurst = 3;
178 };
179 };
180
181 assertions = [
182 {
183 assertion = cfg.dropPrivileges -> cfg.settings.data_dir == "/var/lib/nomad";
184 message = "settings.data_dir must be equal to \"/var/lib/nomad\" if dropPrivileges is true";
185 }
186 ];
187
188 # Docker support requires the Docker daemon to be running.
189 virtualisation.docker.enable = mkIf cfg.enableDocker true;
190 };
191}