pyrox.dev
1 { config, lib, options, pkgs, ... }:
2
3with lib;
4
5let
6 top = config.services.kubernetes;
7 otop = options.services.kubernetes;
8 cfg = top.apiserver;
9
10 isRBACEnabled = elem "RBAC" cfg.authorizationMode;
11
12 apiserverServiceIP = (concatStringsSep "." (
13 take 3 (splitString "." cfg.serviceClusterIpRange
14 )) + ".1");
15in
16{
17
18 imports = [
19 (mkRenamedOptionModule [ "services" "kubernetes" "apiserver" "admissionControl" ] [ "services" "kubernetes" "apiserver" "enableAdmissionPlugins" ])
20 (mkRenamedOptionModule [ "services" "kubernetes" "apiserver" "address" ] ["services" "kubernetes" "apiserver" "bindAddress"])
21 (mkRemovedOptionModule [ "services" "kubernetes" "apiserver" "insecureBindAddress" ] "")
22 (mkRemovedOptionModule [ "services" "kubernetes" "apiserver" "insecurePort" ] "")
23 (mkRemovedOptionModule [ "services" "kubernetes" "apiserver" "publicAddress" ] "")
24 (mkRenamedOptionModule [ "services" "kubernetes" "etcd" "servers" ] [ "services" "kubernetes" "apiserver" "etcd" "servers" ])
25 (mkRenamedOptionModule [ "services" "kubernetes" "etcd" "keyFile" ] [ "services" "kubernetes" "apiserver" "etcd" "keyFile" ])
26 (mkRenamedOptionModule [ "services" "kubernetes" "etcd" "certFile" ] [ "services" "kubernetes" "apiserver" "etcd" "certFile" ])
27 (mkRenamedOptionModule [ "services" "kubernetes" "etcd" "caFile" ] [ "services" "kubernetes" "apiserver" "etcd" "caFile" ])
28 ];
29
30 ###### interface
31 options.services.kubernetes.apiserver = with lib.types; {
32
33 advertiseAddress = mkOption {
34 description = lib.mdDoc ''
35 Kubernetes apiserver IP address on which to advertise the apiserver
36 to members of the cluster. This address must be reachable by the rest
37 of the cluster.
38 '';
39 default = null;
40 type = nullOr str;
41 };
42
43 allowPrivileged = mkOption {
44 description = lib.mdDoc "Whether to allow privileged containers on Kubernetes.";
45 default = false;
46 type = bool;
47 };
48
49 authorizationMode = mkOption {
50 description = lib.mdDoc ''
51 Kubernetes apiserver authorization mode (AlwaysAllow/AlwaysDeny/ABAC/Webhook/RBAC/Node). See
52 <https://kubernetes.io/docs/reference/access-authn-authz/authorization/>
53 '';
54 default = ["RBAC" "Node"]; # Enabling RBAC by default, although kubernetes default is AllowAllow
55 type = listOf (enum ["AlwaysAllow" "AlwaysDeny" "ABAC" "Webhook" "RBAC" "Node"]);
56 };
57
58 authorizationPolicy = mkOption {
59 description = lib.mdDoc ''
60 Kubernetes apiserver authorization policy file. See
61 <https://kubernetes.io/docs/reference/access-authn-authz/authorization/>
62 '';
63 default = [];
64 type = listOf attrs;
65 };
66
67 basicAuthFile = mkOption {
68 description = lib.mdDoc ''
69 Kubernetes apiserver basic authentication file. See
70 <https://kubernetes.io/docs/reference/access-authn-authz/authentication>
71 '';
72 default = null;
73 type = nullOr path;
74 };
75
76 bindAddress = mkOption {
77 description = lib.mdDoc ''
78 The IP address on which to listen for the --secure-port port.
79 The associated interface(s) must be reachable by the rest
80 of the cluster, and by CLI/web clients.
81 '';
82 default = "0.0.0.0";
83 type = str;
84 };
85
86 clientCaFile = mkOption {
87 description = lib.mdDoc "Kubernetes apiserver CA file for client auth.";
88 default = top.caFile;
89 defaultText = literalExpression "config.${otop.caFile}";
90 type = nullOr path;
91 };
92
93 disableAdmissionPlugins = mkOption {
94 description = lib.mdDoc ''
95 Kubernetes admission control plugins to disable. See
96 <https://kubernetes.io/docs/admin/admission-controllers/>
97 '';
98 default = [];
99 type = listOf str;
100 };
101
102 enable = mkEnableOption (lib.mdDoc "Kubernetes apiserver");
103
104 enableAdmissionPlugins = mkOption {
105 description = lib.mdDoc ''
106 Kubernetes admission control plugins to enable. See
107 <https://kubernetes.io/docs/admin/admission-controllers/>
108 '';
109 default = [
110 "NamespaceLifecycle" "LimitRanger" "ServiceAccount"
111 "ResourceQuota" "DefaultStorageClass" "DefaultTolerationSeconds"
112 "NodeRestriction"
113 ];
114 example = [
115 "NamespaceLifecycle" "NamespaceExists" "LimitRanger"
116 "SecurityContextDeny" "ServiceAccount" "ResourceQuota"
117 "PodSecurityPolicy" "NodeRestriction" "DefaultStorageClass"
118 ];
119 type = listOf str;
120 };
121
122 etcd = {
123 servers = mkOption {
124 description = lib.mdDoc "List of etcd servers.";
125 default = ["http://127.0.0.1:2379"];
126 type = types.listOf types.str;
127 };
128
129 keyFile = mkOption {
130 description = lib.mdDoc "Etcd key file.";
131 default = null;
132 type = types.nullOr types.path;
133 };
134
135 certFile = mkOption {
136 description = lib.mdDoc "Etcd cert file.";
137 default = null;
138 type = types.nullOr types.path;
139 };
140
141 caFile = mkOption {
142 description = lib.mdDoc "Etcd ca file.";
143 default = top.caFile;
144 defaultText = literalExpression "config.${otop.caFile}";
145 type = types.nullOr types.path;
146 };
147 };
148
149 extraOpts = mkOption {
150 description = lib.mdDoc "Kubernetes apiserver extra command line options.";
151 default = "";
152 type = separatedString " ";
153 };
154
155 extraSANs = mkOption {
156 description = lib.mdDoc "Extra x509 Subject Alternative Names to be added to the kubernetes apiserver tls cert.";
157 default = [];
158 type = listOf str;
159 };
160
161 featureGates = mkOption {
162 description = lib.mdDoc "List set of feature gates";
163 default = top.featureGates;
164 defaultText = literalExpression "config.${otop.featureGates}";
165 type = listOf str;
166 };
167
168 kubeletClientCaFile = mkOption {
169 description = lib.mdDoc "Path to a cert file for connecting to kubelet.";
170 default = top.caFile;
171 defaultText = literalExpression "config.${otop.caFile}";
172 type = nullOr path;
173 };
174
175 kubeletClientCertFile = mkOption {
176 description = lib.mdDoc "Client certificate to use for connections to kubelet.";
177 default = null;
178 type = nullOr path;
179 };
180
181 kubeletClientKeyFile = mkOption {
182 description = lib.mdDoc "Key to use for connections to kubelet.";
183 default = null;
184 type = nullOr path;
185 };
186
187 preferredAddressTypes = mkOption {
188 description = lib.mdDoc "List of the preferred NodeAddressTypes to use for kubelet connections.";
189 type = nullOr str;
190 default = null;
191 };
192
193 proxyClientCertFile = mkOption {
194 description = lib.mdDoc "Client certificate to use for connections to proxy.";
195 default = null;
196 type = nullOr path;
197 };
198
199 proxyClientKeyFile = mkOption {
200 description = lib.mdDoc "Key to use for connections to proxy.";
201 default = null;
202 type = nullOr path;
203 };
204
205 runtimeConfig = mkOption {
206 description = lib.mdDoc ''
207 Api runtime configuration. See
208 <https://kubernetes.io/docs/tasks/administer-cluster/cluster-management/>
209 '';
210 default = "authentication.k8s.io/v1beta1=true";
211 example = "api/all=false,api/v1=true";
212 type = str;
213 };
214
215 storageBackend = mkOption {
216 description = lib.mdDoc ''
217 Kubernetes apiserver storage backend.
218 '';
219 default = "etcd3";
220 type = enum ["etcd2" "etcd3"];
221 };
222
223 securePort = mkOption {
224 description = lib.mdDoc "Kubernetes apiserver secure port.";
225 default = 6443;
226 type = int;
227 };
228
229 apiAudiences = mkOption {
230 description = lib.mdDoc ''
231 Kubernetes apiserver ServiceAccount issuer.
232 '';
233 default = "api,https://kubernetes.default.svc";
234 type = str;
235 };
236
237 serviceAccountIssuer = mkOption {
238 description = lib.mdDoc ''
239 Kubernetes apiserver ServiceAccount issuer.
240 '';
241 default = "https://kubernetes.default.svc";
242 type = str;
243 };
244
245 serviceAccountSigningKeyFile = mkOption {
246 description = lib.mdDoc ''
247 Path to the file that contains the current private key of the service
248 account token issuer. The issuer will sign issued ID tokens with this
249 private key.
250 '';
251 type = path;
252 };
253
254 serviceAccountKeyFile = mkOption {
255 description = lib.mdDoc ''
256 File containing PEM-encoded x509 RSA or ECDSA private or public keys,
257 used to verify ServiceAccount tokens. The specified file can contain
258 multiple keys, and the flag can be specified multiple times with
259 different files. If unspecified, --tls-private-key-file is used.
260 Must be specified when --service-account-signing-key is provided
261 '';
262 type = path;
263 };
264
265 serviceClusterIpRange = mkOption {
266 description = lib.mdDoc ''
267 A CIDR notation IP range from which to assign service cluster IPs.
268 This must not overlap with any IP ranges assigned to nodes for pods.
269 '';
270 default = "10.0.0.0/24";
271 type = str;
272 };
273
274 tlsCertFile = mkOption {
275 description = lib.mdDoc "Kubernetes apiserver certificate file.";
276 default = null;
277 type = nullOr path;
278 };
279
280 tlsKeyFile = mkOption {
281 description = lib.mdDoc "Kubernetes apiserver private key file.";
282 default = null;
283 type = nullOr path;
284 };
285
286 tokenAuthFile = mkOption {
287 description = lib.mdDoc ''
288 Kubernetes apiserver token authentication file. See
289 <https://kubernetes.io/docs/reference/access-authn-authz/authentication>
290 '';
291 default = null;
292 type = nullOr path;
293 };
294
295 verbosity = mkOption {
296 description = lib.mdDoc ''
297 Optional glog verbosity level for logging statements. See
298 <https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md>
299 '';
300 default = null;
301 type = nullOr int;
302 };
303
304 webhookConfig = mkOption {
305 description = lib.mdDoc ''
306 Kubernetes apiserver Webhook config file. It uses the kubeconfig file format.
307 See <https://kubernetes.io/docs/reference/access-authn-authz/webhook/>
308 '';
309 default = null;
310 type = nullOr path;
311 };
312
313 };
314
315
316 ###### implementation
317 config = mkMerge [
318
319 (mkIf cfg.enable {
320 systemd.services.kube-apiserver = {
321 description = "Kubernetes APIServer Service";
322 wantedBy = [ "kubernetes.target" ];
323 after = [ "network.target" ];
324 serviceConfig = {
325 Slice = "kubernetes.slice";
326 ExecStart = ''${top.package}/bin/kube-apiserver \
327 --allow-privileged=${boolToString cfg.allowPrivileged} \
328 --authorization-mode=${concatStringsSep "," cfg.authorizationMode} \
329 ${optionalString (elem "ABAC" cfg.authorizationMode)
330 "--authorization-policy-file=${
331 pkgs.writeText "kube-auth-policy.jsonl"
332 (concatMapStringsSep "\n" (l: builtins.toJSON l) cfg.authorizationPolicy)
333 }"
334 } \
335 ${optionalString (elem "Webhook" cfg.authorizationMode)
336 "--authorization-webhook-config-file=${cfg.webhookConfig}"
337 } \
338 --bind-address=${cfg.bindAddress} \
339 ${optionalString (cfg.advertiseAddress != null)
340 "--advertise-address=${cfg.advertiseAddress}"} \
341 ${optionalString (cfg.clientCaFile != null)
342 "--client-ca-file=${cfg.clientCaFile}"} \
343 --disable-admission-plugins=${concatStringsSep "," cfg.disableAdmissionPlugins} \
344 --enable-admission-plugins=${concatStringsSep "," cfg.enableAdmissionPlugins} \
345 --etcd-servers=${concatStringsSep "," cfg.etcd.servers} \
346 ${optionalString (cfg.etcd.caFile != null)
347 "--etcd-cafile=${cfg.etcd.caFile}"} \
348 ${optionalString (cfg.etcd.certFile != null)
349 "--etcd-certfile=${cfg.etcd.certFile}"} \
350 ${optionalString (cfg.etcd.keyFile != null)
351 "--etcd-keyfile=${cfg.etcd.keyFile}"} \
352 ${optionalString (cfg.featureGates != [])
353 "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \
354 ${optionalString (cfg.basicAuthFile != null)
355 "--basic-auth-file=${cfg.basicAuthFile}"} \
356 ${optionalString (cfg.kubeletClientCaFile != null)
357 "--kubelet-certificate-authority=${cfg.kubeletClientCaFile}"} \
358 ${optionalString (cfg.kubeletClientCertFile != null)
359 "--kubelet-client-certificate=${cfg.kubeletClientCertFile}"} \
360 ${optionalString (cfg.kubeletClientKeyFile != null)
361 "--kubelet-client-key=${cfg.kubeletClientKeyFile}"} \
362 ${optionalString (cfg.preferredAddressTypes != null)
363 "--kubelet-preferred-address-types=${cfg.preferredAddressTypes}"} \
364 ${optionalString (cfg.proxyClientCertFile != null)
365 "--proxy-client-cert-file=${cfg.proxyClientCertFile}"} \
366 ${optionalString (cfg.proxyClientKeyFile != null)
367 "--proxy-client-key-file=${cfg.proxyClientKeyFile}"} \
368 ${optionalString (cfg.runtimeConfig != "")
369 "--runtime-config=${cfg.runtimeConfig}"} \
370 --secure-port=${toString cfg.securePort} \
371 --api-audiences=${toString cfg.apiAudiences} \
372 --service-account-issuer=${toString cfg.serviceAccountIssuer} \
373 --service-account-signing-key-file=${cfg.serviceAccountSigningKeyFile} \
374 --service-account-key-file=${cfg.serviceAccountKeyFile} \
375 --service-cluster-ip-range=${cfg.serviceClusterIpRange} \
376 --storage-backend=${cfg.storageBackend} \
377 ${optionalString (cfg.tlsCertFile != null)
378 "--tls-cert-file=${cfg.tlsCertFile}"} \
379 ${optionalString (cfg.tlsKeyFile != null)
380 "--tls-private-key-file=${cfg.tlsKeyFile}"} \
381 ${optionalString (cfg.tokenAuthFile != null)
382 "--token-auth-file=${cfg.tokenAuthFile}"} \
383 ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
384 ${cfg.extraOpts}
385 '';
386 WorkingDirectory = top.dataDir;
387 User = "kubernetes";
388 Group = "kubernetes";
389 AmbientCapabilities = "cap_net_bind_service";
390 Restart = "on-failure";
391 RestartSec = 5;
392 };
393
394 unitConfig = {
395 StartLimitIntervalSec = 0;
396 };
397 };
398
399 services.etcd = {
400 clientCertAuth = mkDefault true;
401 peerClientCertAuth = mkDefault true;
402 listenClientUrls = mkDefault ["https://0.0.0.0:2379"];
403 listenPeerUrls = mkDefault ["https://0.0.0.0:2380"];
404 advertiseClientUrls = mkDefault ["https://${top.masterAddress}:2379"];
405 initialCluster = mkDefault ["${top.masterAddress}=https://${top.masterAddress}:2380"];
406 name = mkDefault top.masterAddress;
407 initialAdvertisePeerUrls = mkDefault ["https://${top.masterAddress}:2380"];
408 };
409
410 services.kubernetes.addonManager.bootstrapAddons = mkIf isRBACEnabled {
411
412 apiserver-kubelet-api-admin-crb = {
413 apiVersion = "rbac.authorization.k8s.io/v1";
414 kind = "ClusterRoleBinding";
415 metadata = {
416 name = "system:kube-apiserver:kubelet-api-admin";
417 };
418 roleRef = {
419 apiGroup = "rbac.authorization.k8s.io";
420 kind = "ClusterRole";
421 name = "system:kubelet-api-admin";
422 };
423 subjects = [{
424 kind = "User";
425 name = "system:kube-apiserver";
426 }];
427 };
428
429 };
430
431 services.kubernetes.pki.certs = with top.lib; {
432 apiServer = mkCert {
433 name = "kube-apiserver";
434 CN = "kubernetes";
435 hosts = [
436 "kubernetes.default.svc"
437 "kubernetes.default.svc.${top.addons.dns.clusterDomain}"
438 cfg.advertiseAddress
439 top.masterAddress
440 apiserverServiceIP
441 "127.0.0.1"
442 ] ++ cfg.extraSANs;
443 action = "systemctl restart kube-apiserver.service";
444 };
445 apiserverProxyClient = mkCert {
446 name = "kube-apiserver-proxy-client";
447 CN = "front-proxy-client";
448 action = "systemctl restart kube-apiserver.service";
449 };
450 apiserverKubeletClient = mkCert {
451 name = "kube-apiserver-kubelet-client";
452 CN = "system:kube-apiserver";
453 action = "systemctl restart kube-apiserver.service";
454 };
455 apiserverEtcdClient = mkCert {
456 name = "kube-apiserver-etcd-client";
457 CN = "etcd-client";
458 action = "systemctl restart kube-apiserver.service";
459 };
460 clusterAdmin = mkCert {
461 name = "cluster-admin";
462 CN = "cluster-admin";
463 fields = {
464 O = "system:masters";
465 };
466 privateKeyOwner = "root";
467 };
468 etcd = mkCert {
469 name = "etcd";
470 CN = top.masterAddress;
471 hosts = [
472 "etcd.local"
473 "etcd.${top.addons.dns.clusterDomain}"
474 top.masterAddress
475 cfg.advertiseAddress
476 ];
477 privateKeyOwner = "etcd";
478 action = "systemctl restart etcd.service";
479 };
480 };
481
482 })
483
484 ];
485
486 meta.buildDocsInSandbox = false;
487}