pyrox.dev
1{
2 config,
3 lib,
4 options,
5 pkgs,
6 ...
7}:
8let
9 top = config.services.kubernetes;
10 otop = options.services.kubernetes;
11 cfg = top.apiserver;
12
13 isRBACEnabled = lib.elem "RBAC" cfg.authorizationMode;
14
15 apiserverServiceIP = (
16 lib.concatStringsSep "." (lib.take 3 (lib.splitString "." cfg.serviceClusterIpRange)) + ".1"
17 );
18in
19{
20
21 imports = [
22 (lib.mkRenamedOptionModule
23 [ "services" "kubernetes" "apiserver" "admissionControl" ]
24 [ "services" "kubernetes" "apiserver" "enableAdmissionPlugins" ]
25 )
26 (lib.mkRenamedOptionModule
27 [ "services" "kubernetes" "apiserver" "address" ]
28 [ "services" "kubernetes" "apiserver" "bindAddress" ]
29 )
30 (lib.mkRemovedOptionModule [ "services" "kubernetes" "apiserver" "insecureBindAddress" ] "")
31 (lib.mkRemovedOptionModule [ "services" "kubernetes" "apiserver" "insecurePort" ] "")
32 (lib.mkRemovedOptionModule [ "services" "kubernetes" "apiserver" "publicAddress" ] "")
33 (lib.mkRenamedOptionModule
34 [ "services" "kubernetes" "etcd" "servers" ]
35 [ "services" "kubernetes" "apiserver" "etcd" "servers" ]
36 )
37 (lib.mkRenamedOptionModule
38 [ "services" "kubernetes" "etcd" "keyFile" ]
39 [ "services" "kubernetes" "apiserver" "etcd" "keyFile" ]
40 )
41 (lib.mkRenamedOptionModule
42 [ "services" "kubernetes" "etcd" "certFile" ]
43 [ "services" "kubernetes" "apiserver" "etcd" "certFile" ]
44 )
45 (lib.mkRenamedOptionModule
46 [ "services" "kubernetes" "etcd" "caFile" ]
47 [ "services" "kubernetes" "apiserver" "etcd" "caFile" ]
48 )
49 ];
50
51 ###### interface
52 options.services.kubernetes.apiserver = with lib.types; {
53
54 advertiseAddress = lib.mkOption {
55 description = ''
56 Kubernetes apiserver IP address on which to advertise the apiserver
57 to members of the cluster. This address must be reachable by the rest
58 of the cluster.
59 '';
60 default = null;
61 type = nullOr str;
62 };
63
64 allowPrivileged = lib.mkOption {
65 description = "Whether to allow privileged containers on Kubernetes.";
66 default = false;
67 type = bool;
68 };
69
70 authorizationMode = lib.mkOption {
71 description = ''
72 Kubernetes apiserver authorization mode (AlwaysAllow/AlwaysDeny/ABAC/Webhook/RBAC/Node). See
73 <https://kubernetes.io/docs/reference/access-authn-authz/authorization/>
74 '';
75 default = [
76 "RBAC"
77 "Node"
78 ]; # Enabling RBAC by default, although kubernetes default is AllowAllow
79 type = listOf (enum [
80 "AlwaysAllow"
81 "AlwaysDeny"
82 "ABAC"
83 "Webhook"
84 "RBAC"
85 "Node"
86 ]);
87 };
88
89 authorizationPolicy = lib.mkOption {
90 description = ''
91 Kubernetes apiserver authorization policy file. See
92 <https://kubernetes.io/docs/reference/access-authn-authz/authorization/>
93 '';
94 default = [ ];
95 type = listOf attrs;
96 };
97
98 basicAuthFile = lib.mkOption {
99 description = ''
100 Kubernetes apiserver basic authentication file. See
101 <https://kubernetes.io/docs/reference/access-authn-authz/authentication>
102 '';
103 default = null;
104 type = nullOr path;
105 };
106
107 bindAddress = lib.mkOption {
108 description = ''
109 The IP address on which to listen for the --secure-port port.
110 The associated interface(s) must be reachable by the rest
111 of the cluster, and by CLI/web clients.
112 '';
113 default = "0.0.0.0";
114 type = str;
115 };
116
117 clientCaFile = lib.mkOption {
118 description = "Kubernetes apiserver CA file for client auth.";
119 default = top.caFile;
120 defaultText = lib.literalExpression "config.${otop.caFile}";
121 type = nullOr path;
122 };
123
124 disableAdmissionPlugins = lib.mkOption {
125 description = ''
126 Kubernetes admission control plugins to disable. See
127 <https://kubernetes.io/docs/admin/admission-controllers/>
128 '';
129 default = [ ];
130 type = listOf str;
131 };
132
133 enable = lib.mkEnableOption "Kubernetes apiserver";
134
135 enableAdmissionPlugins = lib.mkOption {
136 description = ''
137 Kubernetes admission control plugins to enable. See
138 <https://kubernetes.io/docs/admin/admission-controllers/>
139 '';
140 default = [
141 "NamespaceLifecycle"
142 "LimitRanger"
143 "ServiceAccount"
144 "ResourceQuota"
145 "DefaultStorageClass"
146 "DefaultTolerationSeconds"
147 "NodeRestriction"
148 ];
149 example = [
150 "NamespaceLifecycle"
151 "NamespaceExists"
152 "LimitRanger"
153 "SecurityContextDeny"
154 "ServiceAccount"
155 "ResourceQuota"
156 "PodSecurityPolicy"
157 "NodeRestriction"
158 "DefaultStorageClass"
159 ];
160 type = listOf str;
161 };
162
163 etcd = {
164 servers = lib.mkOption {
165 description = "List of etcd servers.";
166 default = [ "http://127.0.0.1:2379" ];
167 type = types.listOf types.str;
168 };
169
170 keyFile = lib.mkOption {
171 description = "Etcd key file.";
172 default = null;
173 type = types.nullOr types.path;
174 };
175
176 certFile = lib.mkOption {
177 description = "Etcd cert file.";
178 default = null;
179 type = types.nullOr types.path;
180 };
181
182 caFile = lib.mkOption {
183 description = "Etcd ca file.";
184 default = top.caFile;
185 defaultText = lib.literalExpression "config.${otop.caFile}";
186 type = types.nullOr types.path;
187 };
188 };
189
190 extraOpts = lib.mkOption {
191 description = "Kubernetes apiserver extra command line options.";
192 default = "";
193 type = separatedString " ";
194 };
195
196 extraSANs = lib.mkOption {
197 description = "Extra x509 Subject Alternative Names to be added to the kubernetes apiserver tls cert.";
198 default = [ ];
199 type = listOf str;
200 };
201
202 featureGates = lib.mkOption {
203 description = "Attribute set of feature gates.";
204 default = top.featureGates;
205 defaultText = lib.literalExpression "config.${otop.featureGates}";
206 type = attrsOf bool;
207 };
208
209 kubeletClientCaFile = lib.mkOption {
210 description = "Path to a cert file for connecting to kubelet.";
211 default = top.caFile;
212 defaultText = lib.literalExpression "config.${otop.caFile}";
213 type = nullOr path;
214 };
215
216 kubeletClientCertFile = lib.mkOption {
217 description = "Client certificate to use for connections to kubelet.";
218 default = null;
219 type = nullOr path;
220 };
221
222 kubeletClientKeyFile = lib.mkOption {
223 description = "Key to use for connections to kubelet.";
224 default = null;
225 type = nullOr path;
226 };
227
228 preferredAddressTypes = lib.mkOption {
229 description = "List of the preferred NodeAddressTypes to use for kubelet connections.";
230 type = nullOr str;
231 default = null;
232 };
233
234 proxyClientCertFile = lib.mkOption {
235 description = "Client certificate to use for connections to proxy.";
236 default = null;
237 type = nullOr path;
238 };
239
240 proxyClientKeyFile = lib.mkOption {
241 description = "Key to use for connections to proxy.";
242 default = null;
243 type = nullOr path;
244 };
245
246 runtimeConfig = lib.mkOption {
247 description = ''
248 Api runtime configuration. See
249 <https://kubernetes.io/docs/tasks/administer-cluster/cluster-management/>
250 '';
251 default = "authentication.k8s.io/v1beta1=true";
252 example = "api/all=false,api/v1=true";
253 type = str;
254 };
255
256 storageBackend = lib.mkOption {
257 description = ''
258 Kubernetes apiserver storage backend.
259 '';
260 default = "etcd3";
261 type = enum [
262 "etcd2"
263 "etcd3"
264 ];
265 };
266
267 securePort = lib.mkOption {
268 description = "Kubernetes apiserver secure port.";
269 default = 6443;
270 type = int;
271 };
272
273 apiAudiences = lib.mkOption {
274 description = ''
275 Kubernetes apiserver ServiceAccount issuer.
276 '';
277 default = "api,https://kubernetes.default.svc";
278 type = str;
279 };
280
281 serviceAccountIssuer = lib.mkOption {
282 description = ''
283 Kubernetes apiserver ServiceAccount issuer.
284 '';
285 default = "https://kubernetes.default.svc";
286 type = str;
287 };
288
289 serviceAccountSigningKeyFile = lib.mkOption {
290 description = ''
291 Path to the file that contains the current private key of the service
292 account token issuer. The issuer will sign issued ID tokens with this
293 private key.
294 '';
295 type = path;
296 };
297
298 serviceAccountKeyFile = lib.mkOption {
299 description = ''
300 File containing PEM-encoded x509 RSA or ECDSA private or public keys,
301 used to verify ServiceAccount tokens. The specified file can contain
302 multiple keys, and the flag can be specified multiple times with
303 different files. If unspecified, --tls-private-key-file is used.
304 Must be specified when --service-account-signing-key is provided
305 '';
306 type = path;
307 };
308
309 serviceClusterIpRange = lib.mkOption {
310 description = ''
311 A CIDR notation IP range from which to assign service cluster IPs.
312 This must not overlap with any IP ranges assigned to nodes for pods.
313 '';
314 default = "10.0.0.0/24";
315 type = str;
316 };
317
318 tlsCertFile = lib.mkOption {
319 description = "Kubernetes apiserver certificate file.";
320 default = null;
321 type = nullOr path;
322 };
323
324 tlsKeyFile = lib.mkOption {
325 description = "Kubernetes apiserver private key file.";
326 default = null;
327 type = nullOr path;
328 };
329
330 tokenAuthFile = lib.mkOption {
331 description = ''
332 Kubernetes apiserver token authentication file. See
333 <https://kubernetes.io/docs/reference/access-authn-authz/authentication>
334 '';
335 default = null;
336 type = nullOr path;
337 };
338
339 verbosity = lib.mkOption {
340 description = ''
341 Optional glog verbosity level for logging statements. See
342 <https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md>
343 '';
344 default = null;
345 type = nullOr int;
346 };
347
348 webhookConfig = lib.mkOption {
349 description = ''
350 Kubernetes apiserver Webhook config file. It uses the kubeconfig file format.
351 See <https://kubernetes.io/docs/reference/access-authn-authz/webhook/>
352 '';
353 default = null;
354 type = nullOr path;
355 };
356
357 };
358
359 ###### implementation
360 config = lib.mkMerge [
361
362 (lib.mkIf cfg.enable {
363 systemd.services.kube-apiserver = {
364 description = "Kubernetes APIServer Service";
365 wantedBy = [ "kubernetes.target" ];
366 after = [ "network.target" ];
367 serviceConfig = {
368 Slice = "kubernetes.slice";
369 ExecStart = ''
370 ${top.package}/bin/kube-apiserver \
371 --allow-privileged=${lib.boolToString cfg.allowPrivileged} \
372 --authorization-mode=${lib.concatStringsSep "," cfg.authorizationMode} \
373 ${lib.optionalString (lib.elem "ABAC" cfg.authorizationMode) "--authorization-policy-file=${pkgs.writeText "kube-auth-policy.jsonl" (lib.concatMapStringsSep "\n" (l: builtins.toJSON l) cfg.authorizationPolicy)}"} \
374 ${lib.optionalString (lib.elem "Webhook" cfg.authorizationMode) "--authorization-webhook-config-file=${cfg.webhookConfig}"} \
375 --bind-address=${cfg.bindAddress} \
376 ${lib.optionalString (cfg.advertiseAddress != null) "--advertise-address=${cfg.advertiseAddress}"} \
377 ${lib.optionalString (cfg.clientCaFile != null) "--client-ca-file=${cfg.clientCaFile}"} \
378 --disable-admission-plugins=${lib.concatStringsSep "," cfg.disableAdmissionPlugins} \
379 --enable-admission-plugins=${lib.concatStringsSep "," cfg.enableAdmissionPlugins} \
380 --etcd-servers=${lib.concatStringsSep "," cfg.etcd.servers} \
381 ${lib.optionalString (cfg.etcd.caFile != null) "--etcd-cafile=${cfg.etcd.caFile}"} \
382 ${lib.optionalString (cfg.etcd.certFile != null) "--etcd-certfile=${cfg.etcd.certFile}"} \
383 ${lib.optionalString (cfg.etcd.keyFile != null) "--etcd-keyfile=${cfg.etcd.keyFile}"} \
384 ${
385 lib.optionalString (cfg.featureGates != { })
386 "--feature-gates=${
387 (lib.concatStringsSep "," (
388 builtins.attrValues (lib.mapAttrs (n: v: "${n}=${lib.trivial.boolToString v}") cfg.featureGates)
389 ))
390 }"
391 } \
392 ${lib.optionalString (cfg.basicAuthFile != null) "--basic-auth-file=${cfg.basicAuthFile}"} \
393 ${
394 lib.optionalString (
395 cfg.kubeletClientCaFile != null
396 ) "--kubelet-certificate-authority=${cfg.kubeletClientCaFile}"
397 } \
398 ${
399 lib.optionalString (
400 cfg.kubeletClientCertFile != null
401 ) "--kubelet-client-certificate=${cfg.kubeletClientCertFile}"
402 } \
403 ${
404 lib.optionalString (
405 cfg.kubeletClientKeyFile != null
406 ) "--kubelet-client-key=${cfg.kubeletClientKeyFile}"
407 } \
408 ${
409 lib.optionalString (
410 cfg.preferredAddressTypes != null
411 ) "--kubelet-preferred-address-types=${cfg.preferredAddressTypes}"
412 } \
413 ${
414 lib.optionalString (
415 cfg.proxyClientCertFile != null
416 ) "--proxy-client-cert-file=${cfg.proxyClientCertFile}"
417 } \
418 ${
419 lib.optionalString (
420 cfg.proxyClientKeyFile != null
421 ) "--proxy-client-key-file=${cfg.proxyClientKeyFile}"
422 } \
423 ${lib.optionalString (cfg.runtimeConfig != "") "--runtime-config=${cfg.runtimeConfig}"} \
424 --secure-port=${toString cfg.securePort} \
425 --api-audiences=${toString cfg.apiAudiences} \
426 --service-account-issuer=${toString cfg.serviceAccountIssuer} \
427 --service-account-signing-key-file=${cfg.serviceAccountSigningKeyFile} \
428 --service-account-key-file=${cfg.serviceAccountKeyFile} \
429 --service-cluster-ip-range=${cfg.serviceClusterIpRange} \
430 --storage-backend=${cfg.storageBackend} \
431 ${lib.optionalString (cfg.tlsCertFile != null) "--tls-cert-file=${cfg.tlsCertFile}"} \
432 ${lib.optionalString (cfg.tlsKeyFile != null) "--tls-private-key-file=${cfg.tlsKeyFile}"} \
433 ${lib.optionalString (cfg.tokenAuthFile != null) "--token-auth-file=${cfg.tokenAuthFile}"} \
434 ${lib.optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
435 ${cfg.extraOpts}
436 '';
437 WorkingDirectory = top.dataDir;
438 User = "kubernetes";
439 Group = "kubernetes";
440 AmbientCapabilities = "cap_net_bind_service";
441 Restart = "on-failure";
442 RestartSec = 5;
443 };
444
445 unitConfig = {
446 StartLimitIntervalSec = 0;
447 };
448 };
449
450 services.etcd = {
451 clientCertAuth = lib.mkDefault true;
452 peerClientCertAuth = lib.mkDefault true;
453 listenClientUrls = lib.mkDefault [ "https://0.0.0.0:2379" ];
454 listenPeerUrls = lib.mkDefault [ "https://0.0.0.0:2380" ];
455 advertiseClientUrls = lib.mkDefault [ "https://${top.masterAddress}:2379" ];
456 initialCluster = lib.mkDefault [ "${top.masterAddress}=https://${top.masterAddress}:2380" ];
457 name = lib.mkDefault top.masterAddress;
458 initialAdvertisePeerUrls = lib.mkDefault [ "https://${top.masterAddress}:2380" ];
459 };
460
461 services.kubernetes.addonManager.bootstrapAddons = lib.mkIf isRBACEnabled {
462
463 apiserver-kubelet-api-admin-crb = {
464 apiVersion = "rbac.authorization.k8s.io/v1";
465 kind = "ClusterRoleBinding";
466 metadata = {
467 name = "system:kube-apiserver:kubelet-api-admin";
468 };
469 roleRef = {
470 apiGroup = "rbac.authorization.k8s.io";
471 kind = "ClusterRole";
472 name = "system:kubelet-api-admin";
473 };
474 subjects = [
475 {
476 kind = "User";
477 name = "system:kube-apiserver";
478 }
479 ];
480 };
481
482 };
483
484 services.kubernetes.pki.certs = with top.lib; {
485 apiServer = mkCert {
486 name = "kube-apiserver";
487 CN = "kubernetes";
488 hosts = [
489 "kubernetes.default.svc"
490 "kubernetes.default.svc.${top.addons.dns.clusterDomain}"
491 cfg.advertiseAddress
492 top.masterAddress
493 apiserverServiceIP
494 "127.0.0.1"
495 ] ++ cfg.extraSANs;
496 action = "systemctl restart kube-apiserver.service";
497 };
498 apiserverProxyClient = mkCert {
499 name = "kube-apiserver-proxy-client";
500 CN = "front-proxy-client";
501 action = "systemctl restart kube-apiserver.service";
502 };
503 apiserverKubeletClient = mkCert {
504 name = "kube-apiserver-kubelet-client";
505 CN = "system:kube-apiserver";
506 action = "systemctl restart kube-apiserver.service";
507 };
508 apiserverEtcdClient = mkCert {
509 name = "kube-apiserver-etcd-client";
510 CN = "etcd-client";
511 action = "systemctl restart kube-apiserver.service";
512 };
513 clusterAdmin = mkCert {
514 name = "cluster-admin";
515 CN = "cluster-admin";
516 fields = {
517 O = "system:masters";
518 };
519 privateKeyOwner = "root";
520 };
521 etcd = mkCert {
522 name = "etcd";
523 CN = top.masterAddress;
524 hosts = [
525 "etcd.local"
526 "etcd.${top.addons.dns.clusterDomain}"
527 top.masterAddress
528 cfg.advertiseAddress
529 ];
530 privateKeyOwner = "etcd";
531 action = "systemctl restart etcd.service";
532 };
533 };
534
535 })
536
537 ];
538
539 meta.buildDocsInSandbox = false;
540}