pyrox.dev
1{ config, lib, pkgs, ... }:
2
3with lib;
4
5let
6 cfg = config.services.kubernetes;
7
8 # YAML config; see:
9 # https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/
10 # https://github.com/kubernetes/kubernetes/blob/release-1.10/pkg/kubelet/apis/kubeletconfig/v1beta1/types.go
11 #
12 # TODO: migrate the following flags to this config file
13 #
14 # --pod-manifest-path
15 # --address
16 # --port
17 # --tls-cert-file
18 # --tls-private-key-file
19 # --client-ca-file
20 # --authentication-token-webhook
21 # --authentication-token-webhook-cache-ttl
22 # --authorization-mode
23 # --healthz-bind-address
24 # --healthz-port
25 # --allow-privileged
26 # --cluster-dns
27 # --cluster-domain
28 # --hairpin-mode
29 # --feature-gates
30 kubeletConfig = pkgs.runCommand "kubelet-config.yaml" { } ''
31 echo > $out ${pkgs.lib.escapeShellArg (builtins.toJSON {
32 kind = "KubeletConfiguration";
33 apiVersion = "kubelet.config.k8s.io/v1beta1";
34 ${if cfg.kubelet.applyManifests then "staticPodPath" else null} =
35 manifests;
36 })}
37 '';
38
39 infraContainer = pkgs.dockerTools.buildImage {
40 name = "pause";
41 tag = "latest";
42 contents = cfg.package.pause;
43 config.Cmd = "/bin/pause";
44 };
45
46 mkKubeConfig = name: cfg: pkgs.writeText "${name}-kubeconfig" (builtins.toJSON {
47 apiVersion = "v1";
48 kind = "Config";
49 clusters = [{
50 name = "local";
51 cluster.certificate-authority = cfg.caFile;
52 cluster.server = cfg.server;
53 }];
54 users = [{
55 name = "kubelet";
56 user = {
57 client-certificate = cfg.certFile;
58 client-key = cfg.keyFile;
59 };
60 }];
61 contexts = [{
62 context = {
63 cluster = "local";
64 user = "kubelet";
65 };
66 current-context = "kubelet-context";
67 }];
68 });
69
70 mkKubeConfigOptions = prefix: {
71 server = mkOption {
72 description = "${prefix} kube-apiserver server address.";
73 default = "http://${if cfg.apiserver.advertiseAddress != null
74 then cfg.apiserver.advertiseAddress
75 else "127.0.0.1"}:${toString cfg.apiserver.port}";
76 type = types.str;
77 };
78
79 caFile = mkOption {
80 description = "${prefix} certificate authority file used to connect to kube-apiserver.";
81 type = types.nullOr types.path;
82 default = cfg.caFile;
83 };
84
85 certFile = mkOption {
86 description = "${prefix} client certificate file used to connect to kube-apiserver.";
87 type = types.nullOr types.path;
88 default = null;
89 };
90
91 keyFile = mkOption {
92 description = "${prefix} client key file used to connect to kube-apiserver.";
93 type = types.nullOr types.path;
94 default = null;
95 };
96 };
97
98 kubeConfigDefaults = {
99 server = mkDefault cfg.kubeconfig.server;
100 caFile = mkDefault cfg.kubeconfig.caFile;
101 certFile = mkDefault cfg.kubeconfig.certFile;
102 keyFile = mkDefault cfg.kubeconfig.keyFile;
103 };
104
105 cniConfig =
106 if cfg.kubelet.cni.config != [] && !(isNull cfg.kubelet.cni.configDir) then
107 throw "Verbatim CNI-config and CNI configDir cannot both be set."
108 else if !(isNull cfg.kubelet.cni.configDir) then
109 cfg.kubelet.cni.configDir
110 else
111 (pkgs.buildEnv {
112 name = "kubernetes-cni-config";
113 paths = imap (i: entry:
114 pkgs.writeTextDir "${toString (10+i)}-${entry.type}.conf" (builtins.toJSON entry)
115 ) cfg.kubelet.cni.config;
116 });
117
118 manifests = pkgs.buildEnv {
119 name = "kubernetes-manifests";
120 paths = mapAttrsToList (name: manifest:
121 pkgs.writeTextDir "${name}.json" (builtins.toJSON manifest)
122 ) cfg.kubelet.manifests;
123 };
124
125 addons = pkgs.runCommand "kubernetes-addons" { } ''
126 mkdir -p $out
127 # since we are mounting the addons to the addon manager, they need to be copied
128 ${concatMapStringsSep ";" (a: "cp -v ${a}/* $out/") (mapAttrsToList (name: addon:
129 pkgs.writeTextDir "${name}.json" (builtins.toJSON addon)
130 ) (cfg.addonManager.addons))}
131 '';
132
133 taintOptions = { name, ... }: {
134 options = {
135 key = mkOption {
136 description = "Key of taint.";
137 default = name;
138 type = types.str;
139 };
140 value = mkOption {
141 description = "Value of taint.";
142 type = types.str;
143 };
144 effect = mkOption {
145 description = "Effect of taint.";
146 example = "NoSchedule";
147 type = types.enum ["NoSchedule" "PreferNoSchedule" "NoExecute"];
148 };
149 };
150 };
151
152 taints = concatMapStringsSep "," (v: "${v.key}=${v.value}:${v.effect}") (mapAttrsToList (n: v: v) cfg.kubelet.taints);
153
154 # needed for flannel to pass options to docker
155 mkDockerOpts = pkgs.runCommand "mk-docker-opts" {
156 buildInputs = [ pkgs.makeWrapper ];
157 } ''
158 mkdir -p $out
159 cp ${pkgs.kubernetes.src}/cluster/centos/node/bin/mk-docker-opts.sh $out/mk-docker-opts.sh
160
161 # bashInteractive needed for `compgen`
162 makeWrapper ${pkgs.bashInteractive}/bin/bash $out/mk-docker-opts --add-flags "$out/mk-docker-opts.sh"
163 '';
164in {
165
166 ###### interface
167
168 options.services.kubernetes = {
169 roles = mkOption {
170 description = ''
171 Kubernetes role that this machine should take.
172
173 Master role will enable etcd, apiserver, scheduler and controller manager
174 services. Node role will enable etcd, docker, kubelet and proxy services.
175 '';
176 default = [];
177 type = types.listOf (types.enum ["master" "node"]);
178 };
179
180 package = mkOption {
181 description = "Kubernetes package to use.";
182 type = types.package;
183 default = pkgs.kubernetes;
184 defaultText = "pkgs.kubernetes";
185 };
186
187 verbose = mkOption {
188 description = "Kubernetes enable verbose mode for debugging.";
189 default = false;
190 type = types.bool;
191 };
192
193 etcd = {
194 servers = mkOption {
195 description = "List of etcd servers. By default etcd is started, except if this option is changed.";
196 default = ["http://127.0.0.1:2379"];
197 type = types.listOf types.str;
198 };
199
200 keyFile = mkOption {
201 description = "Etcd key file.";
202 default = null;
203 type = types.nullOr types.path;
204 };
205
206 certFile = mkOption {
207 description = "Etcd cert file.";
208 default = null;
209 type = types.nullOr types.path;
210 };
211
212 caFile = mkOption {
213 description = "Etcd ca file.";
214 default = cfg.caFile;
215 type = types.nullOr types.path;
216 };
217 };
218
219 kubeconfig = mkKubeConfigOptions "Default kubeconfig";
220
221 caFile = mkOption {
222 description = "Default kubernetes certificate authority";
223 type = types.nullOr types.path;
224 default = null;
225 };
226
227 dataDir = mkOption {
228 description = "Kubernetes root directory for managing kubelet files.";
229 default = "/var/lib/kubernetes";
230 type = types.path;
231 };
232
233 featureGates = mkOption {
234 description = "List set of feature gates";
235 default = [];
236 type = types.listOf types.str;
237 };
238
239 apiserver = {
240 enable = mkOption {
241 description = "Whether to enable Kubernetes apiserver.";
242 default = false;
243 type = types.bool;
244 };
245
246 featureGates = mkOption {
247 description = "List set of feature gates";
248 default = cfg.featureGates;
249 type = types.listOf types.str;
250 };
251
252 bindAddress = mkOption {
253 description = ''
254 The IP address on which to listen for the --secure-port port.
255 The associated interface(s) must be reachable by the rest
256 of the cluster, and by CLI/web clients.
257 '';
258 default = "0.0.0.0";
259 type = types.str;
260 };
261
262 advertiseAddress = mkOption {
263 description = ''
264 Kubernetes apiserver IP address on which to advertise the apiserver
265 to members of the cluster. This address must be reachable by the rest
266 of the cluster.
267 '';
268 default = null;
269 type = types.nullOr types.str;
270 };
271
272 storageBackend = mkOption {
273 description = ''
274 Kubernetes apiserver storage backend.
275 '';
276 default = "etcd3";
277 type = types.enum ["etcd2" "etcd3"];
278 };
279
280 port = mkOption {
281 description = "Kubernetes apiserver listening port.";
282 default = 8080;
283 type = types.int;
284 };
285
286 securePort = mkOption {
287 description = "Kubernetes apiserver secure port.";
288 default = 443;
289 type = types.int;
290 };
291
292 tlsCertFile = mkOption {
293 description = "Kubernetes apiserver certificate file.";
294 default = null;
295 type = types.nullOr types.path;
296 };
297
298 tlsKeyFile = mkOption {
299 description = "Kubernetes apiserver private key file.";
300 default = null;
301 type = types.nullOr types.path;
302 };
303
304 clientCaFile = mkOption {
305 description = "Kubernetes apiserver CA file for client auth.";
306 default = cfg.caFile;
307 type = types.nullOr types.path;
308 };
309
310 tokenAuthFile = mkOption {
311 description = ''
312 Kubernetes apiserver token authentication file. See
313 <link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/authentication"/>
314 '';
315 default = null;
316 type = types.nullOr types.path;
317 };
318
319 basicAuthFile = mkOption {
320 description = ''
321 Kubernetes apiserver basic authentication file. See
322 <link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/authentication"/>
323 '';
324 default = pkgs.writeText "users" ''
325 kubernetes,admin,0
326 '';
327 type = types.nullOr types.path;
328 };
329
330 authorizationMode = mkOption {
331 description = ''
332 Kubernetes apiserver authorization mode (AlwaysAllow/AlwaysDeny/ABAC/Webhook/RBAC/Node). See
333 <link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/authorization/"/>
334 '';
335 default = ["RBAC" "Node"];
336 type = types.listOf (types.enum ["AlwaysAllow" "AlwaysDeny" "ABAC" "Webhook" "RBAC" "Node"]);
337 };
338
339 authorizationPolicy = mkOption {
340 description = ''
341 Kubernetes apiserver authorization policy file. See
342 <link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/authorization/"/>
343 '';
344 default = [];
345 type = types.listOf types.attrs;
346 };
347
348 webhookConfig = mkOption {
349 description = ''
350 Kubernetes apiserver Webhook config file. It uses the kubeconfig file format.
351 See <link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/webhook/"/>
352 '';
353 default = null;
354 type = types.nullOr types.path;
355 };
356
357 allowPrivileged = mkOption {
358 description = "Whether to allow privileged containers on Kubernetes.";
359 default = true;
360 type = types.bool;
361 };
362
363 serviceClusterIpRange = mkOption {
364 description = ''
365 A CIDR notation IP range from which to assign service cluster IPs.
366 This must not overlap with any IP ranges assigned to nodes for pods.
367 '';
368 default = "10.0.0.0/24";
369 type = types.str;
370 };
371
372 runtimeConfig = mkOption {
373 description = ''
374 Api runtime configuration. See
375 <link xlink:href="https://kubernetes.io/docs/tasks/administer-cluster/cluster-management/"/>
376 '';
377 default = "authentication.k8s.io/v1beta1=true";
378 example = "api/all=false,api/v1=true";
379 type = types.str;
380 };
381
382 enableAdmissionPlugins = mkOption {
383 description = ''
384 Kubernetes admission control plugins to enable. See
385 <link xlink:href="https://kubernetes.io/docs/admin/admission-controllers/"/>
386 '';
387 default = ["NamespaceLifecycle" "LimitRanger" "ServiceAccount" "ResourceQuota" "DefaultStorageClass" "DefaultTolerationSeconds" "NodeRestriction"];
388 example = [
389 "NamespaceLifecycle" "NamespaceExists" "LimitRanger"
390 "SecurityContextDeny" "ServiceAccount" "ResourceQuota"
391 "PodSecurityPolicy" "NodeRestriction" "DefaultStorageClass"
392 ];
393 type = types.listOf types.str;
394 };
395
396 disableAdmissionPlugins = mkOption {
397 description = ''
398 Kubernetes admission control plugins to disable. See
399 <link xlink:href="https://kubernetes.io/docs/admin/admission-controllers/"/>
400 '';
401 default = [];
402 type = types.listOf types.str;
403 };
404
405 serviceAccountKeyFile = mkOption {
406 description = ''
407 Kubernetes apiserver PEM-encoded x509 RSA private or public key file,
408 used to verify ServiceAccount tokens. By default tls private key file
409 is used.
410 '';
411 default = null;
412 type = types.nullOr types.path;
413 };
414
415 kubeletClientCaFile = mkOption {
416 description = "Path to a cert file for connecting to kubelet.";
417 default = cfg.caFile;
418 type = types.nullOr types.path;
419 };
420
421 kubeletClientCertFile = mkOption {
422 description = "Client certificate to use for connections to kubelet.";
423 default = null;
424 type = types.nullOr types.path;
425 };
426
427 kubeletClientKeyFile = mkOption {
428 description = "Key to use for connections to kubelet.";
429 default = null;
430 type = types.nullOr types.path;
431 };
432
433 kubeletHttps = mkOption {
434 description = "Whether to use https for connections to kubelet.";
435 default = true;
436 type = types.bool;
437 };
438
439 extraOpts = mkOption {
440 description = "Kubernetes apiserver extra command line options.";
441 default = "";
442 type = types.str;
443 };
444 };
445
446 scheduler = {
447 enable = mkOption {
448 description = "Whether to enable Kubernetes scheduler.";
449 default = false;
450 type = types.bool;
451 };
452
453 featureGates = mkOption {
454 description = "List set of feature gates";
455 default = cfg.featureGates;
456 type = types.listOf types.str;
457 };
458
459 address = mkOption {
460 description = "Kubernetes scheduler listening address.";
461 default = "127.0.0.1";
462 type = types.str;
463 };
464
465 port = mkOption {
466 description = "Kubernetes scheduler listening port.";
467 default = 10251;
468 type = types.int;
469 };
470
471 leaderElect = mkOption {
472 description = "Whether to start leader election before executing main loop.";
473 type = types.bool;
474 default = true;
475 };
476
477 kubeconfig = mkKubeConfigOptions "Kubernetes scheduler";
478
479 extraOpts = mkOption {
480 description = "Kubernetes scheduler extra command line options.";
481 default = "";
482 type = types.str;
483 };
484 };
485
486 controllerManager = {
487 enable = mkOption {
488 description = "Whether to enable Kubernetes controller manager.";
489 default = false;
490 type = types.bool;
491 };
492
493 featureGates = mkOption {
494 description = "List set of feature gates";
495 default = cfg.featureGates;
496 type = types.listOf types.str;
497 };
498
499 address = mkOption {
500 description = "Kubernetes controller manager listening address.";
501 default = "127.0.0.1";
502 type = types.str;
503 };
504
505 port = mkOption {
506 description = "Kubernetes controller manager listening port.";
507 default = 10252;
508 type = types.int;
509 };
510
511 leaderElect = mkOption {
512 description = "Whether to start leader election before executing main loop.";
513 type = types.bool;
514 default = true;
515 };
516
517 serviceAccountKeyFile = mkOption {
518 description = ''
519 Kubernetes controller manager PEM-encoded private RSA key file used to
520 sign service account tokens
521 '';
522 default = null;
523 type = types.nullOr types.path;
524 };
525
526 rootCaFile = mkOption {
527 description = ''
528 Kubernetes controller manager certificate authority file included in
529 service account's token secret.
530 '';
531 default = cfg.caFile;
532 type = types.nullOr types.path;
533 };
534
535 kubeconfig = mkKubeConfigOptions "Kubernetes controller manager";
536
537 extraOpts = mkOption {
538 description = "Kubernetes controller manager extra command line options.";
539 default = "";
540 type = types.str;
541 };
542 };
543
544 kubelet = {
545 enable = mkOption {
546 description = "Whether to enable Kubernetes kubelet.";
547 default = false;
548 type = types.bool;
549 };
550
551 featureGates = mkOption {
552 description = "List set of feature gates";
553 default = cfg.featureGates;
554 type = types.listOf types.str;
555 };
556
557 seedDockerImages = mkOption {
558 description = "List of docker images to preload on system";
559 default = [];
560 type = types.listOf types.package;
561 };
562
563 registerNode = mkOption {
564 description = "Whether to auto register kubelet with API server.";
565 default = true;
566 type = types.bool;
567 };
568
569 address = mkOption {
570 description = "Kubernetes kubelet info server listening address.";
571 default = "0.0.0.0";
572 type = types.str;
573 };
574
575 port = mkOption {
576 description = "Kubernetes kubelet info server listening port.";
577 default = 10250;
578 type = types.int;
579 };
580
581 tlsCertFile = mkOption {
582 description = "File containing x509 Certificate for HTTPS.";
583 default = null;
584 type = types.nullOr types.path;
585 };
586
587 tlsKeyFile = mkOption {
588 description = "File containing x509 private key matching tlsCertFile.";
589 default = null;
590 type = types.nullOr types.path;
591 };
592
593 clientCaFile = mkOption {
594 description = "Kubernetes apiserver CA file for client authentication.";
595 default = cfg.caFile;
596 type = types.nullOr types.path;
597 };
598
599 healthz = {
600 bind = mkOption {
601 description = "Kubernetes kubelet healthz listening address.";
602 default = "127.0.0.1";
603 type = types.str;
604 };
605
606 port = mkOption {
607 description = "Kubernetes kubelet healthz port.";
608 default = 10248;
609 type = types.int;
610 };
611 };
612
613 hostname = mkOption {
614 description = "Kubernetes kubelet hostname override.";
615 default = config.networking.hostName;
616 type = types.str;
617 };
618
619 allowPrivileged = mkOption {
620 description = "Whether to allow Kubernetes containers to request privileged mode.";
621 default = true;
622 type = types.bool;
623 };
624
625 # TODO: remove this deprecated flag
626 cadvisorPort = mkOption {
627 description = "Kubernetes kubelet local cadvisor port.";
628 default = 4194;
629 type = types.int;
630 };
631
632 clusterDns = mkOption {
633 description = "Use alternative DNS.";
634 default = "10.1.0.1";
635 type = types.str;
636 };
637
638 clusterDomain = mkOption {
639 description = "Use alternative domain.";
640 default = config.services.kubernetes.addons.dns.clusterDomain;
641 type = types.str;
642 };
643
644 networkPlugin = mkOption {
645 description = "Network plugin to use by Kubernetes.";
646 type = types.nullOr (types.enum ["cni" "kubenet"]);
647 default = "kubenet";
648 };
649
650 cni = {
651 packages = mkOption {
652 description = "List of network plugin packages to install.";
653 type = types.listOf types.package;
654 default = [];
655 };
656
657 config = mkOption {
658 description = "Kubernetes CNI configuration.";
659 type = types.listOf types.attrs;
660 default = [];
661 example = literalExample ''
662 [{
663 "cniVersion": "0.2.0",
664 "name": "mynet",
665 "type": "bridge",
666 "bridge": "cni0",
667 "isGateway": true,
668 "ipMasq": true,
669 "ipam": {
670 "type": "host-local",
671 "subnet": "10.22.0.0/16",
672 "routes": [
673 { "dst": "0.0.0.0/0" }
674 ]
675 }
676 } {
677 "cniVersion": "0.2.0",
678 "type": "loopback"
679 }]
680 '';
681 };
682
683 configDir = mkOption {
684 description = "Path to Kubernetes CNI configuration directory.";
685 type = types.nullOr types.path;
686 default = null;
687 };
688 };
689
690 manifests = mkOption {
691 description = "List of manifests to bootstrap with kubelet (only pods can be created as manifest entry)";
692 type = types.attrsOf types.attrs;
693 default = {};
694 };
695
696 applyManifests = mkOption {
697 description = "Whether to apply manifests (this is true for master node).";
698 default = false;
699 type = types.bool;
700 };
701
702 unschedulable = mkOption {
703 description = "Whether to set node taint to unschedulable=true as it is the case of node that has only master role.";
704 default = false;
705 type = types.bool;
706 };
707
708 taints = mkOption {
709 description = "Node taints (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/).";
710 default = {};
711 type = types.attrsOf (types.submodule [ taintOptions ]);
712 };
713
714 nodeIp = mkOption {
715 description = "IP address of the node. If set, kubelet will use this IP address for the node.";
716 default = null;
717 type = types.nullOr types.str;
718 };
719
720 kubeconfig = mkKubeConfigOptions "Kubelet";
721
722 extraOpts = mkOption {
723 description = "Kubernetes kubelet extra command line options.";
724 default = "";
725 type = types.str;
726 };
727 };
728
729 proxy = {
730 enable = mkOption {
731 description = "Whether to enable Kubernetes proxy.";
732 default = false;
733 type = types.bool;
734 };
735
736 featureGates = mkOption {
737 description = "List set of feature gates";
738 default = cfg.featureGates;
739 type = types.listOf types.str;
740 };
741
742 address = mkOption {
743 description = "Kubernetes proxy listening address.";
744 default = "0.0.0.0";
745 type = types.str;
746 };
747
748 kubeconfig = mkKubeConfigOptions "Kubernetes proxy";
749
750 extraOpts = mkOption {
751 description = "Kubernetes proxy extra command line options.";
752 default = "";
753 type = types.str;
754 };
755 };
756
757 addonManager = {
758 enable = mkOption {
759 description = "Whether to enable Kubernetes addon manager.";
760 default = false;
761 type = types.bool;
762 };
763
764 addons = mkOption {
765 description = "Kubernetes addons (any kind of Kubernetes resource can be an addon).";
766 default = { };
767 type = types.attrsOf (types.either types.attrs (types.listOf types.attrs));
768 example = literalExample ''
769 {
770 "my-service" = {
771 "apiVersion" = "v1";
772 "kind" = "Service";
773 "metadata" = {
774 "name" = "my-service";
775 "namespace" = "default";
776 };
777 "spec" = { ... };
778 };
779 }
780 // import <nixpkgs/nixos/modules/services/cluster/kubernetes/dashboard.nix> { cfg = config.services.kubernetes; };
781 '';
782 };
783 };
784
785 path = mkOption {
786 description = "Packages added to the services' PATH environment variable. Both the bin and sbin subdirectories of each package are added.";
787 type = types.listOf types.package;
788 default = [];
789 };
790
791 clusterCidr = mkOption {
792 description = "Kubernetes controller manager and proxy CIDR Range for Pods in cluster.";
793 default = "10.1.0.0/16";
794 type = types.str;
795 };
796
797 flannel.enable = mkOption {
798 description = "Whether to enable flannel networking";
799 default = false;
800 type = types.bool;
801 };
802
803 };
804
805 ###### implementation
806
807 config = mkMerge [
808 (mkIf cfg.kubelet.enable {
809 services.kubernetes.kubelet.seedDockerImages = [infraContainer];
810
811 systemd.services.kubelet-bootstrap = {
812 description = "Boostrap Kubelet";
813 wantedBy = ["kubernetes.target"];
814 after = ["docker.service" "network.target"];
815 path = with pkgs; [ docker ];
816 script = ''
817 ${concatMapStrings (img: ''
818 echo "Seeding docker image: ${img}"
819 docker load <${img}
820 '') cfg.kubelet.seedDockerImages}
821
822 rm /opt/cni/bin/* || true
823 ${concatMapStrings (package: ''
824 echo "Linking cni package: ${package}"
825 ln -fs ${package}/bin/* /opt/cni/bin
826 '') cfg.kubelet.cni.packages}
827 '';
828 serviceConfig = {
829 Slice = "kubernetes.slice";
830 Type = "oneshot";
831 };
832 };
833
834 systemd.services.kubelet = {
835 description = "Kubernetes Kubelet Service";
836 wantedBy = [ "kubernetes.target" ];
837 after = [ "network.target" "docker.service" "kube-apiserver.service" "kubelet-bootstrap.service" ];
838 path = with pkgs; [ gitMinimal openssh docker utillinux iproute ethtool thin-provisioning-tools iptables socat ] ++ cfg.path;
839 serviceConfig = {
840 Slice = "kubernetes.slice";
841 CPUAccounting = true;
842 MemoryAccounting = true;
843 ExecStart = ''${cfg.package}/bin/kubelet \
844 ${optionalString (taints != "")
845 "--register-with-taints=${taints}"} \
846 --kubeconfig=${mkKubeConfig "kubelet" cfg.kubelet.kubeconfig} \
847 --config=${kubeletConfig} \
848 --address=${cfg.kubelet.address} \
849 --port=${toString cfg.kubelet.port} \
850 --register-node=${boolToString cfg.kubelet.registerNode} \
851 ${optionalString (cfg.kubelet.tlsCertFile != null)
852 "--tls-cert-file=${cfg.kubelet.tlsCertFile}"} \
853 ${optionalString (cfg.kubelet.tlsKeyFile != null)
854 "--tls-private-key-file=${cfg.kubelet.tlsKeyFile}"} \
855 ${optionalString (cfg.kubelet.clientCaFile != null)
856 "--client-ca-file=${cfg.kubelet.clientCaFile}"} \
857 --authentication-token-webhook \
858 --authentication-token-webhook-cache-ttl="10s" \
859 --authorization-mode=Webhook \
860 --healthz-bind-address=${cfg.kubelet.healthz.bind} \
861 --healthz-port=${toString cfg.kubelet.healthz.port} \
862 --hostname-override=${cfg.kubelet.hostname} \
863 --allow-privileged=${boolToString cfg.kubelet.allowPrivileged} \
864 --root-dir=${cfg.dataDir} \
865 --cadvisor_port=${toString cfg.kubelet.cadvisorPort} \
866 ${optionalString (cfg.kubelet.clusterDns != "")
867 "--cluster-dns=${cfg.kubelet.clusterDns}"} \
868 ${optionalString (cfg.kubelet.clusterDomain != "")
869 "--cluster-domain=${cfg.kubelet.clusterDomain}"} \
870 --pod-infra-container-image=pause \
871 ${optionalString (cfg.kubelet.networkPlugin != null)
872 "--network-plugin=${cfg.kubelet.networkPlugin}"} \
873 --cni-conf-dir=${cniConfig} \
874 --hairpin-mode=hairpin-veth \
875 ${optionalString (cfg.kubelet.nodeIp != null)
876 "--node-ip=${cfg.kubelet.nodeIp}"} \
877 ${optionalString (cfg.kubelet.featureGates != [])
878 "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.kubelet.featureGates}"} \
879 ${optionalString cfg.verbose "--v=6 --log_flush_frequency=1s"} \
880 ${cfg.kubelet.extraOpts}
881 '';
882 WorkingDirectory = cfg.dataDir;
883 };
884 };
885
886 # Allways include cni plugins
887 services.kubernetes.kubelet.cni.packages = [pkgs.cni-plugins];
888
889 boot.kernelModules = ["br_netfilter"];
890
891 services.kubernetes.kubelet.kubeconfig = kubeConfigDefaults;
892 })
893
894 (mkIf (cfg.kubelet.applyManifests && cfg.kubelet.enable) {
895 environment.etc = mapAttrs' (name: manifest:
896 nameValuePair "kubernetes/manifests/${name}.json" {
897 text = builtins.toJSON manifest;
898 mode = "0755";
899 }
900 ) cfg.kubelet.manifests;
901 })
902
903 (mkIf (cfg.kubelet.unschedulable && cfg.kubelet.enable) {
904 services.kubernetes.kubelet.taints.unschedulable = {
905 value = "true";
906 effect = "NoSchedule";
907 };
908 })
909
910 (mkIf cfg.apiserver.enable {
911 systemd.services.kube-apiserver = {
912 description = "Kubernetes APIServer Service";
913 wantedBy = [ "kubernetes.target" ];
914 after = [ "network.target" "docker.service" ];
915 serviceConfig = {
916 Slice = "kubernetes.slice";
917 ExecStart = ''${cfg.package}/bin/kube-apiserver \
918 --etcd-servers=${concatStringsSep "," cfg.etcd.servers} \
919 ${optionalString (cfg.etcd.caFile != null)
920 "--etcd-cafile=${cfg.etcd.caFile}"} \
921 ${optionalString (cfg.etcd.certFile != null)
922 "--etcd-certfile=${cfg.etcd.certFile}"} \
923 ${optionalString (cfg.etcd.keyFile != null)
924 "--etcd-keyfile=${cfg.etcd.keyFile}"} \
925 --insecure-port=${toString cfg.apiserver.port} \
926 --bind-address=${cfg.apiserver.bindAddress} \
927 ${optionalString (cfg.apiserver.advertiseAddress != null)
928 "--advertise-address=${cfg.apiserver.advertiseAddress}"} \
929 --allow-privileged=${boolToString cfg.apiserver.allowPrivileged}\
930 ${optionalString (cfg.apiserver.tlsCertFile != null)
931 "--tls-cert-file=${cfg.apiserver.tlsCertFile}"} \
932 ${optionalString (cfg.apiserver.tlsKeyFile != null)
933 "--tls-private-key-file=${cfg.apiserver.tlsKeyFile}"} \
934 ${optionalString (cfg.apiserver.tokenAuthFile != null)
935 "--token-auth-file=${cfg.apiserver.tokenAuthFile}"} \
936 ${optionalString (cfg.apiserver.basicAuthFile != null)
937 "--basic-auth-file=${cfg.apiserver.basicAuthFile}"} \
938 --kubelet-https=${if cfg.apiserver.kubeletHttps then "true" else "false"} \
939 ${optionalString (cfg.apiserver.kubeletClientCaFile != null)
940 "--kubelet-certificate-authority=${cfg.apiserver.kubeletClientCaFile}"} \
941 ${optionalString (cfg.apiserver.kubeletClientCertFile != null)
942 "--kubelet-client-certificate=${cfg.apiserver.kubeletClientCertFile}"} \
943 ${optionalString (cfg.apiserver.kubeletClientKeyFile != null)
944 "--kubelet-client-key=${cfg.apiserver.kubeletClientKeyFile}"} \
945 ${optionalString (cfg.apiserver.clientCaFile != null)
946 "--client-ca-file=${cfg.apiserver.clientCaFile}"} \
947 --authorization-mode=${concatStringsSep "," cfg.apiserver.authorizationMode} \
948 ${optionalString (elem "ABAC" cfg.apiserver.authorizationMode)
949 "--authorization-policy-file=${
950 pkgs.writeText "kube-auth-policy.jsonl"
951 (concatMapStringsSep "\n" (l: builtins.toJSON l) cfg.apiserver.authorizationPolicy)
952 }"
953 } \
954 ${optionalString (elem "Webhook" cfg.apiserver.authorizationMode)
955 "--authorization-webhook-config-file=${cfg.apiserver.webhookConfig}"
956 } \
957 --secure-port=${toString cfg.apiserver.securePort} \
958 --service-cluster-ip-range=${cfg.apiserver.serviceClusterIpRange} \
959 ${optionalString (cfg.apiserver.runtimeConfig != "")
960 "--runtime-config=${cfg.apiserver.runtimeConfig}"} \
961 --enable-admission-plugins=${concatStringsSep "," cfg.apiserver.enableAdmissionPlugins} \
962 --disable-admission-plugins=${concatStringsSep "," cfg.apiserver.disableAdmissionPlugins} \
963 ${optionalString (cfg.apiserver.serviceAccountKeyFile!=null)
964 "--service-account-key-file=${cfg.apiserver.serviceAccountKeyFile}"} \
965 ${optionalString cfg.verbose "--v=6"} \
966 ${optionalString cfg.verbose "--log-flush-frequency=1s"} \
967 --storage-backend=${cfg.apiserver.storageBackend} \
968 ${optionalString (cfg.kubelet.featureGates != [])
969 "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.kubelet.featureGates}"} \
970 ${cfg.apiserver.extraOpts}
971 '';
972 WorkingDirectory = cfg.dataDir;
973 User = "kubernetes";
974 Group = "kubernetes";
975 AmbientCapabilities = "cap_net_bind_service";
976 Restart = "on-failure";
977 RestartSec = 5;
978 };
979 };
980 })
981
982 (mkIf cfg.scheduler.enable {
983 systemd.services.kube-scheduler = {
984 description = "Kubernetes Scheduler Service";
985 wantedBy = [ "kubernetes.target" ];
986 after = [ "kube-apiserver.service" ];
987 serviceConfig = {
988 Slice = "kubernetes.slice";
989 ExecStart = ''${cfg.package}/bin/kube-scheduler \
990 --address=${cfg.scheduler.address} \
991 --port=${toString cfg.scheduler.port} \
992 --leader-elect=${boolToString cfg.scheduler.leaderElect} \
993 --kubeconfig=${mkKubeConfig "kube-scheduler" cfg.scheduler.kubeconfig} \
994 ${optionalString cfg.verbose "--v=6"} \
995 ${optionalString cfg.verbose "--log-flush-frequency=1s"} \
996 ${optionalString (cfg.scheduler.featureGates != [])
997 "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.scheduler.featureGates}"} \
998 ${cfg.scheduler.extraOpts}
999 '';
1000 WorkingDirectory = cfg.dataDir;
1001 User = "kubernetes";
1002 Group = "kubernetes";
1003 };
1004 };
1005
1006 services.kubernetes.scheduler.kubeconfig = kubeConfigDefaults;
1007 })
1008
1009 (mkIf cfg.controllerManager.enable {
1010 systemd.services.kube-controller-manager = {
1011 description = "Kubernetes Controller Manager Service";
1012 wantedBy = [ "kubernetes.target" ];
1013 after = [ "kube-apiserver.service" ];
1014 serviceConfig = {
1015 RestartSec = "30s";
1016 Restart = "on-failure";
1017 Slice = "kubernetes.slice";
1018 ExecStart = ''${cfg.package}/bin/kube-controller-manager \
1019 --address=${cfg.controllerManager.address} \
1020 --port=${toString cfg.controllerManager.port} \
1021 --kubeconfig=${mkKubeConfig "kube-controller-manager" cfg.controllerManager.kubeconfig} \
1022 --leader-elect=${boolToString cfg.controllerManager.leaderElect} \
1023 ${if (cfg.controllerManager.serviceAccountKeyFile!=null)
1024 then "--service-account-private-key-file=${cfg.controllerManager.serviceAccountKeyFile}"
1025 else "--service-account-private-key-file=/var/run/kubernetes/apiserver.key"} \
1026 ${if (cfg.controllerManager.rootCaFile!=null)
1027 then "--root-ca-file=${cfg.controllerManager.rootCaFile}"
1028 else "--root-ca-file=/var/run/kubernetes/apiserver.crt"} \
1029 ${optionalString (cfg.clusterCidr!=null)
1030 "--cluster-cidr=${cfg.clusterCidr}"} \
1031 --allocate-node-cidrs=true \
1032 ${optionalString (cfg.controllerManager.featureGates != [])
1033 "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.controllerManager.featureGates}"} \
1034 ${optionalString cfg.verbose "--v=6"} \
1035 ${optionalString cfg.verbose "--log-flush-frequency=1s"} \
1036 ${cfg.controllerManager.extraOpts}
1037 '';
1038 WorkingDirectory = cfg.dataDir;
1039 User = "kubernetes";
1040 Group = "kubernetes";
1041 };
1042 path = cfg.path;
1043 };
1044
1045 services.kubernetes.controllerManager.kubeconfig = kubeConfigDefaults;
1046 })
1047
1048 (mkIf cfg.proxy.enable {
1049 systemd.services.kube-proxy = {
1050 description = "Kubernetes Proxy Service";
1051 wantedBy = [ "kubernetes.target" ];
1052 after = [ "kube-apiserver.service" ];
1053 path = [pkgs.iptables pkgs.conntrack_tools];
1054 serviceConfig = {
1055 Slice = "kubernetes.slice";
1056 ExecStart = ''${cfg.package}/bin/kube-proxy \
1057 --kubeconfig=${mkKubeConfig "kube-proxy" cfg.proxy.kubeconfig} \
1058 --bind-address=${cfg.proxy.address} \
1059 ${optionalString (cfg.proxy.featureGates != [])
1060 "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.proxy.featureGates}"} \
1061 ${optionalString cfg.verbose "--v=6"} \
1062 ${optionalString cfg.verbose "--log-flush-frequency=1s"} \
1063 ${optionalString (cfg.clusterCidr!=null)
1064 "--cluster-cidr=${cfg.clusterCidr}"} \
1065 ${cfg.proxy.extraOpts}
1066 '';
1067 WorkingDirectory = cfg.dataDir;
1068 };
1069 };
1070
1071 # kube-proxy needs iptables
1072 networking.firewall.enable = mkDefault true;
1073
1074 services.kubernetes.proxy.kubeconfig = kubeConfigDefaults;
1075 })
1076
1077 (mkIf (any (el: el == "master") cfg.roles) {
1078 virtualisation.docker.enable = mkDefault true;
1079 services.kubernetes.kubelet.enable = mkDefault true;
1080 services.kubernetes.kubelet.allowPrivileged = mkDefault true;
1081 services.kubernetes.kubelet.applyManifests = mkDefault true;
1082 services.kubernetes.apiserver.enable = mkDefault true;
1083 services.kubernetes.scheduler.enable = mkDefault true;
1084 services.kubernetes.controllerManager.enable = mkDefault true;
1085 services.etcd.enable = mkDefault (cfg.etcd.servers == ["http://127.0.0.1:2379"]);
1086 services.kubernetes.addonManager.enable = mkDefault true;
1087 services.kubernetes.proxy.enable = mkDefault true;
1088 })
1089
1090 # if this node is only a master make it unschedulable by default
1091 (mkIf (all (el: el == "master") cfg.roles) {
1092 services.kubernetes.kubelet.unschedulable = mkDefault true;
1093 })
1094
1095 (mkIf (any (el: el == "node") cfg.roles) {
1096 virtualisation.docker = {
1097 enable = mkDefault true;
1098
1099 # kubernetes needs access to logs
1100 logDriver = mkDefault "json-file";
1101
1102 # iptables must be disabled for kubernetes
1103 extraOptions = "--iptables=false --ip-masq=false";
1104 };
1105
1106 services.kubernetes.kubelet.enable = mkDefault true;
1107 services.kubernetes.proxy.enable = mkDefault true;
1108 })
1109
1110 (mkIf cfg.addonManager.enable {
1111 environment.etc."kubernetes/addons".source = "${addons}/";
1112
1113 systemd.services.kube-addon-manager = {
1114 description = "Kubernetes addon manager";
1115 wantedBy = [ "kubernetes.target" ];
1116 after = [ "kube-apiserver.service" ];
1117 environment.ADDON_PATH = "/etc/kubernetes/addons/";
1118 path = [ pkgs.gawk ];
1119 serviceConfig = {
1120 Slice = "kubernetes.slice";
1121 ExecStart = "${cfg.package}/bin/kube-addons";
1122 WorkingDirectory = cfg.dataDir;
1123 User = "kubernetes";
1124 Group = "kubernetes";
1125 };
1126 };
1127 })
1128
1129 (mkIf (
1130 cfg.apiserver.enable ||
1131 cfg.scheduler.enable ||
1132 cfg.controllerManager.enable ||
1133 cfg.kubelet.enable ||
1134 cfg.proxy.enable
1135 ) {
1136 systemd.targets.kubernetes = {
1137 description = "Kubernetes";
1138 wantedBy = [ "multi-user.target" ];
1139 };
1140
1141 systemd.tmpfiles.rules = [
1142 "d /opt/cni/bin 0755 root root -"
1143 "d /var/run/kubernetes 0755 kubernetes kubernetes -"
1144 "d /var/lib/kubernetes 0755 kubernetes kubernetes -"
1145 ];
1146
1147 environment.systemPackages = [ cfg.package ];
1148 users.users = singleton {
1149 name = "kubernetes";
1150 uid = config.ids.uids.kubernetes;
1151 description = "Kubernetes user";
1152 extraGroups = [ "docker" ];
1153 group = "kubernetes";
1154 home = cfg.dataDir;
1155 createHome = true;
1156 };
1157 users.groups.kubernetes.gid = config.ids.gids.kubernetes;
1158
1159 # dns addon is enabled by default
1160 services.kubernetes.addons.dns.enable = mkDefault true;
1161 })
1162
1163 (mkIf cfg.flannel.enable {
1164 services.flannel = {
1165 enable = mkDefault true;
1166 network = mkDefault cfg.clusterCidr;
1167 etcd = mkDefault {
1168 endpoints = cfg.etcd.servers;
1169 inherit (cfg.etcd) caFile certFile keyFile;
1170 };
1171 };
1172
1173 services.kubernetes.kubelet = {
1174 networkPlugin = mkDefault "cni";
1175 cni.config = mkDefault [{
1176 name = "mynet";
1177 type = "flannel";
1178 delegate = {
1179 isDefaultGateway = true;
1180 bridge = "docker0";
1181 };
1182 }];
1183 };
1184
1185 systemd.services."mk-docker-opts" = {
1186 description = "Pre-Docker Actions";
1187 wantedBy = [ "flannel.service" ];
1188 before = [ "docker.service" ];
1189 after = [ "flannel.service" ];
1190 path = [ pkgs.gawk pkgs.gnugrep ];
1191 script = ''
1192 mkdir -p /run/flannel
1193 ${mkDockerOpts}/mk-docker-opts -d /run/flannel/docker
1194 '';
1195 serviceConfig.Type = "oneshot";
1196 };
1197 systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/docker";
1198
1199 # read environment variables generated by mk-docker-opts
1200 virtualisation.docker.extraOptions = "$DOCKER_OPTS";
1201
1202 networking.firewall.allowedUDPPorts = [
1203 8285 # flannel udp
1204 8472 # flannel vxlan
1205 ];
1206 })
1207 ];
1208}