pyrox.dev
1{ config, lib, pkgs, ... }:
2
3with lib;
4
5let
6 cfg = config.services.kubernetes;
7
8 skipAttrs = attrs: map (filterAttrs (k: v: k != "enable"))
9 (filter (v: !(hasAttr "enable" v) || v.enable) attrs);
10
11 infraContainer = pkgs.dockerTools.buildImage {
12 name = "pause";
13 tag = "latest";
14 contents = cfg.package.pause;
15 config.Cmd = "/bin/pause";
16 };
17
18 mkKubeConfig = name: cfg: pkgs.writeText "${name}-kubeconfig" (builtins.toJSON {
19 apiVersion = "v1";
20 kind = "Config";
21 clusters = [{
22 name = "local";
23 cluster.certificate-authority = cfg.caFile;
24 cluster.server = cfg.server;
25 }];
26 users = [{
27 name = "kubelet";
28 user = {
29 client-certificate = cfg.certFile;
30 client-key = cfg.keyFile;
31 };
32 }];
33 contexts = [{
34 context = {
35 cluster = "local";
36 user = "kubelet";
37 };
38 current-context = "kubelet-context";
39 }];
40 });
41
42 mkKubeConfigOptions = prefix: {
43 server = mkOption {
44 description = "${prefix} kube-apiserver server address.";
45 default = "http://${cfg.apiserver.address}:${toString cfg.apiserver.port}";
46 type = types.str;
47 };
48
49 caFile = mkOption {
50 description = "${prefix} certificate authrority file used to connect to kube-apiserver.";
51 type = types.nullOr types.path;
52 default = cfg.caFile;
53 };
54
55 certFile = mkOption {
56 description = "${prefix} client certificate file used to connect to kube-apiserver.";
57 type = types.nullOr types.path;
58 default = null;
59 };
60
61 keyFile = mkOption {
62 description = "${prefix} client key file used to connect to kube-apiserver.";
63 type = types.nullOr types.path;
64 default = null;
65 };
66 };
67
68 kubeConfigDefaults = {
69 server = mkDefault cfg.kubeconfig.server;
70 caFile = mkDefault cfg.kubeconfig.caFile;
71 certFile = mkDefault cfg.kubeconfig.certFile;
72 keyFile = mkDefault cfg.kubeconfig.keyFile;
73 };
74
75 cniConfig = pkgs.buildEnv {
76 name = "kubernetes-cni-config";
77 paths = imap (i: entry:
78 pkgs.writeTextDir "${toString (10+i)}-${entry.type}.conf" (builtins.toJSON entry)
79 ) cfg.kubelet.cni.config;
80 };
81
82 manifests = pkgs.buildEnv {
83 name = "kubernetes-manifests";
84 paths = mapAttrsToList (name: manifest:
85 pkgs.writeTextDir "${name}.json" (builtins.toJSON manifest)
86 ) cfg.kubelet.manifests;
87 };
88
89 addons = pkgs.runCommand "kubernetes-addons" { } ''
90 mkdir -p $out
91 # since we are mounting the addons to the addon manager, they need to be copied
92 ${concatMapStringsSep ";" (a: "cp -v ${a}/* $out/") (mapAttrsToList (name: addon:
93 pkgs.writeTextDir "${name}.json" (builtins.toJSON addon)
94 ) (cfg.addonManager.addons))}
95 '';
96
97 taintOptions = { name, ... }: {
98 options = {
99 key = mkOption {
100 description = "Key of taint.";
101 default = name;
102 type = types.str;
103 };
104 value = mkOption {
105 description = "Value of taint.";
106 type = types.str;
107 };
108 effect = mkOption {
109 description = "Effect of taint.";
110 example = "NoSchedule";
111 type = types.enum ["NoSchedule" "PreferNoSchedule" "NoExecute"];
112 };
113 };
114 };
115
116 taints = concatMapStringsSep "," (v: "${v.key}=${v.value}:${v.effect}") (mapAttrsToList (n: v: v) cfg.kubelet.taints);
117
118 # needed for flannel to pass options to docker
119 mkDockerOpts = pkgs.runCommand "mk-docker-opts" {
120 buildInputs = [ pkgs.makeWrapper ];
121 } ''
122 mkdir -p $out
123 cp ${pkgs.kubernetes.src}/cluster/centos/node/bin/mk-docker-opts.sh $out/mk-docker-opts.sh
124
125 # bashInteractive needed for `compgen`
126 makeWrapper ${pkgs.bashInteractive}/bin/bash $out/mk-docker-opts --add-flags "$out/mk-docker-opts.sh"
127 '';
128in {
129
130 ###### interface
131
132 options.services.kubernetes = {
133 roles = mkOption {
134 description = ''
135 Kubernetes role that this machine should take.
136
137 Master role will enable etcd, apiserver, scheduler and controller manager
138 services. Node role will enable etcd, docker, kubelet and proxy services.
139 '';
140 default = [];
141 type = types.listOf (types.enum ["master" "node"]);
142 };
143
144 package = mkOption {
145 description = "Kubernetes package to use.";
146 type = types.package;
147 default = pkgs.kubernetes;
148 defaultText = "pkgs.kubernetes";
149 };
150
151 verbose = mkOption {
152 description = "Kubernetes enable verbose mode for debugging.";
153 default = false;
154 type = types.bool;
155 };
156
157 etcd = {
158 servers = mkOption {
159 description = "List of etcd servers. By default etcd is started, except if this option is changed.";
160 default = ["http://127.0.0.1:2379"];
161 type = types.listOf types.str;
162 };
163
164 keyFile = mkOption {
165 description = "Etcd key file.";
166 default = null;
167 type = types.nullOr types.path;
168 };
169
170 certFile = mkOption {
171 description = "Etcd cert file.";
172 default = null;
173 type = types.nullOr types.path;
174 };
175
176 caFile = mkOption {
177 description = "Etcd ca file.";
178 default = cfg.caFile;
179 type = types.nullOr types.path;
180 };
181 };
182
183 kubeconfig = mkKubeConfigOptions "Default kubeconfig";
184
185 caFile = mkOption {
186 description = "Default kubernetes certificate authority";
187 type = types.nullOr types.path;
188 default = null;
189 };
190
191 dataDir = mkOption {
192 description = "Kubernetes root directory for managing kubelet files.";
193 default = "/var/lib/kubernetes";
194 type = types.path;
195 };
196
197 featureGates = mkOption {
198 description = "List set of feature gates";
199 default = [];
200 type = types.listOf types.str;
201 };
202
203 apiserver = {
204 enable = mkOption {
205 description = "Whether to enable Kubernetes apiserver.";
206 default = false;
207 type = types.bool;
208 };
209
210 featureGates = mkOption {
211 description = "List set of feature gates";
212 default = cfg.featureGates;
213 type = types.listOf types.str;
214 };
215
216 address = mkOption {
217 description = "Kubernetes apiserver listening address.";
218 default = "127.0.0.1";
219 type = types.str;
220 };
221
222 publicAddress = mkOption {
223 description = ''
224 Kubernetes apiserver public listening address used for read only and
225 secure port.
226 '';
227 default = cfg.apiserver.address;
228 type = types.str;
229 };
230
231 advertiseAddress = mkOption {
232 description = ''
233 Kubernetes apiserver IP address on which to advertise the apiserver
234 to members of the cluster. This address must be reachable by the rest
235 of the cluster.
236 '';
237 default = null;
238 type = types.nullOr types.str;
239 };
240
241 storageBackend = mkOption {
242 description = ''
243 Kubernetes apiserver storage backend.
244 '';
245 default = "etcd3";
246 type = types.enum ["etcd2" "etcd3"];
247 };
248
249 port = mkOption {
250 description = "Kubernetes apiserver listening port.";
251 default = 8080;
252 type = types.int;
253 };
254
255 securePort = mkOption {
256 description = "Kubernetes apiserver secure port.";
257 default = 443;
258 type = types.int;
259 };
260
261 tlsCertFile = mkOption {
262 description = "Kubernetes apiserver certificate file.";
263 default = null;
264 type = types.nullOr types.path;
265 };
266
267 tlsKeyFile = mkOption {
268 description = "Kubernetes apiserver private key file.";
269 default = null;
270 type = types.nullOr types.path;
271 };
272
273 clientCaFile = mkOption {
274 description = "Kubernetes apiserver CA file for client auth.";
275 default = cfg.caFile;
276 type = types.nullOr types.path;
277 };
278
279 tokenAuthFile = mkOption {
280 description = ''
281 Kubernetes apiserver token authentication file. See
282 <link xlink:href="http://kubernetes.io/docs/admin/authentication.html"/>
283 '';
284 default = null;
285 type = types.nullOr types.path;
286 };
287
288 basicAuthFile = mkOption {
289 description = ''
290 Kubernetes apiserver basic authentication file. See
291 <link xlink:href="http://kubernetes.io/docs/admin/authentication.html"/>
292 '';
293 default = pkgs.writeText "users" ''
294 kubernetes,admin,0
295 '';
296 type = types.nullOr types.path;
297 };
298
299 authorizationMode = mkOption {
300 description = ''
301 Kubernetes apiserver authorization mode (AlwaysAllow/AlwaysDeny/ABAC/RBAC). See
302 <link xlink:href="http://kubernetes.io/docs/admin/authorization.html"/>
303 '';
304 default = ["RBAC" "Node"];
305 type = types.listOf (types.enum ["AlwaysAllow" "AlwaysDeny" "ABAC" "RBAC" "Node"]);
306 };
307
308 authorizationPolicy = mkOption {
309 description = ''
310 Kubernetes apiserver authorization policy file. See
311 <link xlink:href="http://kubernetes.io/docs/admin/authorization.html"/>
312 '';
313 default = [];
314 type = types.listOf types.attrs;
315 };
316
317 allowPrivileged = mkOption {
318 description = "Whether to allow privileged containers on Kubernetes.";
319 default = true;
320 type = types.bool;
321 };
322
323 serviceClusterIpRange = mkOption {
324 description = ''
325 A CIDR notation IP range from which to assign service cluster IPs.
326 This must not overlap with any IP ranges assigned to nodes for pods.
327 '';
328 default = "10.0.0.0/24";
329 type = types.str;
330 };
331
332 runtimeConfig = mkOption {
333 description = ''
334 Api runtime configuration. See
335 <link xlink:href="http://kubernetes.io/docs/admin/cluster-management.html"/>
336 '';
337 default = "authentication.k8s.io/v1beta1=true";
338 example = "api/all=false,api/v1=true";
339 type = types.str;
340 };
341
342 admissionControl = mkOption {
343 description = ''
344 Kubernetes admission control plugins to use. See
345 <link xlink:href="http://kubernetes.io/docs/admin/admission-controllers/"/>
346 '';
347 default = ["NamespaceLifecycle" "LimitRanger" "ServiceAccount" "ResourceQuota" "DefaultStorageClass" "DefaultTolerationSeconds" "NodeRestriction"];
348 example = [
349 "NamespaceLifecycle" "NamespaceExists" "LimitRanger"
350 "SecurityContextDeny" "ServiceAccount" "ResourceQuota"
351 "PodSecurityPolicy" "NodeRestriction" "DefaultStorageClass"
352 ];
353 type = types.listOf types.str;
354 };
355
356 serviceAccountKeyFile = mkOption {
357 description = ''
358 Kubernetes apiserver PEM-encoded x509 RSA private or public key file,
359 used to verify ServiceAccount tokens. By default tls private key file
360 is used.
361 '';
362 default = null;
363 type = types.nullOr types.path;
364 };
365
366 kubeletClientCaFile = mkOption {
367 description = "Path to a cert file for connecting to kubelet.";
368 default = cfg.caFile;
369 type = types.nullOr types.path;
370 };
371
372 kubeletClientCertFile = mkOption {
373 description = "Client certificate to use for connections to kubelet.";
374 default = null;
375 type = types.nullOr types.path;
376 };
377
378 kubeletClientKeyFile = mkOption {
379 description = "Key to use for connections to kubelet.";
380 default = null;
381 type = types.nullOr types.path;
382 };
383
384 kubeletHttps = mkOption {
385 description = "Whether to use https for connections to kubelet.";
386 default = true;
387 type = types.bool;
388 };
389
390 extraOpts = mkOption {
391 description = "Kubernetes apiserver extra command line options.";
392 default = "";
393 type = types.str;
394 };
395 };
396
397 scheduler = {
398 enable = mkOption {
399 description = "Whether to enable Kubernetes scheduler.";
400 default = false;
401 type = types.bool;
402 };
403
404 featureGates = mkOption {
405 description = "List set of feature gates";
406 default = cfg.featureGates;
407 type = types.listOf types.str;
408 };
409
410 address = mkOption {
411 description = "Kubernetes scheduler listening address.";
412 default = "127.0.0.1";
413 type = types.str;
414 };
415
416 port = mkOption {
417 description = "Kubernetes scheduler listening port.";
418 default = 10251;
419 type = types.int;
420 };
421
422 leaderElect = mkOption {
423 description = "Whether to start leader election before executing main loop.";
424 type = types.bool;
425 default = true;
426 };
427
428 kubeconfig = mkKubeConfigOptions "Kubernetes scheduler";
429
430 extraOpts = mkOption {
431 description = "Kubernetes scheduler extra command line options.";
432 default = "";
433 type = types.str;
434 };
435 };
436
437 controllerManager = {
438 enable = mkOption {
439 description = "Whether to enable Kubernetes controller manager.";
440 default = false;
441 type = types.bool;
442 };
443
444 featureGates = mkOption {
445 description = "List set of feature gates";
446 default = cfg.featureGates;
447 type = types.listOf types.str;
448 };
449
450 address = mkOption {
451 description = "Kubernetes controller manager listening address.";
452 default = "127.0.0.1";
453 type = types.str;
454 };
455
456 port = mkOption {
457 description = "Kubernetes controller manager listening port.";
458 default = 10252;
459 type = types.int;
460 };
461
462 leaderElect = mkOption {
463 description = "Whether to start leader election before executing main loop.";
464 type = types.bool;
465 default = true;
466 };
467
468 serviceAccountKeyFile = mkOption {
469 description = ''
470 Kubernetes controller manager PEM-encoded private RSA key file used to
471 sign service account tokens
472 '';
473 default = null;
474 type = types.nullOr types.path;
475 };
476
477 rootCaFile = mkOption {
478 description = ''
479 Kubernetes controller manager certificate authority file included in
480 service account's token secret.
481 '';
482 default = cfg.caFile;
483 type = types.nullOr types.path;
484 };
485
486 kubeconfig = mkKubeConfigOptions "Kubernetes controller manager";
487
488 extraOpts = mkOption {
489 description = "Kubernetes controller manager extra command line options.";
490 default = "";
491 type = types.str;
492 };
493 };
494
495 kubelet = {
496 enable = mkOption {
497 description = "Whether to enable Kubernetes kubelet.";
498 default = false;
499 type = types.bool;
500 };
501
502 featureGates = mkOption {
503 description = "List set of feature gates";
504 default = cfg.featureGates;
505 type = types.listOf types.str;
506 };
507
508 seedDockerImages = mkOption {
509 description = "List of docker images to preload on system";
510 default = [];
511 type = types.listOf types.package;
512 };
513
514 registerNode = mkOption {
515 description = "Whether to auto register kubelet with API server.";
516 default = true;
517 type = types.bool;
518 };
519
520 address = mkOption {
521 description = "Kubernetes kubelet info server listening address.";
522 default = "0.0.0.0";
523 type = types.str;
524 };
525
526 port = mkOption {
527 description = "Kubernetes kubelet info server listening port.";
528 default = 10250;
529 type = types.int;
530 };
531
532 tlsCertFile = mkOption {
533 description = "File containing x509 Certificate for HTTPS.";
534 default = null;
535 type = types.nullOr types.path;
536 };
537
538 tlsKeyFile = mkOption {
539 description = "File containing x509 private key matching tlsCertFile.";
540 default = null;
541 type = types.nullOr types.path;
542 };
543
544 clientCaFile = mkOption {
545 description = "Kubernetes apiserver CA file for client authentication.";
546 default = cfg.caFile;
547 type = types.nullOr types.path;
548 };
549
550 healthz = {
551 bind = mkOption {
552 description = "Kubernetes kubelet healthz listening address.";
553 default = "127.0.0.1";
554 type = types.str;
555 };
556
557 port = mkOption {
558 description = "Kubernetes kubelet healthz port.";
559 default = 10248;
560 type = types.int;
561 };
562 };
563
564 hostname = mkOption {
565 description = "Kubernetes kubelet hostname override.";
566 default = config.networking.hostName;
567 type = types.str;
568 };
569
570 allowPrivileged = mkOption {
571 description = "Whether to allow Kubernetes containers to request privileged mode.";
572 default = true;
573 type = types.bool;
574 };
575
576 cadvisorPort = mkOption {
577 description = "Kubernetes kubelet local cadvisor port.";
578 default = 4194;
579 type = types.int;
580 };
581
582 clusterDns = mkOption {
583 description = "Use alternative DNS.";
584 default = "10.1.0.1";
585 type = types.str;
586 };
587
588 clusterDomain = mkOption {
589 description = "Use alternative domain.";
590 default = config.services.kubernetes.addons.dns.clusterDomain;
591 type = types.str;
592 };
593
594 networkPlugin = mkOption {
595 description = "Network plugin to use by Kubernetes.";
596 type = types.nullOr (types.enum ["cni" "kubenet"]);
597 default = "kubenet";
598 };
599
600 cni = {
601 packages = mkOption {
602 description = "List of network plugin packages to install.";
603 type = types.listOf types.package;
604 default = [];
605 };
606
607 config = mkOption {
608 description = "Kubernetes CNI configuration.";
609 type = types.listOf types.attrs;
610 default = [];
611 example = literalExample ''
612 [{
613 "cniVersion": "0.2.0",
614 "name": "mynet",
615 "type": "bridge",
616 "bridge": "cni0",
617 "isGateway": true,
618 "ipMasq": true,
619 "ipam": {
620 "type": "host-local",
621 "subnet": "10.22.0.0/16",
622 "routes": [
623 { "dst": "0.0.0.0/0" }
624 ]
625 }
626 } {
627 "cniVersion": "0.2.0",
628 "type": "loopback"
629 }]
630 '';
631 };
632 };
633
634 manifests = mkOption {
635 description = "List of manifests to bootstrap with kubelet (only pods can be created as manifest entry)";
636 type = types.attrsOf types.attrs;
637 default = {};
638 };
639
640 applyManifests = mkOption {
641 description = "Whether to apply manifests (this is true for master node).";
642 default = false;
643 type = types.bool;
644 };
645
646 unschedulable = mkOption {
647 description = "Whether to set node taint to unschedulable=true as it is the case of node that has only master role.";
648 default = false;
649 type = types.bool;
650 };
651
652 taints = mkOption {
653 description = "Node taints (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/).";
654 default = {};
655 type = types.attrsOf (types.submodule [ taintOptions ]);
656 };
657
658 nodeIp = mkOption {
659 description = "IP address of the node. If set, kubelet will use this IP address for the node.";
660 default = null;
661 type = types.nullOr types.str;
662 };
663
664 kubeconfig = mkKubeConfigOptions "Kubelet";
665
666 extraOpts = mkOption {
667 description = "Kubernetes kubelet extra command line options.";
668 default = "";
669 type = types.str;
670 };
671 };
672
673 proxy = {
674 enable = mkOption {
675 description = "Whether to enable Kubernetes proxy.";
676 default = false;
677 type = types.bool;
678 };
679
680 featureGates = mkOption {
681 description = "List set of feature gates";
682 default = cfg.featureGates;
683 type = types.listOf types.str;
684 };
685
686 address = mkOption {
687 description = "Kubernetes proxy listening address.";
688 default = "0.0.0.0";
689 type = types.str;
690 };
691
692 kubeconfig = mkKubeConfigOptions "Kubernetes proxy";
693
694 extraOpts = mkOption {
695 description = "Kubernetes proxy extra command line options.";
696 default = "";
697 type = types.str;
698 };
699 };
700
701 addonManager = {
702 enable = mkOption {
703 description = "Whether to enable Kubernetes addon manager.";
704 default = false;
705 type = types.bool;
706 };
707
708 addons = mkOption {
709 description = "Kubernetes addons (any kind of Kubernetes resource can be an addon).";
710 default = { };
711 type = types.attrsOf (types.either types.attrs (types.listOf types.attrs));
712 example = literalExample ''
713 {
714 "my-service" = {
715 "apiVersion" = "v1";
716 "kind" = "Service";
717 "metadata" = {
718 "name" = "my-service";
719 "namespace" = "default";
720 };
721 "spec" = { ... };
722 };
723 }
724 // import <nixpkgs/nixos/modules/services/cluster/kubernetes/dashboard.nix> { cfg = config.services.kubernetes; };
725 '';
726 };
727 };
728
729 path = mkOption {
730 description = "Packages added to the services' PATH environment variable. Both the bin and sbin subdirectories of each package are added.";
731 type = types.listOf types.package;
732 default = [];
733 };
734
735 clusterCidr = mkOption {
736 description = "Kubernetes controller manager and proxy CIDR Range for Pods in cluster.";
737 default = "10.1.0.0/16";
738 type = types.str;
739 };
740
741 flannel.enable = mkOption {
742 description = "Whether to enable flannel networking";
743 default = false;
744 type = types.bool;
745 };
746
747 };
748
749 ###### implementation
750
751 config = mkMerge [
752 (mkIf cfg.kubelet.enable {
753 services.kubernetes.kubelet.seedDockerImages = [infraContainer];
754
755 systemd.services.kubelet-bootstrap = {
756 description = "Boostrap Kubelet";
757 wantedBy = ["kubernetes.target"];
758 after = ["docker.service" "network.target"];
759 path = with pkgs; [ docker ];
760 script = ''
761 ${concatMapStrings (img: ''
762 echo "Seeding docker image: ${img}"
763 docker load <${img}
764 '') cfg.kubelet.seedDockerImages}
765
766 rm /opt/cni/bin/* || true
767 ${concatMapStrings (package: ''
768 echo "Linking cni package: ${package}"
769 ln -fs ${package.plugins}/* /opt/cni/bin
770 '') cfg.kubelet.cni.packages}
771 '';
772 serviceConfig = {
773 Slice = "kubernetes.slice";
774 Type = "oneshot";
775 };
776 };
777
778 systemd.services.kubelet = {
779 description = "Kubernetes Kubelet Service";
780 wantedBy = [ "kubernetes.target" ];
781 after = [ "network.target" "docker.service" "kube-apiserver.service" "kubelet-bootstrap.service" ];
782 path = with pkgs; [ gitMinimal openssh docker utillinux iproute ethtool thin-provisioning-tools iptables socat ] ++ cfg.path;
783 serviceConfig = {
784 Slice = "kubernetes.slice";
785 ExecStart = ''${cfg.package}/bin/kubelet \
786 ${optionalString cfg.kubelet.applyManifests
787 "--pod-manifest-path=${manifests}"} \
788 ${optionalString (taints != "")
789 "--register-with-taints=${taints}"} \
790 --kubeconfig=${mkKubeConfig "kubelet" cfg.kubelet.kubeconfig} \
791 --require-kubeconfig \
792 --address=${cfg.kubelet.address} \
793 --port=${toString cfg.kubelet.port} \
794 --register-node=${boolToString cfg.kubelet.registerNode} \
795 ${optionalString (cfg.kubelet.tlsCertFile != null)
796 "--tls-cert-file=${cfg.kubelet.tlsCertFile}"} \
797 ${optionalString (cfg.kubelet.tlsKeyFile != null)
798 "--tls-private-key-file=${cfg.kubelet.tlsKeyFile}"} \
799 ${optionalString (cfg.kubelet.clientCaFile != null)
800 "--client-ca-file=${cfg.kubelet.clientCaFile}"} \
801 --authentication-token-webhook \
802 --authentication-token-webhook-cache-ttl="10s" \
803 --authorization-mode=Webhook \
804 --healthz-bind-address=${cfg.kubelet.healthz.bind} \
805 --healthz-port=${toString cfg.kubelet.healthz.port} \
806 --hostname-override=${cfg.kubelet.hostname} \
807 --allow-privileged=${boolToString cfg.kubelet.allowPrivileged} \
808 --root-dir=${cfg.dataDir} \
809 --cadvisor_port=${toString cfg.kubelet.cadvisorPort} \
810 ${optionalString (cfg.kubelet.clusterDns != "")
811 "--cluster-dns=${cfg.kubelet.clusterDns}"} \
812 ${optionalString (cfg.kubelet.clusterDomain != "")
813 "--cluster-domain=${cfg.kubelet.clusterDomain}"} \
814 --pod-infra-container-image=pause \
815 ${optionalString (cfg.kubelet.networkPlugin != null)
816 "--network-plugin=${cfg.kubelet.networkPlugin}"} \
817 --cni-conf-dir=${cniConfig} \
818 --hairpin-mode=hairpin-veth \
819 ${optionalString (cfg.kubelet.nodeIp != null)
820 "--node-ip=${cfg.kubelet.nodeIp}"} \
821 ${optionalString (cfg.kubelet.featureGates != [])
822 "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.kubelet.featureGates}"} \
823 ${optionalString cfg.verbose "--v=6 --log_flush_frequency=1s"} \
824 ${cfg.kubelet.extraOpts}
825 '';
826 WorkingDirectory = cfg.dataDir;
827 };
828 };
829
830 # Allways include cni plugins
831 services.kubernetes.kubelet.cni.packages = [pkgs.cni];
832
833 boot.kernelModules = ["br_netfilter"];
834
835 services.kubernetes.kubelet.kubeconfig = kubeConfigDefaults;
836 })
837
838 (mkIf (cfg.kubelet.applyManifests && cfg.kubelet.enable) {
839 environment.etc = mapAttrs' (name: manifest:
840 nameValuePair "kubernetes/manifests/${name}.json" {
841 text = builtins.toJSON manifest;
842 mode = "0755";
843 }
844 ) cfg.kubelet.manifests;
845 })
846
847 (mkIf (cfg.kubelet.unschedulable && cfg.kubelet.enable) {
848 services.kubernetes.kubelet.taints.unschedulable = {
849 value = "true";
850 effect = "NoSchedule";
851 };
852 })
853
854 (mkIf cfg.apiserver.enable {
855 systemd.services.kube-apiserver = {
856 description = "Kubernetes Kubelet Service";
857 wantedBy = [ "kubernetes.target" ];
858 after = [ "network.target" "docker.service" ];
859 serviceConfig = {
860 Slice = "kubernetes.slice";
861 ExecStart = ''${cfg.package}/bin/kube-apiserver \
862 --etcd-servers=${concatStringsSep "," cfg.etcd.servers} \
863 ${optionalString (cfg.etcd.caFile != null)
864 "--etcd-cafile=${cfg.etcd.caFile}"} \
865 ${optionalString (cfg.etcd.certFile != null)
866 "--etcd-certfile=${cfg.etcd.certFile}"} \
867 ${optionalString (cfg.etcd.keyFile != null)
868 "--etcd-keyfile=${cfg.etcd.keyFile}"} \
869 --insecure-port=${toString cfg.apiserver.port} \
870 --bind-address=0.0.0.0 \
871 ${optionalString (cfg.apiserver.advertiseAddress != null)
872 "--advertise-address=${cfg.apiserver.advertiseAddress}"} \
873 --allow-privileged=${boolToString cfg.apiserver.allowPrivileged}\
874 ${optionalString (cfg.apiserver.tlsCertFile != null)
875 "--tls-cert-file=${cfg.apiserver.tlsCertFile}"} \
876 ${optionalString (cfg.apiserver.tlsKeyFile != null)
877 "--tls-private-key-file=${cfg.apiserver.tlsKeyFile}"} \
878 ${optionalString (cfg.apiserver.tokenAuthFile != null)
879 "--token-auth-file=${cfg.apiserver.tokenAuthFile}"} \
880 ${optionalString (cfg.apiserver.basicAuthFile != null)
881 "--basic-auth-file=${cfg.apiserver.basicAuthFile}"} \
882 --kubelet-https=${if cfg.apiserver.kubeletHttps then "true" else "false"} \
883 ${optionalString (cfg.apiserver.kubeletClientCaFile != null)
884 "--kubelet-certificate-authority=${cfg.apiserver.kubeletClientCaFile}"} \
885 ${optionalString (cfg.apiserver.kubeletClientCertFile != null)
886 "--kubelet-client-certificate=${cfg.apiserver.kubeletClientCertFile}"} \
887 ${optionalString (cfg.apiserver.kubeletClientKeyFile != null)
888 "--kubelet-client-key=${cfg.apiserver.kubeletClientKeyFile}"} \
889 ${optionalString (cfg.apiserver.clientCaFile != null)
890 "--client-ca-file=${cfg.apiserver.clientCaFile}"} \
891 --authorization-mode=${concatStringsSep "," cfg.apiserver.authorizationMode} \
892 ${optionalString (elem "ABAC" cfg.apiserver.authorizationMode)
893 "--authorization-policy-file=${
894 pkgs.writeText "kube-auth-policy.jsonl"
895 (concatMapStringsSep "\n" (l: builtins.toJSON l) cfg.apiserver.authorizationPolicy)
896 }"
897 } \
898 --secure-port=${toString cfg.apiserver.securePort} \
899 --service-cluster-ip-range=${cfg.apiserver.serviceClusterIpRange} \
900 ${optionalString (cfg.apiserver.runtimeConfig != "")
901 "--runtime-config=${cfg.apiserver.runtimeConfig}"} \
902 --admission_control=${concatStringsSep "," cfg.apiserver.admissionControl} \
903 ${optionalString (cfg.apiserver.serviceAccountKeyFile!=null)
904 "--service-account-key-file=${cfg.apiserver.serviceAccountKeyFile}"} \
905 ${optionalString cfg.verbose "--v=6"} \
906 ${optionalString cfg.verbose "--log-flush-frequency=1s"} \
907 --storage-backend=${cfg.apiserver.storageBackend} \
908 ${optionalString (cfg.kubelet.featureGates != [])
909 "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.kubelet.featureGates}"} \
910 ${cfg.apiserver.extraOpts}
911 '';
912 WorkingDirectory = cfg.dataDir;
913 User = "kubernetes";
914 Group = "kubernetes";
915 AmbientCapabilities = "cap_net_bind_service";
916 Restart = "on-failure";
917 RestartSec = 5;
918 };
919 };
920 })
921
922 (mkIf cfg.scheduler.enable {
923 systemd.services.kube-scheduler = {
924 description = "Kubernetes Scheduler Service";
925 wantedBy = [ "kubernetes.target" ];
926 after = [ "kube-apiserver.service" ];
927 serviceConfig = {
928 Slice = "kubernetes.slice";
929 ExecStart = ''${cfg.package}/bin/kube-scheduler \
930 --address=${cfg.scheduler.address} \
931 --port=${toString cfg.scheduler.port} \
932 --leader-elect=${boolToString cfg.scheduler.leaderElect} \
933 --kubeconfig=${mkKubeConfig "kube-scheduler" cfg.scheduler.kubeconfig} \
934 ${optionalString cfg.verbose "--v=6"} \
935 ${optionalString cfg.verbose "--log-flush-frequency=1s"} \
936 ${optionalString (cfg.scheduler.featureGates != [])
937 "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.scheduler.featureGates}"} \
938 ${cfg.scheduler.extraOpts}
939 '';
940 WorkingDirectory = cfg.dataDir;
941 User = "kubernetes";
942 Group = "kubernetes";
943 };
944 };
945
946 services.kubernetes.scheduler.kubeconfig = kubeConfigDefaults;
947 })
948
949 (mkIf cfg.controllerManager.enable {
950 systemd.services.kube-controller-manager = {
951 description = "Kubernetes Controller Manager Service";
952 wantedBy = [ "kubernetes.target" ];
953 after = [ "kube-apiserver.service" ];
954 serviceConfig = {
955 RestartSec = "30s";
956 Restart = "on-failure";
957 Slice = "kubernetes.slice";
958 ExecStart = ''${cfg.package}/bin/kube-controller-manager \
959 --address=${cfg.controllerManager.address} \
960 --port=${toString cfg.controllerManager.port} \
961 --kubeconfig=${mkKubeConfig "kube-controller-manager" cfg.controllerManager.kubeconfig} \
962 --leader-elect=${boolToString cfg.controllerManager.leaderElect} \
963 ${if (cfg.controllerManager.serviceAccountKeyFile!=null)
964 then "--service-account-private-key-file=${cfg.controllerManager.serviceAccountKeyFile}"
965 else "--service-account-private-key-file=/var/run/kubernetes/apiserver.key"} \
966 ${if (cfg.controllerManager.rootCaFile!=null)
967 then "--root-ca-file=${cfg.controllerManager.rootCaFile}"
968 else "--root-ca-file=/var/run/kubernetes/apiserver.crt"} \
969 ${optionalString (cfg.clusterCidr!=null)
970 "--cluster-cidr=${cfg.clusterCidr}"} \
971 --allocate-node-cidrs=true \
972 ${optionalString (cfg.controllerManager.featureGates != [])
973 "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.controllerManager.featureGates}"} \
974 ${optionalString cfg.verbose "--v=6"} \
975 ${optionalString cfg.verbose "--log-flush-frequency=1s"} \
976 ${cfg.controllerManager.extraOpts}
977 '';
978 WorkingDirectory = cfg.dataDir;
979 User = "kubernetes";
980 Group = "kubernetes";
981 };
982 path = cfg.path;
983 };
984
985 services.kubernetes.controllerManager.kubeconfig = kubeConfigDefaults;
986 })
987
988 (mkIf cfg.proxy.enable {
989 systemd.services.kube-proxy = {
990 description = "Kubernetes Proxy Service";
991 wantedBy = [ "kubernetes.target" ];
992 after = [ "kube-apiserver.service" ];
993 path = [pkgs.iptables pkgs.conntrack_tools];
994 serviceConfig = {
995 Slice = "kubernetes.slice";
996 ExecStart = ''${cfg.package}/bin/kube-proxy \
997 --kubeconfig=${mkKubeConfig "kube-proxy" cfg.proxy.kubeconfig} \
998 --bind-address=${cfg.proxy.address} \
999 ${optionalString (cfg.proxy.featureGates != [])
1000 "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.proxy.featureGates}"} \
1001 ${optionalString cfg.verbose "--v=6"} \
1002 ${optionalString cfg.verbose "--log-flush-frequency=1s"} \
1003 ${optionalString (cfg.clusterCidr!=null)
1004 "--cluster-cidr=${cfg.clusterCidr}"} \
1005 ${cfg.proxy.extraOpts}
1006 '';
1007 WorkingDirectory = cfg.dataDir;
1008 };
1009 };
1010
1011 # kube-proxy needs iptables
1012 networking.firewall.enable = mkDefault true;
1013
1014 services.kubernetes.proxy.kubeconfig = kubeConfigDefaults;
1015 })
1016
1017 (mkIf (any (el: el == "master") cfg.roles) {
1018 virtualisation.docker.enable = mkDefault true;
1019 services.kubernetes.kubelet.enable = mkDefault true;
1020 services.kubernetes.kubelet.allowPrivileged = mkDefault true;
1021 services.kubernetes.kubelet.applyManifests = mkDefault true;
1022 services.kubernetes.apiserver.enable = mkDefault true;
1023 services.kubernetes.scheduler.enable = mkDefault true;
1024 services.kubernetes.controllerManager.enable = mkDefault true;
1025 services.etcd.enable = mkDefault (cfg.etcd.servers == ["http://127.0.0.1:2379"]);
1026 services.kubernetes.addonManager.enable = mkDefault true;
1027 services.kubernetes.proxy.enable = mkDefault true;
1028 })
1029
1030 # if this node is only a master make it unschedulable by default
1031 (mkIf (all (el: el == "master") cfg.roles) {
1032 services.kubernetes.kubelet.unschedulable = mkDefault true;
1033 })
1034
1035 (mkIf (any (el: el == "node") cfg.roles) {
1036 virtualisation.docker = {
1037 enable = mkDefault true;
1038
1039 # kubernetes needs access to logs
1040 logDriver = mkDefault "json-file";
1041
1042 # iptables must be disabled for kubernetes
1043 extraOptions = "--iptables=false --ip-masq=false";
1044 };
1045
1046 services.kubernetes.kubelet.enable = mkDefault true;
1047 services.kubernetes.proxy.enable = mkDefault true;
1048 })
1049
1050 (mkIf cfg.addonManager.enable {
1051 environment.etc."kubernetes/addons".source = "${addons}/";
1052
1053 systemd.services.kube-addon-manager = {
1054 description = "Kubernetes addon manager";
1055 wantedBy = [ "kubernetes.target" ];
1056 after = [ "kube-apiserver.service" ];
1057 environment.ADDON_PATH = "/etc/kubernetes/addons/";
1058 serviceConfig = {
1059 Slice = "kubernetes.slice";
1060 ExecStart = "${cfg.package}/bin/kube-addons";
1061 WorkingDirectory = cfg.dataDir;
1062 User = "kubernetes";
1063 Group = "kubernetes";
1064 };
1065 };
1066 })
1067
1068 (mkIf (
1069 cfg.apiserver.enable ||
1070 cfg.scheduler.enable ||
1071 cfg.controllerManager.enable ||
1072 cfg.kubelet.enable ||
1073 cfg.proxy.enable
1074 ) {
1075 systemd.targets.kubernetes = {
1076 description = "Kubernetes";
1077 wantedBy = [ "multi-user.target" ];
1078 };
1079
1080 systemd.tmpfiles.rules = [
1081 "d /opt/cni/bin 0755 root root -"
1082 "d /var/run/kubernetes 0755 kubernetes kubernetes -"
1083 "d /var/lib/kubernetes 0755 kubernetes kubernetes -"
1084 ];
1085
1086 environment.systemPackages = [ cfg.package ];
1087 users.extraUsers = singleton {
1088 name = "kubernetes";
1089 uid = config.ids.uids.kubernetes;
1090 description = "Kubernetes user";
1091 extraGroups = [ "docker" ];
1092 group = "kubernetes";
1093 home = cfg.dataDir;
1094 createHome = true;
1095 };
1096 users.extraGroups.kubernetes.gid = config.ids.gids.kubernetes;
1097
1098 # dns addon is enabled by default
1099 services.kubernetes.addons.dns.enable = mkDefault true;
1100 })
1101
1102 (mkIf cfg.flannel.enable {
1103 services.flannel = {
1104 enable = mkDefault true;
1105 network = mkDefault cfg.clusterCidr;
1106 etcd = mkDefault {
1107 endpoints = cfg.etcd.servers;
1108 inherit (cfg.etcd) caFile certFile keyFile;
1109 };
1110 };
1111
1112 services.kubernetes.kubelet = {
1113 networkPlugin = mkDefault "cni";
1114 cni.config = mkDefault [{
1115 name = "mynet";
1116 type = "flannel";
1117 delegate = {
1118 isDefaultGateway = true;
1119 bridge = "docker0";
1120 };
1121 }];
1122 };
1123
1124 systemd.services."mk-docker-opts" = {
1125 description = "Pre-Docker Actions";
1126 wantedBy = [ "flannel.service" ];
1127 before = [ "docker.service" ];
1128 after = [ "flannel.service" ];
1129 path = [ pkgs.gawk pkgs.gnugrep ];
1130 script = ''
1131 mkdir -p /run/flannel
1132 ${mkDockerOpts}/mk-docker-opts -d /run/flannel/docker
1133 '';
1134 serviceConfig.Type = "oneshot";
1135 };
1136 systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/docker";
1137
1138 # read environment variables generated by mk-docker-opts
1139 virtualisation.docker.extraOptions = "$DOCKER_OPTS";
1140
1141 networking.firewall.allowedUDPPorts = [
1142 8285 # flannel udp
1143 8472 # flannel vxlan
1144 ];
1145 })
1146 ];
1147}