pyrox.dev
1{ config, lib, pkgs, ... }:
2
3with lib;
4
5let
6 cfg = config.services.kubernetes;
7
8 skipAttrs = attrs: map (filterAttrs (k: v: k != "enable"))
9 (filter (v: !(hasAttr "enable" v) || v.enable) attrs);
10
11 infraContainer = pkgs.dockerTools.buildImage {
12 name = "pause";
13 tag = "latest";
14 contents = cfg.package.pause;
15 config.Cmd = "/bin/pause";
16 };
17
18 kubeconfig = pkgs.writeText "kubeconfig" (builtins.toJSON {
19 apiVersion = "v1";
20 kind = "Config";
21 clusters = [{
22 name = "local";
23 cluster.certificate-authority = cfg.kubeconfig.caFile;
24 cluster.server = cfg.kubeconfig.server;
25 }];
26 users = [{
27 name = "kubelet";
28 user = {
29 client-certificate = cfg.kubeconfig.certFile;
30 client-key = cfg.kubeconfig.keyFile;
31 };
32 }];
33 contexts = [{
34 context = {
35 cluster = "local";
36 user = "kubelet";
37 };
38 current-context = "kubelet-context";
39 }];
40 });
41
42 policyFile = pkgs.writeText "kube-policy"
43 (concatStringsSep "\n" (map builtins.toJSON cfg.apiserver.authorizationPolicy));
44
45 cniConfig = pkgs.buildEnv {
46 name = "kubernetes-cni-config";
47 paths = imap1 (i: entry:
48 pkgs.writeTextDir "${toString (10+i)}-${entry.type}.conf" (builtins.toJSON entry)
49 ) cfg.kubelet.cni.config;
50 };
51
52 manifests = pkgs.buildEnv {
53 name = "kubernetes-manifests";
54 paths = mapAttrsToList (name: manifest:
55 pkgs.writeTextDir "${name}.json" (builtins.toJSON manifest)
56 ) cfg.kubelet.manifests;
57 };
58
59in {
60
61 ###### interface
62
63 options.services.kubernetes = {
64 roles = mkOption {
65 description = ''
66 Kubernetes role that this machine should take.
67
68 Master role will enable etcd, apiserver, scheduler and controller manager
69 services. Node role will enable etcd, docker, kubelet and proxy services.
70 '';
71 default = [];
72 type = types.listOf (types.enum ["master" "node"]);
73 };
74
75 package = mkOption {
76 description = "Kubernetes package to use.";
77 type = types.package;
78 default = pkgs.kubernetes;
79 defaultText = "pkgs.kubernetes";
80 };
81
82 verbose = mkOption {
83 description = "Kubernetes enable verbose mode for debugging";
84 default = false;
85 type = types.bool;
86 };
87
88 etcd = {
89 servers = mkOption {
90 description = "List of etcd servers. By default etcd is started, except if this option is changed.";
91 default = ["http://127.0.0.1:2379"];
92 type = types.listOf types.str;
93 };
94
95 keyFile = mkOption {
96 description = "Etcd key file";
97 default = null;
98 type = types.nullOr types.path;
99 };
100
101 certFile = mkOption {
102 description = "Etcd cert file";
103 default = null;
104 type = types.nullOr types.path;
105 };
106
107 caFile = mkOption {
108 description = "Etcd ca file";
109 default = null;
110 type = types.nullOr types.path;
111 };
112 };
113
114 kubeconfig = {
115 server = mkOption {
116 description = "Kubernetes apiserver server address";
117 default = "http://${cfg.apiserver.address}:${toString cfg.apiserver.port}";
118 type = types.str;
119 };
120
121 caFile = mkOption {
122 description = "Certificate authrority file to use to connect to kuberentes apiserver";
123 type = types.nullOr types.path;
124 default = null;
125 };
126
127 certFile = mkOption {
128 description = "Client certificate file to use to connect to kubernetes";
129 type = types.nullOr types.path;
130 default = null;
131 };
132
133 keyFile = mkOption {
134 description = "Client key file to use to connect to kubernetes";
135 type = types.nullOr types.path;
136 default = null;
137 };
138 };
139
140 dataDir = mkOption {
141 description = "Kubernetes root directory for managing kubelet files.";
142 default = "/var/lib/kubernetes";
143 type = types.path;
144 };
145
146 apiserver = {
147 enable = mkOption {
148 description = "Whether to enable kubernetes apiserver.";
149 default = false;
150 type = types.bool;
151 };
152
153 address = mkOption {
154 description = "Kubernetes apiserver listening address.";
155 default = "127.0.0.1";
156 type = types.str;
157 };
158
159 publicAddress = mkOption {
160 description = ''
161 Kubernetes apiserver public listening address used for read only and
162 secure port.
163 '';
164 default = cfg.apiserver.address;
165 type = types.str;
166 };
167
168 advertiseAddress = mkOption {
169 description = ''
170 Kubernetes apiserver IP address on which to advertise the apiserver
171 to members of the cluster. This address must be reachable by the rest
172 of the cluster.
173 '';
174 default = null;
175 type = types.nullOr types.str;
176 };
177
178 port = mkOption {
179 description = "Kubernetes apiserver listening port.";
180 default = 8080;
181 type = types.int;
182 };
183
184 securePort = mkOption {
185 description = "Kubernetes apiserver secure port.";
186 default = 443;
187 type = types.int;
188 };
189
190 tlsCertFile = mkOption {
191 description = "Kubernetes apiserver certificate file.";
192 default = null;
193 type = types.nullOr types.path;
194 };
195
196 tlsKeyFile = mkOption {
197 description = "Kubernetes apiserver private key file.";
198 default = null;
199 type = types.nullOr types.path;
200 };
201
202 clientCaFile = mkOption {
203 description = "Kubernetes apiserver CA file for client auth.";
204 default = null;
205 type = types.nullOr types.path;
206 };
207
208 tokenAuth = mkOption {
209 description = ''
210 Kubernetes apiserver token authentication file. See
211 <link xlink:href="http://kubernetes.io/docs/admin/authentication.html"/>
212 '';
213 default = null;
214 example = ''token,user,uid,"group1,group2,group3"'';
215 type = types.nullOr types.lines;
216 };
217
218 authorizationMode = mkOption {
219 description = ''
220 Kubernetes apiserver authorization mode (AlwaysAllow/AlwaysDeny/ABAC). See
221 <link xlink:href="http://kubernetes.io/v1.0/docs/admin/authorization.html"/>
222 '';
223 default = "AlwaysAllow";
224 type = types.enum ["AlwaysAllow" "AlwaysDeny" "ABAC"];
225 };
226
227 authorizationPolicy = mkOption {
228 description = ''
229 Kubernetes apiserver authorization policy file. See
230 <link xlink:href="http://kubernetes.io/v1.0/docs/admin/authorization.html"/>
231 '';
232 default = [];
233 example = literalExample ''
234 [
235 {user = "admin";}
236 {user = "scheduler"; readonly = true; kind= "pods";}
237 {user = "scheduler"; kind = "bindings";}
238 {user = "kubelet"; readonly = true; kind = "bindings";}
239 {user = "kubelet"; kind = "events";}
240 {user= "alice"; ns = "projectCaribou";}
241 {user = "bob"; readonly = true; ns = "projectCaribou";}
242 ]
243 '';
244 type = types.listOf types.attrs;
245 };
246
247 allowPrivileged = mkOption {
248 description = "Whether to allow privileged containers on kubernetes.";
249 default = true;
250 type = types.bool;
251 };
252
253 portalNet = mkOption {
254 description = "Kubernetes CIDR notation IP range from which to assign portal IPs";
255 default = "10.10.10.10/24";
256 type = types.str;
257 };
258
259 runtimeConfig = mkOption {
260 description = ''
261 Api runtime configuration. See
262 <link xlink:href="http://kubernetes.io/v1.0/docs/admin/cluster-management.html"/>
263 '';
264 default = "";
265 example = "api/all=false,api/v1=true";
266 type = types.str;
267 };
268
269 admissionControl = mkOption {
270 description = ''
271 Kubernetes admission control plugins to use. See
272 <link xlink:href="http://kubernetes.io/docs/admin/admission-controllers/"/>
273 '';
274 default = ["NamespaceLifecycle" "LimitRanger" "ServiceAccount" "ResourceQuota"];
275 example = [
276 "NamespaceLifecycle" "NamespaceExists" "LimitRanger"
277 "SecurityContextDeny" "ServiceAccount" "ResourceQuota"
278 ];
279 type = types.listOf types.str;
280 };
281
282 serviceAccountKeyFile = mkOption {
283 description = ''
284 Kubernetes apiserver PEM-encoded x509 RSA private or public key file,
285 used to verify ServiceAccount tokens. By default tls private key file
286 is used.
287 '';
288 default = null;
289 type = types.nullOr types.path;
290 };
291
292 kubeletClientCaFile = mkOption {
293 description = "Path to a cert file for connecting to kubelet";
294 default = null;
295 type = types.nullOr types.path;
296 };
297
298 kubeletClientCertFile = mkOption {
299 description = "Client certificate to use for connections to kubelet";
300 default = null;
301 type = types.nullOr types.path;
302 };
303
304 kubeletClientKeyFile = mkOption {
305 description = "Key to use for connections to kubelet";
306 default = null;
307 type = types.nullOr types.path;
308 };
309
310 kubeletHttps = mkOption {
311 description = "Whether to use https for connections to kubelet";
312 default = true;
313 type = types.bool;
314 };
315
316 extraOpts = mkOption {
317 description = "Kubernetes apiserver extra command line options.";
318 default = "";
319 type = types.str;
320 };
321 };
322
323 scheduler = {
324 enable = mkOption {
325 description = "Whether to enable kubernetes scheduler.";
326 default = false;
327 type = types.bool;
328 };
329
330 address = mkOption {
331 description = "Kubernetes scheduler listening address.";
332 default = "127.0.0.1";
333 type = types.str;
334 };
335
336 port = mkOption {
337 description = "Kubernetes scheduler listening port.";
338 default = 10251;
339 type = types.int;
340 };
341
342 leaderElect = mkOption {
343 description = "Whether to start leader election before executing main loop";
344 type = types.bool;
345 default = false;
346 };
347
348 extraOpts = mkOption {
349 description = "Kubernetes scheduler extra command line options.";
350 default = "";
351 type = types.str;
352 };
353 };
354
355 controllerManager = {
356 enable = mkOption {
357 description = "Whether to enable kubernetes controller manager.";
358 default = false;
359 type = types.bool;
360 };
361
362 address = mkOption {
363 description = "Kubernetes controller manager listening address.";
364 default = "127.0.0.1";
365 type = types.str;
366 };
367
368 port = mkOption {
369 description = "Kubernetes controller manager listening port.";
370 default = 10252;
371 type = types.int;
372 };
373
374 leaderElect = mkOption {
375 description = "Whether to start leader election before executing main loop";
376 type = types.bool;
377 default = false;
378 };
379
380 serviceAccountKeyFile = mkOption {
381 description = ''
382 Kubernetes controller manager PEM-encoded private RSA key file used to
383 sign service account tokens
384 '';
385 default = null;
386 type = types.nullOr types.path;
387 };
388
389 rootCaFile = mkOption {
390 description = ''
391 Kubernetes controller manager certificate authority file included in
392 service account's token secret.
393 '';
394 default = null;
395 type = types.nullOr types.path;
396 };
397
398 clusterCidr = mkOption {
399 description = "Kubernetes controller manager CIDR Range for Pods in cluster";
400 default = "10.10.0.0/16";
401 type = types.str;
402 };
403
404 extraOpts = mkOption {
405 description = "Kubernetes controller manager extra command line options.";
406 default = "";
407 type = types.str;
408 };
409 };
410
411 kubelet = {
412 enable = mkOption {
413 description = "Whether to enable kubernetes kubelet.";
414 default = false;
415 type = types.bool;
416 };
417
418 registerNode = mkOption {
419 description = "Whether to auto register kubelet with API server.";
420 default = true;
421 type = types.bool;
422 };
423
424 registerSchedulable = mkOption {
425 description = "Register the node as schedulable. No-op if register-node is false.";
426 default = true;
427 type = types.bool;
428 };
429
430 address = mkOption {
431 description = "Kubernetes kubelet info server listening address.";
432 default = "0.0.0.0";
433 type = types.str;
434 };
435
436 port = mkOption {
437 description = "Kubernetes kubelet info server listening port.";
438 default = 10250;
439 type = types.int;
440 };
441
442 tlsCertFile = mkOption {
443 description = "File containing x509 Certificate for HTTPS.";
444 default = null;
445 type = types.nullOr types.path;
446 };
447
448 tlsKeyFile = mkOption {
449 description = "File containing x509 private key matching tlsCertFile.";
450 default = null;
451 type = types.nullOr types.path;
452 };
453
454 healthz = {
455 bind = mkOption {
456 description = "Kubernetes kubelet healthz listening address.";
457 default = "127.0.0.1";
458 type = types.str;
459 };
460
461 port = mkOption {
462 description = "Kubernetes kubelet healthz port.";
463 default = 10248;
464 type = types.int;
465 };
466 };
467
468 hostname = mkOption {
469 description = "Kubernetes kubelet hostname override";
470 default = config.networking.hostName;
471 type = types.str;
472 };
473
474 allowPrivileged = mkOption {
475 description = "Whether to allow kubernetes containers to request privileged mode.";
476 default = true;
477 type = types.bool;
478 };
479
480 cadvisorPort = mkOption {
481 description = "Kubernetes kubelet local cadvisor port.";
482 default = 4194;
483 type = types.int;
484 };
485
486 clusterDns = mkOption {
487 description = "Use alternative dns.";
488 default = "10.10.0.1";
489 type = types.str;
490 };
491
492 clusterDomain = mkOption {
493 description = "Use alternative domain.";
494 default = "cluster.local";
495 type = types.str;
496 };
497
498 networkPlugin = mkOption {
499 description = "Network plugin to use by kubernetes";
500 type = types.nullOr (types.enum ["cni" "kubenet"]);
501 default = "kubenet";
502 };
503
504 cni = {
505 packages = mkOption {
506 description = "List of network plugin packages to install";
507 type = types.listOf types.package;
508 default = [];
509 };
510
511 config = mkOption {
512 description = "Kubernetes CNI configuration";
513 type = types.listOf types.attrs;
514 default = [];
515 example = literalExample ''
516 [{
517 "cniVersion": "0.2.0",
518 "name": "mynet",
519 "type": "bridge",
520 "bridge": "cni0",
521 "isGateway": true,
522 "ipMasq": true,
523 "ipam": {
524 "type": "host-local",
525 "subnet": "10.22.0.0/16",
526 "routes": [
527 { "dst": "0.0.0.0/0" }
528 ]
529 }
530 } {
531 "cniVersion": "0.2.0",
532 "type": "loopback"
533 }]
534 '';
535 };
536 };
537
538 manifests = mkOption {
539 description = "List of manifests to bootstrap with kubelet";
540 type = types.attrsOf types.attrs;
541 default = {};
542 };
543
544 extraOpts = mkOption {
545 description = "Kubernetes kubelet extra command line options.";
546 default = "";
547 type = types.str;
548 };
549 };
550
551 proxy = {
552 enable = mkOption {
553 description = "Whether to enable kubernetes proxy.";
554 default = false;
555 type = types.bool;
556 };
557
558 address = mkOption {
559 description = "Kubernetes proxy listening address.";
560 default = "0.0.0.0";
561 type = types.str;
562 };
563
564 extraOpts = mkOption {
565 description = "Kubernetes proxy extra command line options.";
566 default = "";
567 type = types.str;
568 };
569 };
570
571 dns = {
572 enable = mkEnableOption "kubernetes dns service.";
573
574 port = mkOption {
575 description = "Kubernetes dns listening port";
576 default = 53;
577 type = types.int;
578 };
579
580 domain = mkOption {
581 description = "Kuberntes dns domain under which to create names.";
582 default = cfg.kubelet.clusterDomain;
583 type = types.str;
584 };
585
586 extraOpts = mkOption {
587 description = "Kubernetes dns extra command line options.";
588 default = "";
589 type = types.str;
590 };
591 };
592 };
593
594 ###### implementation
595
596 config = mkMerge [
597 (mkIf cfg.kubelet.enable {
598 systemd.services.kubelet = {
599 description = "Kubernetes Kubelet Service";
600 wantedBy = [ "kubernetes.target" ];
601 after = [ "network.target" "docker.service" "kube-apiserver.service" ];
602 path = with pkgs; [ gitMinimal openssh docker utillinux iproute ethtool thin-provisioning-tools iptables ];
603 preStart = ''
604 docker load < ${infraContainer}
605 rm /opt/cni/bin/* || true
606 ${concatMapStringsSep "\n" (p: "ln -fs ${p.plugins}/* /opt/cni/bin") cfg.kubelet.cni.packages}
607 '';
608 serviceConfig = {
609 Slice = "kubernetes.slice";
610 ExecStart = ''${cfg.package}/bin/kubelet \
611 --pod-manifest-path=${manifests} \
612 --kubeconfig=${kubeconfig} \
613 --require-kubeconfig \
614 --address=${cfg.kubelet.address} \
615 --port=${toString cfg.kubelet.port} \
616 --register-node=${boolToString cfg.kubelet.registerNode} \
617 --register-schedulable=${boolToString cfg.kubelet.registerSchedulable} \
618 ${optionalString (cfg.kubelet.tlsCertFile != null)
619 "--tls-cert-file=${cfg.kubelet.tlsCertFile}"} \
620 ${optionalString (cfg.kubelet.tlsKeyFile != null)
621 "--tls-private-key-file=${cfg.kubelet.tlsKeyFile}"} \
622 --healthz-bind-address=${cfg.kubelet.healthz.bind} \
623 --healthz-port=${toString cfg.kubelet.healthz.port} \
624 --hostname-override=${cfg.kubelet.hostname} \
625 --allow-privileged=${boolToString cfg.kubelet.allowPrivileged} \
626 --root-dir=${cfg.dataDir} \
627 --cadvisor_port=${toString cfg.kubelet.cadvisorPort} \
628 ${optionalString (cfg.kubelet.clusterDns != "")
629 "--cluster-dns=${cfg.kubelet.clusterDns}"} \
630 ${optionalString (cfg.kubelet.clusterDomain != "")
631 "--cluster-domain=${cfg.kubelet.clusterDomain}"} \
632 --pod-infra-container-image=pause \
633 ${optionalString (cfg.kubelet.networkPlugin != null)
634 "--network-plugin=${cfg.kubelet.networkPlugin}"} \
635 --cni-conf-dir=${cniConfig} \
636 --reconcile-cidr \
637 --hairpin-mode=hairpin-veth \
638 ${optionalString cfg.verbose "--v=6 --log_flush_frequency=1s"} \
639 ${cfg.kubelet.extraOpts}
640 '';
641 WorkingDirectory = cfg.dataDir;
642 };
643 };
644
645 environment.etc = mapAttrs' (name: manifest:
646 nameValuePair "kubernetes/manifests/${name}.json" {
647 text = builtins.toJSON manifest;
648 mode = "0755";
649 }
650 ) cfg.kubelet.manifests;
651
652 # Allways include cni plugins
653 services.kubernetes.kubelet.cni.packages = [pkgs.cni];
654 })
655
656 (mkIf cfg.apiserver.enable {
657 systemd.services.kube-apiserver = {
658 description = "Kubernetes Kubelet Service";
659 wantedBy = [ "kubernetes.target" ];
660 after = [ "network.target" "docker.service" ];
661 serviceConfig = {
662 Slice = "kubernetes.slice";
663 ExecStart = ''${cfg.package}/bin/kube-apiserver \
664 --etcd-servers=${concatStringsSep "," cfg.etcd.servers} \
665 ${optionalString (cfg.etcd.caFile != null)
666 "--etcd-cafile=${cfg.etcd.caFile}"} \
667 ${optionalString (cfg.etcd.certFile != null)
668 "--etcd-certfile=${cfg.etcd.certFile}"} \
669 ${optionalString (cfg.etcd.keyFile != null)
670 "--etcd-keyfile=${cfg.etcd.keyFile}"} \
671 --insecure-port=${toString cfg.apiserver.port} \
672 --bind-address=0.0.0.0 \
673 ${optionalString (cfg.apiserver.advertiseAddress != null)
674 "--advertise-address=${cfg.apiserver.advertiseAddress}"} \
675 --allow-privileged=${boolToString cfg.apiserver.allowPrivileged}\
676 ${optionalString (cfg.apiserver.tlsCertFile != null)
677 "--tls-cert-file=${cfg.apiserver.tlsCertFile}"} \
678 ${optionalString (cfg.apiserver.tlsKeyFile != null)
679 "--tls-private-key-file=${cfg.apiserver.tlsKeyFile}"} \
680 ${optionalString (cfg.apiserver.tokenAuth != null)
681 "--token-auth-file=${cfg.apiserver.tokenAuth}"} \
682 --kubelet-https=${boolToString cfg.apiserver.kubeletHttps} \
683 ${optionalString (cfg.apiserver.kubeletClientCaFile != null)
684 "--kubelet-certificate-authority=${cfg.apiserver.kubeletClientCaFile}"} \
685 ${optionalString (cfg.apiserver.kubeletClientCertFile != null)
686 "--kubelet-client-certificate=${cfg.apiserver.kubeletClientCertFile}"} \
687 ${optionalString (cfg.apiserver.kubeletClientKeyFile != null)
688 "--kubelet-client-key=${cfg.apiserver.kubeletClientKeyFile}"} \
689 ${optionalString (cfg.apiserver.clientCaFile != null)
690 "--client-ca-file=${cfg.apiserver.clientCaFile}"} \
691 --authorization-mode=${cfg.apiserver.authorizationMode} \
692 ${optionalString (cfg.apiserver.authorizationMode == "ABAC")
693 "--authorization-policy-file=${policyFile}"} \
694 --secure-port=${toString cfg.apiserver.securePort} \
695 --service-cluster-ip-range=${cfg.apiserver.portalNet} \
696 ${optionalString (cfg.apiserver.runtimeConfig != "")
697 "--runtime-config=${cfg.apiserver.runtimeConfig}"} \
698 --admission_control=${concatStringsSep "," cfg.apiserver.admissionControl} \
699 ${optionalString (cfg.apiserver.serviceAccountKeyFile!=null)
700 "--service-account-key-file=${cfg.apiserver.serviceAccountKeyFile}"} \
701 ${optionalString cfg.verbose "--v=6"} \
702 ${optionalString cfg.verbose "--log-flush-frequency=1s"} \
703 ${cfg.apiserver.extraOpts}
704 '';
705 WorkingDirectory = cfg.dataDir;
706 User = "kubernetes";
707 Group = "kubernetes";
708 AmbientCapabilities = "cap_net_bind_service";
709 Restart = "on-failure";
710 RestartSec = 5;
711 };
712 };
713 })
714
715 (mkIf cfg.scheduler.enable {
716 systemd.services.kube-scheduler = {
717 description = "Kubernetes Scheduler Service";
718 wantedBy = [ "kubernetes.target" ];
719 after = [ "kube-apiserver.service" ];
720 serviceConfig = {
721 Slice = "kubernetes.slice";
722 ExecStart = ''${cfg.package}/bin/kube-scheduler \
723 --address=${cfg.scheduler.address} \
724 --port=${toString cfg.scheduler.port} \
725 --leader-elect=${boolToString cfg.scheduler.leaderElect} \
726 --kubeconfig=${kubeconfig} \
727 ${optionalString cfg.verbose "--v=6"} \
728 ${optionalString cfg.verbose "--log-flush-frequency=1s"} \
729 ${cfg.scheduler.extraOpts}
730 '';
731 WorkingDirectory = cfg.dataDir;
732 User = "kubernetes";
733 Group = "kubernetes";
734 };
735 };
736 })
737
738 (mkIf cfg.controllerManager.enable {
739 systemd.services.kube-controller-manager = {
740 description = "Kubernetes Controller Manager Service";
741 wantedBy = [ "kubernetes.target" ];
742 after = [ "kube-apiserver.service" ];
743 serviceConfig = {
744 RestartSec = "30s";
745 Restart = "on-failure";
746 Slice = "kubernetes.slice";
747 ExecStart = ''${cfg.package}/bin/kube-controller-manager \
748 --address=${cfg.controllerManager.address} \
749 --port=${toString cfg.controllerManager.port} \
750 --kubeconfig=${kubeconfig} \
751 --leader-elect=${boolToString cfg.controllerManager.leaderElect} \
752 ${if (cfg.controllerManager.serviceAccountKeyFile!=null)
753 then "--service-account-private-key-file=${cfg.controllerManager.serviceAccountKeyFile}"
754 else "--service-account-private-key-file=/var/run/kubernetes/apiserver.key"} \
755 ${optionalString (cfg.controllerManager.rootCaFile!=null)
756 "--root-ca-file=${cfg.controllerManager.rootCaFile}"} \
757 ${optionalString (cfg.controllerManager.clusterCidr!=null)
758 "--cluster-cidr=${cfg.controllerManager.clusterCidr}"} \
759 --allocate-node-cidrs=true \
760 ${optionalString cfg.verbose "--v=6"} \
761 ${optionalString cfg.verbose "--log-flush-frequency=1s"} \
762 ${cfg.controllerManager.extraOpts}
763 '';
764 WorkingDirectory = cfg.dataDir;
765 User = "kubernetes";
766 Group = "kubernetes";
767 };
768 };
769 })
770
771 (mkIf cfg.proxy.enable {
772 systemd.services.kube-proxy = {
773 description = "Kubernetes Proxy Service";
774 wantedBy = [ "kubernetes.target" ];
775 after = [ "kube-apiserver.service" ];
776 path = [pkgs.iptables];
777 serviceConfig = {
778 Slice = "kubernetes.slice";
779 ExecStart = ''${cfg.package}/bin/kube-proxy \
780 --kubeconfig=${kubeconfig} \
781 --bind-address=${cfg.proxy.address} \
782 ${optionalString cfg.verbose "--v=6"} \
783 ${optionalString cfg.verbose "--log-flush-frequency=1s"} \
784 ${cfg.proxy.extraOpts}
785 '';
786 WorkingDirectory = cfg.dataDir;
787 };
788 };
789 })
790
791 (mkIf cfg.dns.enable {
792 systemd.services.kube-dns = {
793 description = "Kubernetes Dns Service";
794 wantedBy = [ "kubernetes.target" ];
795 after = [ "kube-apiserver.service" ];
796 serviceConfig = {
797 Slice = "kubernetes.slice";
798 ExecStart = ''${cfg.package}/bin/kube-dns \
799 --kubecfg-file=${kubeconfig} \
800 --dns-port=${toString cfg.dns.port} \
801 --domain=${cfg.dns.domain} \
802 ${optionalString cfg.verbose "--v=6"} \
803 ${optionalString cfg.verbose "--log-flush-frequency=1s"} \
804 ${cfg.dns.extraOpts}
805 '';
806 WorkingDirectory = cfg.dataDir;
807 User = "kubernetes";
808 Group = "kubernetes";
809 AmbientCapabilities = "cap_net_bind_service";
810 SendSIGHUP = true;
811 };
812 };
813 })
814
815 (mkIf cfg.kubelet.enable {
816 boot.kernelModules = ["br_netfilter"];
817 })
818
819 (mkIf (any (el: el == "master") cfg.roles) {
820 virtualisation.docker.enable = mkDefault true;
821 services.kubernetes.kubelet.enable = mkDefault true;
822 services.kubernetes.kubelet.allowPrivileged = mkDefault true;
823 services.kubernetes.apiserver.enable = mkDefault true;
824 services.kubernetes.scheduler.enable = mkDefault true;
825 services.kubernetes.controllerManager.enable = mkDefault true;
826 services.etcd.enable = mkDefault (cfg.etcd.servers == ["http://127.0.0.1:2379"]);
827 })
828
829 (mkIf (any (el: el == "node") cfg.roles) {
830 virtualisation.docker.enable = mkDefault true;
831 virtualisation.docker.logDriver = mkDefault "json-file";
832 services.kubernetes.kubelet.enable = mkDefault true;
833 services.kubernetes.proxy.enable = mkDefault true;
834 services.kubernetes.dns.enable = mkDefault true;
835 })
836
837 (mkIf (
838 cfg.apiserver.enable ||
839 cfg.scheduler.enable ||
840 cfg.controllerManager.enable ||
841 cfg.kubelet.enable ||
842 cfg.proxy.enable ||
843 cfg.dns.enable
844 ) {
845 systemd.targets.kubernetes = {
846 description = "Kubernetes";
847 wantedBy = [ "multi-user.target" ];
848 };
849
850 systemd.tmpfiles.rules = [
851 "d /opt/cni/bin 0755 root root -"
852 "d /var/run/kubernetes 0755 kubernetes kubernetes -"
853 "d /var/lib/kubernetes 0755 kubernetes kubernetes -"
854 ];
855
856 environment.systemPackages = [ cfg.package ];
857 users.extraUsers = singleton {
858 name = "kubernetes";
859 uid = config.ids.uids.kubernetes;
860 description = "Kubernetes user";
861 extraGroups = [ "docker" ];
862 group = "kubernetes";
863 home = cfg.dataDir;
864 createHome = true;
865 };
866 users.extraGroups.kubernetes.gid = config.ids.gids.kubernetes;
867 })
868 ];
869}