pyrox.dev
1{ config, lib, pkgs, ... }:
2
3with lib;
4
5let
6 cfg = config.services.kubernetes;
7
8in {
9
10 ###### interface
11
12 options.services.kubernetes = {
13 package = mkOption {
14 description = "Kubernetes package to use.";
15 type = types.package;
16 };
17
18 verbose = mkOption {
19 description = "Kubernetes enable verbose mode for debugging";
20 default = false;
21 type = types.bool;
22 };
23
24 etcdServers = mkOption {
25 description = "Kubernetes list of etcd servers to watch.";
26 default = [ "127.0.0.1:4001" ];
27 type = types.listOf types.str;
28 };
29
30 roles = mkOption {
31 description = ''
32 Kubernetes role that this machine should take.
33
34 Master role will enable etcd, apiserver, scheduler and controller manager
35 services. Node role will enable etcd, docker, kubelet and proxy services.
36 '';
37 default = [];
38 type = types.listOf (types.enum ["master" "node"]);
39 };
40
41 dataDir = mkOption {
42 description = "Kubernetes root directory for managing kubelet files.";
43 default = "/var/lib/kubernetes";
44 type = types.path;
45 };
46
47 dockerCfg = mkOption {
48 description = "Kubernetes contents of dockercfg file.";
49 default = "";
50 type = types.lines;
51 };
52
53 apiserver = {
54 enable = mkOption {
55 description = "Whether to enable kubernetes apiserver.";
56 default = false;
57 type = types.bool;
58 };
59
60 address = mkOption {
61 description = "Kubernetes apiserver listening address.";
62 default = "127.0.0.1";
63 type = types.str;
64 };
65
66 publicAddress = mkOption {
67 description = ''
68 Kubernetes apiserver public listening address used for read only and
69 secure port.
70 '';
71 default = cfg.apiserver.address;
72 type = types.str;
73 };
74
75 port = mkOption {
76 description = "Kubernetes apiserver listening port.";
77 default = 8080;
78 type = types.int;
79 };
80
81 securePort = mkOption {
82 description = "Kubernetes apiserver secure port.";
83 default = 6443;
84 type = types.int;
85 };
86
87 tlsCertFile = mkOption {
88 description = "Kubernetes apiserver certificate file.";
89 default = "";
90 type = types.str;
91 };
92
93 tlsPrivateKeyFile = mkOption {
94 description = "Kubernetes apiserver private key file.";
95 default = "";
96 type = types.str;
97 };
98
99 clientCaFile = mkOption {
100 description = "Kubernetes apiserver CA file for client auth.";
101 default = "";
102 type = types.str;
103 };
104
105 tokenAuth = mkOption {
106 description = ''
107 Kubernetes apiserver token authentication file. See
108 <link xlink:href="http://kubernetes.io/v1.0/docs/admin/authentication.html"/>
109 '';
110 default = {};
111 example = literalExample ''
112 {
113 alice = "abc123";
114 bob = "xyz987";
115 }
116 '';
117 type = types.attrsOf types.str;
118 };
119
120 authorizationMode = mkOption {
121 description = ''
122 Kubernetes apiserver authorization mode (AlwaysAllow/AlwaysDeny/ABAC). See
123 <link xlink:href="http://kubernetes.io/v1.0/docs/admin/authorization.html"/>
124 '';
125 default = "AlwaysAllow";
126 type = types.enum ["AlwaysAllow" "AlwaysDeny" "ABAC"];
127 };
128
129 authorizationPolicy = mkOption {
130 description = ''
131 Kubernetes apiserver authorization policy file. See
132 <link xlink:href="http://kubernetes.io/v1.0/docs/admin/authorization.html"/>
133 '';
134 default = [];
135 example = literalExample ''
136 [
137 {user = "admin";}
138 {user = "scheduler"; readonly = true; kind= "pods";}
139 {user = "scheduler"; kind = "bindings";}
140 {user = "kubelet"; readonly = true; kind = "bindings";}
141 {user = "kubelet"; kind = "events";}
142 {user= "alice"; ns = "projectCaribou";}
143 {user = "bob"; readonly = true; ns = "projectCaribou";}
144 ]
145 '';
146 type = types.listOf types.attrs;
147 };
148
149 allowPrivileged = mkOption {
150 description = "Whether to allow privileged containers on kubernetes.";
151 default = false;
152 type = types.bool;
153 };
154
155 portalNet = mkOption {
156 description = "Kubernetes CIDR notation IP range from which to assign portal IPs";
157 default = "10.10.10.10/16";
158 type = types.str;
159 };
160
161 runtimeConfig = mkOption {
162 description = ''
163 Api runtime configuration. See
164 <link xlink:href="http://kubernetes.io/v1.0/docs/admin/cluster-management.html"/>
165 '';
166 default = "";
167 example = "api/all=false,api/v1=true";
168 type = types.str;
169 };
170
171 admissionControl = mkOption {
172 description = ''
173 Kubernetes admission control plugins to use. See
174 <link xlink:href="http://kubernetes.io/v1.0/docs/admin/admission-controllers.html"/>
175 '';
176 default = ["AlwaysAdmit"];
177 example = [
178 "NamespaceLifecycle" "NamespaceExists" "LimitRanger"
179 "SecurityContextDeny" "ServiceAccount" "ResourceQuota"
180 ];
181 type = types.listOf types.str;
182 };
183
184 serviceAccountKey = mkOption {
185 description = ''
186 Kubernetes apiserver PEM-encoded x509 RSA private or public key file,
187 used to verify ServiceAccount tokens.
188 '';
189 default = null;
190 type = types.nullOr types.path;
191 };
192
193 extraOpts = mkOption {
194 description = "Kubernetes apiserver extra command line options.";
195 default = "";
196 type = types.str;
197 };
198 };
199
200 scheduler = {
201 enable = mkOption {
202 description = "Whether to enable kubernetes scheduler.";
203 default = false;
204 type = types.bool;
205 };
206
207 address = mkOption {
208 description = "Kubernetes scheduler listening address.";
209 default = "127.0.0.1";
210 type = types.str;
211 };
212
213 port = mkOption {
214 description = "Kubernetes scheduler listening port.";
215 default = 10251;
216 type = types.int;
217 };
218
219 master = mkOption {
220 description = "Kubernetes apiserver address";
221 default = "${cfg.apiserver.address}:${toString cfg.apiserver.port}";
222 type = types.str;
223 };
224
225 extraOpts = mkOption {
226 description = "Kubernetes scheduler extra command line options.";
227 default = "";
228 type = types.str;
229 };
230 };
231
232 controllerManager = {
233 enable = mkOption {
234 description = "Whether to enable kubernetes controller manager.";
235 default = false;
236 type = types.bool;
237 };
238
239 address = mkOption {
240 description = "Kubernetes controller manager listening address.";
241 default = "127.0.0.1";
242 type = types.str;
243 };
244
245 port = mkOption {
246 description = "Kubernetes controller manager listening port.";
247 default = 10252;
248 type = types.int;
249 };
250
251 master = mkOption {
252 description = "Kubernetes apiserver address";
253 default = "${cfg.apiserver.address}:${toString cfg.apiserver.port}";
254 type = types.str;
255 };
256
257 serviceAccountPrivateKey = mkOption {
258 description = ''
259 Kubernetes controller manager PEM-encoded private RSA key file used to
260 sign service account tokens
261 '';
262 default = null;
263 type = types.nullOr types.path;
264 };
265
266 rootCaFile = mkOption {
267 description = ''
268 Kubernetes controller manager certificate authority file included in
269 service account's token secret.
270 '';
271 default = null;
272 type = types.nullOr types.path;
273 };
274
275 extraOpts = mkOption {
276 description = "Kubernetes controller manager extra command line options.";
277 default = "";
278 type = types.str;
279 };
280 };
281
282 kubelet = {
283 enable = mkOption {
284 description = "Whether to enable kubernetes kubelet.";
285 default = false;
286 type = types.bool;
287 };
288
289 registerNode = mkOption {
290 description = "Whether to auto register kubelet with API server.";
291 default = true;
292 type = types.bool;
293 };
294
295 address = mkOption {
296 description = "Kubernetes kubelet info server listening address.";
297 default = "0.0.0.0";
298 type = types.str;
299 };
300
301 port = mkOption {
302 description = "Kubernetes kubelet info server listening port.";
303 default = 10250;
304 type = types.int;
305 };
306
307 healthz = {
308 bind = mkOption {
309 description = "Kubernetes kubelet healthz listening address.";
310 default = "127.0.0.1";
311 type = types.str;
312 };
313
314 port = mkOption {
315 description = "Kubernetes kubelet healthz port.";
316 default = 10248;
317 type = types.int;
318 };
319 };
320
321 hostname = mkOption {
322 description = "Kubernetes kubelet hostname override";
323 default = config.networking.hostName;
324 type = types.str;
325 };
326
327 allowPrivileged = mkOption {
328 description = "Whether to allow kubernetes containers to request privileged mode.";
329 default = false;
330 type = types.bool;
331 };
332
333 apiServers = mkOption {
334 description = ''
335 Kubernetes kubelet list of Kubernetes API servers for publishing events,
336 and reading pods and services.
337 '';
338 default = ["${cfg.apiserver.address}:${toString cfg.apiserver.port}"];
339 type = types.listOf types.str;
340 };
341
342 cadvisorPort = mkOption {
343 description = "Kubernetes kubelet local cadvisor port.";
344 default = 4194;
345 type = types.int;
346 };
347
348 clusterDns = mkOption {
349 description = "Use alternative dns.";
350 default = "";
351 type = types.str;
352 };
353
354 clusterDomain = mkOption {
355 description = "Use alternative domain.";
356 default = "kubernetes.io";
357 type = types.str;
358 };
359
360 extraOpts = mkOption {
361 description = "Kubernetes kubelet extra command line options.";
362 default = "";
363 type = types.str;
364 };
365 };
366
367 proxy = {
368 enable = mkOption {
369 description = "Whether to enable kubernetes proxy.";
370 default = false;
371 type = types.bool;
372 };
373
374 address = mkOption {
375 description = "Kubernetes proxy listening address.";
376 default = "0.0.0.0";
377 type = types.str;
378 };
379
380 master = mkOption {
381 description = "Kubernetes apiserver address";
382 default = "${cfg.apiserver.address}:${toString cfg.apiserver.port}";
383 type = types.str;
384 };
385
386 extraOpts = mkOption {
387 description = "Kubernetes proxy extra command line options.";
388 default = "";
389 type = types.str;
390 };
391 };
392
393 kube2sky = {
394 enable = mkEnableOption "Whether to enable kube2sky dns service.";
395
396 domain = mkOption {
397 description = "Kuberntes kube2sky domain under which all DNS names will be hosted.";
398 default = cfg.kubelet.clusterDomain;
399 type = types.str;
400 };
401
402 master = mkOption {
403 description = "Kubernetes apiserver address";
404 default = "${cfg.apiserver.address}:${toString cfg.apiserver.port}";
405 type = types.str;
406 };
407
408 extraOpts = mkOption {
409 description = "Kubernetes kube2sky extra command line options.";
410 default = "";
411 type = types.str;
412 };
413 };
414 };
415
416 ###### implementation
417
418 config = mkMerge [
419 (mkIf cfg.apiserver.enable {
420 systemd.services.kube-apiserver = {
421 description = "Kubernetes Api Server";
422 wantedBy = [ "multi-user.target" ];
423 requires = ["kubernetes-setup.service"];
424 after = [ "network-interfaces.target" "etcd.service" ];
425 serviceConfig = {
426 ExecStart = let
427 authorizationPolicyFile =
428 pkgs.writeText "kubernetes-policy"
429 (builtins.toJSON cfg.apiserver.authorizationPolicy);
430 tokenAuthFile =
431 pkgs.writeText "kubernetes-auth"
432 (concatImapStringsSep "\n" (i: v: v + "," + (toString i))
433 (mapAttrsToList (name: token: token + "," + name) cfg.apiserver.tokenAuth));
434 in ''${cfg.package}/bin/kube-apiserver \
435 --etcd-servers=${concatMapStringsSep "," (f: "http://${f}") cfg.etcdServers} \
436 --insecure-bind-address=${cfg.apiserver.address} \
437 --insecure-port=${toString cfg.apiserver.port} \
438 --bind-address=${cfg.apiserver.publicAddress} \
439 --allow-privileged=${if cfg.apiserver.allowPrivileged then "true" else "false"} \
440 ${optionalString (cfg.apiserver.tlsCertFile!="")
441 "--tls-cert-file=${cfg.apiserver.tlsCertFile}"} \
442 ${optionalString (cfg.apiserver.tlsPrivateKeyFile!="")
443 "--tls-private-key-file=${cfg.apiserver.tlsPrivateKeyFile}"} \
444 ${optionalString (cfg.apiserver.tokenAuth!=[])
445 "--token-auth-file=${tokenAuthFile}"} \
446 ${optionalString (cfg.apiserver.clientCaFile!="")
447 "--client-ca-file=${cfg.apiserver.clientCaFile}"} \
448 --authorization-mode=${cfg.apiserver.authorizationMode} \
449 ${optionalString (cfg.apiserver.authorizationMode == "ABAC")
450 "--authorization-policy-file=${authorizationPolicyFile}"} \
451 --secure-port=${toString cfg.apiserver.securePort} \
452 --service-cluster-ip-range=${cfg.apiserver.portalNet} \
453 ${optionalString (cfg.apiserver.runtimeConfig!="")
454 "--runtime-config=${cfg.apiserver.runtimeConfig}"} \
455 --admission_control=${concatStringsSep "," cfg.apiserver.admissionControl} \
456 ${optionalString (cfg.apiserver.serviceAccountKey!=null)
457 "--service-account-key-file=${cfg.apiserver.serviceAccountKey}"} \
458 --logtostderr=true \
459 ${optionalString cfg.verbose "--v=6 --log-flush-frequency=1s"} \
460 ${cfg.apiserver.extraOpts}
461 '';
462 User = "kubernetes";
463 };
464 };
465 })
466
467 (mkIf cfg.scheduler.enable {
468 systemd.services.kube-scheduler = {
469 description = "Kubernetes Scheduler Service";
470 wantedBy = [ "multi-user.target" ];
471 after = [ "network-interfaces.target" "kubernetes-apiserver.service" ];
472 serviceConfig = {
473 ExecStart = ''${cfg.package}/bin/kube-scheduler \
474 --address=${cfg.scheduler.address} \
475 --port=${toString cfg.scheduler.port} \
476 --master=${cfg.scheduler.master} \
477 --logtostderr=true \
478 ${optionalString cfg.verbose "--v=6 --log-flush-frequency=1s"} \
479 ${cfg.scheduler.extraOpts}
480 '';
481 User = "kubernetes";
482 };
483 };
484 })
485
486 (mkIf cfg.controllerManager.enable {
487 systemd.services.kube-controller-manager = {
488 description = "Kubernetes Controller Manager Service";
489 wantedBy = [ "multi-user.target" ];
490 after = [ "network-interfaces.target" "kubernetes-apiserver.service" ];
491 serviceConfig = {
492 ExecStart = ''${cfg.package}/bin/kube-controller-manager \
493 --address=${cfg.controllerManager.address} \
494 --port=${toString cfg.controllerManager.port} \
495 --master=${cfg.controllerManager.master} \
496 ${optionalString (cfg.controllerManager.serviceAccountPrivateKey!=null)
497 "--service-account-private-key-file=${cfg.controllerManager.serviceAccountPrivateKey}"} \
498 ${optionalString (cfg.controllerManager.rootCaFile!=null)
499 "--root-ca-file=${cfg.controllerManager.rootCaFile}"} \
500 --logtostderr=true \
501 ${optionalString cfg.verbose "--v=6 --log-flush-frequency=1s"} \
502 ${cfg.controllerManager.extraOpts}
503 '';
504 User = "kubernetes";
505 };
506 };
507 })
508
509 (mkIf cfg.kubelet.enable {
510 systemd.services.kubelet = {
511 description = "Kubernetes Kubelet Service";
512 wantedBy = [ "multi-user.target" ];
513 requires = ["kubernetes-setup.service"];
514 after = [ "network-interfaces.target" "etcd.service" "docker.service" ];
515 path = [ pkgs.gitMinimal pkgs.openssh ];
516 script = ''
517 export PATH="/bin:/sbin:/usr/bin:/usr/sbin:$PATH"
518 exec ${cfg.package}/bin/kubelet \
519 --api-servers=${concatMapStringsSep "," (f: "http://${f}") cfg.kubelet.apiServers} \
520 --register-node=${if cfg.kubelet.registerNode then "true" else "false"} \
521 --address=${cfg.kubelet.address} \
522 --port=${toString cfg.kubelet.port} \
523 --healthz-bind-address=${cfg.kubelet.healthz.bind} \
524 --healthz-port=${toString cfg.kubelet.healthz.port} \
525 --hostname-override=${cfg.kubelet.hostname} \
526 --allow-privileged=${if cfg.kubelet.allowPrivileged then "true" else "false"} \
527 --root-dir=${cfg.dataDir} \
528 --cadvisor_port=${toString cfg.kubelet.cadvisorPort} \
529 ${optionalString (cfg.kubelet.clusterDns != "")
530 ''--cluster-dns=${cfg.kubelet.clusterDns}''} \
531 ${optionalString (cfg.kubelet.clusterDomain != "")
532 ''--cluster-domain=${cfg.kubelet.clusterDomain}''} \
533 --logtostderr=true \
534 ${optionalString cfg.verbose "--v=6 --log_flush_frequency=1s"} \
535 ${cfg.kubelet.extraOpts}
536 '';
537 serviceConfig.WorkingDirectory = cfg.dataDir;
538 };
539 })
540
541 (mkIf cfg.proxy.enable {
542 systemd.services.kube-proxy = {
543 description = "Kubernetes Proxy Service";
544 wantedBy = [ "multi-user.target" ];
545 after = [ "network-interfaces.target" "etcd.service" ];
546 serviceConfig = {
547 ExecStart = ''${cfg.package}/bin/kube-proxy \
548 --master=${cfg.proxy.master} \
549 --bind-address=${cfg.proxy.address} \
550 --logtostderr=true \
551 ${optionalString cfg.verbose "--v=6 --log-flush-frequency=1s"} \
552 ${cfg.proxy.extraOpts}
553 '';
554 Restart = "always"; # Retry connection
555 RestartSec = "5s";
556 };
557 };
558 })
559
560 (mkIf cfg.kube2sky.enable {
561 systemd.services.kube2sky = {
562 description = "Kubernetes Dns Bridge Service";
563 wantedBy = [ "multi-user.target" ];
564 after = [ "network.target" "skydns.service" "etcd.service" "kubernetes-apiserver.service" ];
565 serviceConfig = {
566 ExecStart = ''${cfg.package}/bin/kube2sky \
567 -etcd-server=http://${head cfg.etcdServers} \
568 -domain=${cfg.kube2sky.domain} \
569 -kube_master_url=http://${cfg.kube2sky.master} \
570 -logtostderr=true \
571 ${optionalString cfg.verbose "--v=6 --log-flush-frequency=1s"} \
572 ${cfg.kube2sky.extraOpts}
573 '';
574 User = "kubernetes";
575 };
576 };
577 })
578
579 (mkIf (any (el: el == "master") cfg.roles) {
580 services.kubernetes.apiserver.enable = mkDefault true;
581 services.kubernetes.scheduler.enable = mkDefault true;
582 services.kubernetes.controllerManager.enable = mkDefault true;
583 services.kubernetes.kube2sky.enable = mkDefault true;
584 })
585
586 (mkIf (any (el: el == "node") cfg.roles) {
587 virtualisation.docker.enable = mkDefault true;
588 services.kubernetes.kubelet.enable = mkDefault true;
589 services.kubernetes.proxy.enable = mkDefault true;
590 })
591
592 (mkIf (any (el: el == "node" || el == "master") cfg.roles) {
593 services.etcd.enable = mkDefault true;
594
595 services.skydns.enable = mkDefault true;
596 services.skydns.domain = mkDefault cfg.kubelet.clusterDomain;
597 })
598
599 (mkIf (
600 cfg.apiserver.enable ||
601 cfg.scheduler.enable ||
602 cfg.controllerManager.enable ||
603 cfg.kubelet.enable ||
604 cfg.proxy.enable
605 ) {
606 systemd.services.kubernetes-setup = {
607 description = "Kubernetes setup.";
608 serviceConfig.Type = "oneshot";
609 script = ''
610 mkdir -p /var/run/kubernetes
611 chown kubernetes /var/lib/kubernetes
612
613 rm ${cfg.dataDir}/.dockercfg || true
614 ln -fs ${pkgs.writeText "kubernetes-dockercfg" cfg.dockerCfg} ${cfg.dataDir}/.dockercfg
615 '';
616 };
617
618 services.kubernetes.package = mkDefault pkgs.kubernetes;
619
620 environment.systemPackages = [ cfg.package ];
621
622 users.extraUsers = singleton {
623 name = "kubernetes";
624 uid = config.ids.uids.kubernetes;
625 description = "Kubernetes user";
626 extraGroups = [ "docker" ];
627 group = "kubernetes";
628 home = cfg.dataDir;
629 createHome = true;
630 };
631 users.extraGroups.kubernetes.gid = config.ids.gids.kubernetes;
632 })
633
634 ];
635}