pyrox.dev
1{ config, lib, pkgs, ... }:
2
3with lib;
4
5let
6 cfg = config.services.kubernetes;
7
8in {
9
10 ###### interface
11
12 options.services.kubernetes = {
13 package = mkOption {
14 description = "Kubernetes package to use.";
15 type = types.package;
16 };
17
18 verbose = mkOption {
19 description = "Kubernetes enable verbose mode for debugging";
20 default = false;
21 type = types.bool;
22 };
23
24 etcdServers = mkOption {
25 description = "Kubernetes list of etcd servers to watch.";
26 default = [ "127.0.0.1:4001" ];
27 type = types.listOf types.str;
28 };
29
30 roles = mkOption {
31 description = ''
32 Kubernetes role that this machine should take.
33
34 Master role will enable etcd, apiserver, scheduler and controller manager
35 services. Node role will enable etcd, docker, kubelet and proxy services.
36 '';
37 default = [];
38 type = types.listOf (types.enum ["master" "node"]);
39 };
40
41 dataDir = mkOption {
42 description = "Kubernetes root directory for managing kubelet files.";
43 default = "/var/lib/kubernetes";
44 type = types.path;
45 };
46
47 dockerCfg = mkOption {
48 description = "Kubernetes contents of dockercfg file.";
49 default = "";
50 type = types.lines;
51 };
52
53 apiserver = {
54 enable = mkOption {
55 description = "Whether to enable kubernetes apiserver.";
56 default = false;
57 type = types.bool;
58 };
59
60 address = mkOption {
61 description = "Kubernetes apiserver listening address.";
62 default = "127.0.0.1";
63 type = types.str;
64 };
65
66 publicAddress = mkOption {
67 description = ''
68 Kubernetes apiserver public listening address used for read only and
69 secure port.
70 '';
71 default = cfg.apiserver.address;
72 type = types.str;
73 };
74
75 port = mkOption {
76 description = "Kubernets apiserver listening port.";
77 default = 8080;
78 type = types.int;
79 };
80
81 securePort = mkOption {
82 description = "Kubernetes apiserver secure port.";
83 default = 6443;
84 type = types.int;
85 };
86
87 tlsCertFile = mkOption {
88 description = "Kubernetes apiserver certificate file.";
89 default = "";
90 type = types.str;
91 };
92
93 tlsPrivateKeyFile = mkOption {
94 description = "Kubernetes apiserver private key file.";
95 default = "";
96 type = types.str;
97 };
98
99 clientCaFile = mkOption {
100 description = "Kubernetes apiserver CA file for client auth.";
101 default = "";
102 type = types.str;
103 };
104
105 tokenAuth = mkOption {
106 description = ''
107 Kubernetes apiserver token authentication file. See
108 <link xlink:href="https://github.com/GoogleCloudPlatform/kubernetes/blob/master/docs/authentication.md"/>
109 '';
110 default = {};
111 example = literalExample ''
112 {
113 alice = "abc123";
114 bob = "xyz987";
115 }
116 '';
117 type = types.attrsOf types.str;
118 };
119
120 authorizationMode = mkOption {
121 description = ''
122 Kubernetes apiserver authorization mode (AlwaysAllow/AlwaysDeny/ABAC). See
123 <link xlink:href="https://github.com/GoogleCloudPlatform/kubernetes/blob/master/docs/authorization.md"/>
124 '';
125 default = "AlwaysAllow";
126 type = types.enum ["AlwaysAllow" "AlwaysDeny" "ABAC"];
127 };
128
129 authorizationPolicy = mkOption {
130 description = ''
131 Kubernetes apiserver authorization policy file. See
132 <link xlink:href="https://github.com/GoogleCloudPlatform/kubernetes/blob/master/docs/authorization.md"/>
133 '';
134 default = [];
135 example = literalExample ''
136 [
137 {user = "admin";}
138 {user = "scheduler"; readonly = true; kind= "pods";}
139 {user = "scheduler"; kind = "bindings";}
140 {user = "kubelet"; readonly = true; kind = "bindings";}
141 {user = "kubelet"; kind = "events";}
142 {user= "alice"; ns = "projectCaribou";}
143 {user = "bob"; readonly = true; ns = "projectCaribou";}
144 ]
145 '';
146 type = types.listOf types.attrs;
147 };
148
149 allowPrivileged = mkOption {
150 description = "Whether to allow privileged containers on kubernetes.";
151 default = false;
152 type = types.bool;
153 };
154
155 portalNet = mkOption {
156 description = "Kubernetes CIDR notation IP range from which to assign portal IPs";
157 default = "10.10.10.10/16";
158 type = types.str;
159 };
160
161 runtimeConfig = mkOption {
162 description = "Api runtime configuration";
163 default = "";
164 example = "api/all=false,api/v1=true";
165 type = types.str;
166 };
167
168 admissionControl = mkOption {
169 description = "Kubernetes admission control plugins to use.";
170 default = ["AlwaysAdmit"];
171 type = types.listOf types.str;
172 };
173
174 extraOpts = mkOption {
175 description = "Kubernetes apiserver extra command line options.";
176 default = "";
177 type = types.str;
178 };
179 };
180
181 scheduler = {
182 enable = mkOption {
183 description = "Whether to enable kubernetes scheduler.";
184 default = false;
185 type = types.bool;
186 };
187
188 address = mkOption {
189 description = "Kubernetes scheduler listening address.";
190 default = "127.0.0.1";
191 type = types.str;
192 };
193
194 port = mkOption {
195 description = "Kubernets scheduler listening port.";
196 default = 10251;
197 type = types.int;
198 };
199
200 master = mkOption {
201 description = "Kubernetes apiserver address";
202 default = "${cfg.apiserver.address}:${toString cfg.apiserver.port}";
203 type = types.str;
204 };
205
206 extraOpts = mkOption {
207 description = "Kubernetes scheduler extra command line options.";
208 default = "";
209 type = types.str;
210 };
211 };
212
213 controllerManager = {
214 enable = mkOption {
215 description = "Whether to enable kubernetes controller manager.";
216 default = false;
217 type = types.bool;
218 };
219
220 address = mkOption {
221 description = "Kubernetes controller manager listening address.";
222 default = "127.0.0.1";
223 type = types.str;
224 };
225
226 port = mkOption {
227 description = "Kubernets controller manager listening port.";
228 default = 10252;
229 type = types.int;
230 };
231
232 master = mkOption {
233 description = "Kubernetes apiserver address";
234 default = "${cfg.apiserver.address}:${toString cfg.apiserver.port}";
235 type = types.str;
236 };
237
238 extraOpts = mkOption {
239 description = "Kubernetes controller extra command line options.";
240 default = "";
241 type = types.str;
242 };
243 };
244
245 kubelet = {
246 enable = mkOption {
247 description = "Whether to enable kubernetes kubelet.";
248 default = false;
249 type = types.bool;
250 };
251
252 registerNode = mkOption {
253 description = "Whether to auto register kubelet with API server.";
254 default = true;
255 type = types.bool;
256 };
257
258 address = mkOption {
259 description = "Kubernetes kubelet info server listening address.";
260 default = "0.0.0.0";
261 type = types.str;
262 };
263
264 port = mkOption {
265 description = "Kubernets kubelet info server listening port.";
266 default = 10250;
267 type = types.int;
268 };
269
270 healthz = {
271 bind = mkOption {
272 description = "Kubernetes kubelet healthz listening address.";
273 default = "127.0.0.1";
274 type = types.str;
275 };
276
277 port = mkOption {
278 description = "Kubernetes kubelet healthz port.";
279 default = 10248;
280 type = types.int;
281 };
282 };
283
284 hostname = mkOption {
285 description = "Kubernetes kubelet hostname override";
286 default = config.networking.hostName;
287 type = types.str;
288 };
289
290 allowPrivileged = mkOption {
291 description = "Whether to allow kubernetes containers to request privileged mode.";
292 default = false;
293 type = types.bool;
294 };
295
296 apiServers = mkOption {
297 description = "Kubernetes kubelet list of Kubernetes API servers for publishing events, and reading pods and services.";
298 default = ["${cfg.apiserver.address}:${toString cfg.apiserver.port}"];
299 type = types.listOf types.str;
300 };
301
302 cadvisorPort = mkOption {
303 description = "Kubernetes kubelet local cadvisor port.";
304 default = 4194;
305 type = types.int;
306 };
307
308 clusterDns = mkOption {
309 description = "Use alternative dns.";
310 default = "";
311 type = types.str;
312 };
313
314 clusterDomain = mkOption {
315 description = "Use alternative domain.";
316 default = "kubernetes.io";
317 type = types.str;
318 };
319
320 extraOpts = mkOption {
321 description = "Kubernetes kubelet extra command line options.";
322 default = "";
323 type = types.str;
324 };
325 };
326
327 proxy = {
328 enable = mkOption {
329 description = "Whether to enable kubernetes proxy.";
330 default = false;
331 type = types.bool;
332 };
333
334 address = mkOption {
335 description = "Kubernetes proxy listening address.";
336 default = "0.0.0.0";
337 type = types.str;
338 };
339
340 master = mkOption {
341 description = "Kubernetes apiserver address";
342 default = "${cfg.apiserver.address}:${toString cfg.apiserver.port}";
343 type = types.str;
344 };
345
346 extraOpts = mkOption {
347 description = "Kubernetes proxy extra command line options.";
348 default = "";
349 type = types.str;
350 };
351 };
352
353 kube2sky = {
354 enable = mkEnableOption "Whether to enable kube2sky dns service.";
355
356 domain = mkOption {
357 description = "Kuberntes kube2sky domain under which all DNS names will be hosted.";
358 default = cfg.kubelet.clusterDomain;
359 type = types.str;
360 };
361
362 master = mkOption {
363 description = "Kubernetes apiserver address";
364 default = "${cfg.apiserver.address}:${toString cfg.apiserver.port}";
365 type = types.str;
366 };
367
368 extraOpts = mkOption {
369 description = "Kubernetes kube2sky extra command line options.";
370 default = "";
371 type = types.str;
372 };
373 };
374 };
375
376 ###### implementation
377
378 config = mkMerge [
379 (mkIf cfg.apiserver.enable {
380 systemd.services.kube-apiserver = {
381 description = "Kubernetes Api Server";
382 wantedBy = [ "multi-user.target" ];
383 requires = ["kubernetes-setup.service"];
384 after = [ "network-interfaces.target" "etcd.service" ];
385 serviceConfig = {
386 ExecStart = let
387 authorizationPolicyFile =
388 pkgs.writeText "kubernetes-policy"
389 (builtins.toJSON cfg.apiserver.authorizationPolicy);
390 tokenAuthFile =
391 pkgs.writeText "kubernetes-auth"
392 (concatImapStringsSep "\n" (i: v: v + "," + (toString i))
393 (mapAttrsToList (name: token: token + "," + name) cfg.apiserver.tokenAuth));
394 in ''${cfg.package}/bin/kube-apiserver \
395 --etcd-servers=${concatMapStringsSep "," (f: "http://${f}") cfg.etcdServers} \
396 --insecure-bind-address=${cfg.apiserver.address} \
397 --insecure-port=${toString cfg.apiserver.port} \
398 --bind-address=${cfg.apiserver.publicAddress} \
399 --allow-privileged=${if cfg.apiserver.allowPrivileged then "true" else "false"} \
400 ${optionalString (cfg.apiserver.tlsCertFile!="")
401 "--tls-cert-file=${cfg.apiserver.tlsCertFile}"} \
402 ${optionalString (cfg.apiserver.tlsPrivateKeyFile!="")
403 "--tls-private-key-file=${cfg.apiserver.tlsPrivateKeyFile}"} \
404 ${optionalString (cfg.apiserver.tokenAuth!=[])
405 "--token-auth-file=${tokenAuthFile}"} \
406 ${optionalString (cfg.apiserver.clientCaFile!="")
407 "--client-ca-file=${cfg.apiserver.clientCaFile}"} \
408 --authorization-mode=${cfg.apiserver.authorizationMode} \
409 ${optionalString (cfg.apiserver.authorizationMode == "ABAC")
410 "--authorization-policy-file=${authorizationPolicyFile}"} \
411 --secure-port=${toString cfg.apiserver.securePort} \
412 --service-cluster-ip-range=${cfg.apiserver.portalNet} \
413 ${optionalString (cfg.apiserver.runtimeConfig!="")
414 "--runtime-config=${cfg.apiserver.runtimeConfig}"} \
415 --admission_control=${concatStringsSep "," cfg.apiserver.admissionControl} \
416 --logtostderr=true \
417 ${optionalString cfg.verbose "--v=6 --log-flush-frequency=1s"} \
418 ${cfg.apiserver.extraOpts}
419 '';
420 User = "kubernetes";
421 };
422 postStart = ''
423 until ${pkgs.curl}/bin/curl -s -o /dev/null 'http://${cfg.apiserver.address}:${toString cfg.apiserver.port}/'; do
424 sleep 1;
425 done
426 '';
427 };
428 })
429
430 (mkIf cfg.scheduler.enable {
431 systemd.services.kube-scheduler = {
432 description = "Kubernetes Scheduler Service";
433 wantedBy = [ "multi-user.target" ];
434 after = [ "network-interfaces.target" "kubernetes-apiserver.service" ];
435 serviceConfig = {
436 ExecStart = ''${cfg.package}/bin/kube-scheduler \
437 --address=${cfg.scheduler.address} \
438 --port=${toString cfg.scheduler.port} \
439 --master=${cfg.scheduler.master} \
440 --logtostderr=true \
441 ${optionalString cfg.verbose "--v=6 --log-flush-frequency=1s"} \
442 ${cfg.scheduler.extraOpts}
443 '';
444 User = "kubernetes";
445 };
446 };
447 })
448
449 (mkIf cfg.controllerManager.enable {
450 systemd.services.kube-controller-manager = {
451 description = "Kubernetes Controller Manager Service";
452 wantedBy = [ "multi-user.target" ];
453 after = [ "network-interfaces.target" "kubernetes-apiserver.service" ];
454 serviceConfig = {
455 ExecStart = ''${cfg.package}/bin/kube-controller-manager \
456 --address=${cfg.controllerManager.address} \
457 --port=${toString cfg.controllerManager.port} \
458 --master=${cfg.controllerManager.master} \
459 --logtostderr=true \
460 ${optionalString cfg.verbose "--v=6 --log-flush-frequency=1s"} \
461 ${cfg.controllerManager.extraOpts}
462 '';
463 User = "kubernetes";
464 };
465 };
466 })
467
468 (mkIf cfg.kubelet.enable {
469 systemd.services.kubelet = {
470 description = "Kubernetes Kubelet Service";
471 wantedBy = [ "multi-user.target" ];
472 requires = ["kubernetes-setup.service"];
473 after = [ "network-interfaces.target" "etcd.service" "docker.service" ];
474 script = ''
475 export PATH="/bin:/sbin:/usr/bin:/usr/sbin:$PATH"
476 exec ${cfg.package}/bin/kubelet \
477 --api-servers=${concatMapStringsSep "," (f: "http://${f}") cfg.kubelet.apiServers} \
478 --register-node=${if cfg.kubelet.registerNode then "true" else "false"} \
479 --address=${cfg.kubelet.address} \
480 --port=${toString cfg.kubelet.port} \
481 --healthz-bind-address=${cfg.kubelet.healthz.bind} \
482 --healthz-port=${toString cfg.kubelet.healthz.port} \
483 --hostname-override=${cfg.kubelet.hostname} \
484 --allow-privileged=${if cfg.kubelet.allowPrivileged then "true" else "false"} \
485 --root-dir=${cfg.dataDir} \
486 --cadvisor_port=${toString cfg.kubelet.cadvisorPort} \
487 ${optionalString (cfg.kubelet.clusterDns != "")
488 ''--cluster-dns=${cfg.kubelet.clusterDns}''} \
489 ${optionalString (cfg.kubelet.clusterDomain != "")
490 ''--cluster-domain=${cfg.kubelet.clusterDomain}''} \
491 --logtostderr=true \
492 ${optionalString cfg.verbose "--v=6 --log_flush_frequency=1s"} \
493 ${cfg.kubelet.extraOpts}
494 '';
495 serviceConfig.WorkingDirectory = cfg.dataDir;
496 };
497 })
498
499 (mkIf cfg.proxy.enable {
500 systemd.services.kube-proxy = {
501 description = "Kubernetes Proxy Service";
502 wantedBy = [ "multi-user.target" ];
503 after = [ "network-interfaces.target" "etcd.service" ];
504 serviceConfig = {
505 ExecStart = ''${cfg.package}/bin/kube-proxy \
506 --master=${cfg.proxy.master} \
507 --bind-address=${cfg.proxy.address} \
508 --logtostderr=true \
509 ${optionalString cfg.verbose "--v=6 --log-flush-frequency=1s"} \
510 ${cfg.proxy.extraOpts}
511 '';
512 };
513 };
514 })
515
516 (mkIf cfg.kube2sky.enable {
517 systemd.services.kube2sky = {
518 description = "Kubernetes Dns Bridge Service";
519 wantedBy = [ "multi-user.target" ];
520 after = [ "network.target" "skydns.service" "etcd.service" "kubernetes-apiserver.service" ];
521 serviceConfig = {
522 ExecStart = ''${cfg.package}/bin/kube2sky \
523 -etcd-server=http://${head cfg.etcdServers} \
524 -domain=${cfg.kube2sky.domain} \
525 -kube_master_url=http://${cfg.kube2sky.master} \
526 -logtostderr=true \
527 ${optionalString cfg.verbose "--v=6 --log-flush-frequency=1s"} \
528 ${cfg.kube2sky.extraOpts}
529 '';
530 User = "kubernetes";
531 };
532 };
533 })
534
535 (mkIf (any (el: el == "master") cfg.roles) {
536 services.kubernetes.apiserver.enable = mkDefault true;
537 services.kubernetes.scheduler.enable = mkDefault true;
538 services.kubernetes.controllerManager.enable = mkDefault true;
539 services.kubernetes.kube2sky.enable = mkDefault true;
540 })
541
542 (mkIf (any (el: el == "node") cfg.roles) {
543 virtualisation.docker.enable = mkDefault true;
544 services.kubernetes.kubelet.enable = mkDefault true;
545 services.kubernetes.proxy.enable = mkDefault true;
546 })
547
548 (mkIf (any (el: el == "node" || el == "master") cfg.roles) {
549 services.etcd.enable = mkDefault true;
550
551 services.skydns.enable = mkDefault true;
552 services.skydns.domain = mkDefault cfg.kubelet.clusterDomain;
553 })
554
555 (mkIf (
556 cfg.apiserver.enable ||
557 cfg.scheduler.enable ||
558 cfg.controllerManager.enable ||
559 cfg.kubelet.enable ||
560 cfg.proxy.enable
561 ) {
562 systemd.services.kubernetes-setup = {
563 description = "Kubernetes setup.";
564 serviceConfig.Type = "oneshot";
565 script = ''
566 mkdir -p /var/run/kubernetes
567 chown kubernetes /var/lib/kubernetes
568
569 rm ${cfg.dataDir}/.dockercfg || true
570 ln -fs ${pkgs.writeText "kubernetes-dockercfg" cfg.dockerCfg} ${cfg.dataDir}/.dockercfg
571 '';
572 };
573
574 services.kubernetes.package = mkDefault pkgs.kubernetes;
575
576 environment.systemPackages = [ cfg.package ];
577
578 users.extraUsers = singleton {
579 name = "kubernetes";
580 uid = config.ids.uids.kubernetes;
581 description = "Kubernetes user";
582 extraGroups = [ "docker" ];
583 group = "kubernetes";
584 home = cfg.dataDir;
585 createHome = true;
586 };
587 users.extraGroups.kubernetes.gid = config.ids.gids.kubernetes;
588 })
589
590 ];
591}