pyrox.dev
1{
2 config,
3 lib,
4 options,
5 pkgs,
6 ...
7}:
8
9with lib;
10
11let
12 top = config.services.kubernetes;
13 otop = options.services.kubernetes;
14 cfg = top.kubelet;
15
16 cniConfig =
17 if cfg.cni.config != [ ] && cfg.cni.configDir != null then
18 throw "Verbatim CNI-config and CNI configDir cannot both be set."
19 else if cfg.cni.configDir != null then
20 cfg.cni.configDir
21 else
22 (pkgs.buildEnv {
23 name = "kubernetes-cni-config";
24 paths = imap (
25 i: entry: pkgs.writeTextDir "${toString (10 + i)}-${entry.type}.conf" (builtins.toJSON entry)
26 ) cfg.cni.config;
27 });
28
29 infraContainer = pkgs.dockerTools.buildImage {
30 name = "pause";
31 tag = "latest";
32 copyToRoot = pkgs.buildEnv {
33 name = "image-root";
34 pathsToLink = [ "/bin" ];
35 paths = [ top.package.pause ];
36 };
37 config.Cmd = [ "/bin/pause" ];
38 };
39
40 kubeconfig = top.lib.mkKubeConfig "kubelet" cfg.kubeconfig;
41
42 # Flag based settings are deprecated, use the `--config` flag with a
43 # `KubeletConfiguration` struct.
44 # https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/
45 #
46 # NOTE: registerWithTaints requires a []core/v1.Taint, therefore requires
47 # additional work to be put in config format.
48 #
49 kubeletConfig = pkgs.writeText "kubelet-config" (
50 builtins.toJSON (
51 {
52 apiVersion = "kubelet.config.k8s.io/v1beta1";
53 kind = "KubeletConfiguration";
54 address = cfg.address;
55 port = cfg.port;
56 authentication = {
57 x509 = lib.optionalAttrs (cfg.clientCaFile != null) { clientCAFile = cfg.clientCaFile; };
58 webhook = {
59 enabled = true;
60 cacheTTL = "10s";
61 };
62 };
63 authorization = {
64 mode = "Webhook";
65 };
66 cgroupDriver = "systemd";
67 hairpinMode = "hairpin-veth";
68 registerNode = cfg.registerNode;
69 containerRuntimeEndpoint = cfg.containerRuntimeEndpoint;
70 healthzPort = cfg.healthz.port;
71 healthzBindAddress = cfg.healthz.bind;
72 }
73 // lib.optionalAttrs (cfg.tlsCertFile != null) { tlsCertFile = cfg.tlsCertFile; }
74 // lib.optionalAttrs (cfg.tlsKeyFile != null) { tlsPrivateKeyFile = cfg.tlsKeyFile; }
75 // lib.optionalAttrs (cfg.clusterDomain != "") { clusterDomain = cfg.clusterDomain; }
76 // lib.optionalAttrs (cfg.clusterDns != [ ]) { clusterDNS = cfg.clusterDns; }
77 // lib.optionalAttrs (cfg.featureGates != { }) { featureGates = cfg.featureGates; }
78 // lib.optionalAttrs (cfg.extraConfig != { }) cfg.extraConfig
79 )
80 );
81
82 manifestPath = "kubernetes/manifests";
83
84 taintOptions =
85 with lib.types;
86 { name, ... }:
87 {
88 options = {
89 key = mkOption {
90 description = "Key of taint.";
91 default = name;
92 defaultText = literalMD "Name of this submodule.";
93 type = str;
94 };
95 value = mkOption {
96 description = "Value of taint.";
97 type = str;
98 };
99 effect = mkOption {
100 description = "Effect of taint.";
101 example = "NoSchedule";
102 type = enum [
103 "NoSchedule"
104 "PreferNoSchedule"
105 "NoExecute"
106 ];
107 };
108 };
109 };
110
111 taints = concatMapStringsSep "," (v: "${v.key}=${v.value}:${v.effect}") (
112 mapAttrsToList (n: v: v) cfg.taints
113 );
114in
115{
116 imports = [
117 (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "applyManifests" ] "")
118 (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "cadvisorPort" ] "")
119 (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "allowPrivileged" ] "")
120 (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "networkPlugin" ] "")
121 (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "containerRuntime" ] "")
122 ];
123
124 ###### interface
125 options.services.kubernetes.kubelet = with lib.types; {
126
127 address = mkOption {
128 description = "Kubernetes kubelet info server listening address.";
129 default = "0.0.0.0";
130 type = str;
131 };
132
133 clusterDns = mkOption {
134 description = "Use alternative DNS.";
135 default = [ "10.1.0.1" ];
136 type = listOf str;
137 };
138
139 clusterDomain = mkOption {
140 description = "Use alternative domain.";
141 default = config.services.kubernetes.addons.dns.clusterDomain;
142 defaultText = literalExpression "config.${options.services.kubernetes.addons.dns.clusterDomain}";
143 type = str;
144 };
145
146 clientCaFile = mkOption {
147 description = "Kubernetes apiserver CA file for client authentication.";
148 default = top.caFile;
149 defaultText = literalExpression "config.${otop.caFile}";
150 type = nullOr path;
151 };
152
153 cni = {
154 packages = mkOption {
155 description = "List of network plugin packages to install.";
156 type = listOf package;
157 default = [ ];
158 };
159
160 config = mkOption {
161 description = "Kubernetes CNI configuration.";
162 type = listOf attrs;
163 default = [ ];
164 example = literalExpression ''
165 [{
166 "cniVersion": "0.3.1",
167 "name": "mynet",
168 "type": "bridge",
169 "bridge": "cni0",
170 "isGateway": true,
171 "ipMasq": true,
172 "ipam": {
173 "type": "host-local",
174 "subnet": "10.22.0.0/16",
175 "routes": [
176 { "dst": "0.0.0.0/0" }
177 ]
178 }
179 } {
180 "cniVersion": "0.3.1",
181 "type": "loopback"
182 }]
183 '';
184 };
185
186 configDir = mkOption {
187 description = "Path to Kubernetes CNI configuration directory.";
188 type = nullOr path;
189 default = null;
190 };
191 };
192
193 containerRuntimeEndpoint = mkOption {
194 description = "Endpoint at which to find the container runtime api interface/socket";
195 type = str;
196 default = "unix:///run/containerd/containerd.sock";
197 };
198
199 enable = mkEnableOption "Kubernetes kubelet";
200
201 extraOpts = mkOption {
202 description = "Kubernetes kubelet extra command line options.";
203 default = "";
204 type = separatedString " ";
205 };
206
207 extraConfig = mkOption {
208 description = ''
209 Kubernetes kubelet extra configuration file entries.
210
211 See also [Set Kubelet Parameters Via A Configuration File](https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/)
212 and [Kubelet Configuration](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/).
213 '';
214 default = { };
215 type = attrsOf ((pkgs.formats.json { }).type);
216 };
217
218 featureGates = mkOption {
219 description = "Attribute set of feature gate";
220 default = top.featureGates;
221 defaultText = literalExpression "config.${otop.featureGates}";
222 type = attrsOf bool;
223 };
224
225 healthz = {
226 bind = mkOption {
227 description = "Kubernetes kubelet healthz listening address.";
228 default = "127.0.0.1";
229 type = str;
230 };
231
232 port = mkOption {
233 description = "Kubernetes kubelet healthz port.";
234 default = 10248;
235 type = port;
236 };
237 };
238
239 hostname = mkOption {
240 description = "Kubernetes kubelet hostname override.";
241 defaultText = literalExpression "config.networking.fqdnOrHostName";
242 type = str;
243 };
244
245 kubeconfig = top.lib.mkKubeConfigOptions "Kubelet";
246
247 manifests = mkOption {
248 description = "List of manifests to bootstrap with kubelet (only pods can be created as manifest entry)";
249 type = attrsOf attrs;
250 default = { };
251 };
252
253 nodeIp = mkOption {
254 description = "IP address of the node. If set, kubelet will use this IP address for the node.";
255 default = null;
256 type = nullOr str;
257 };
258
259 registerNode = mkOption {
260 description = "Whether to auto register kubelet with API server.";
261 default = true;
262 type = bool;
263 };
264
265 port = mkOption {
266 description = "Kubernetes kubelet info server listening port.";
267 default = 10250;
268 type = port;
269 };
270
271 seedDockerImages = mkOption {
272 description = "List of docker images to preload on system";
273 default = [ ];
274 type = listOf package;
275 };
276
277 taints = mkOption {
278 description = "Node taints (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/).";
279 default = { };
280 type = attrsOf (submodule [ taintOptions ]);
281 };
282
283 tlsCertFile = mkOption {
284 description = "File containing x509 Certificate for HTTPS.";
285 default = null;
286 type = nullOr path;
287 };
288
289 tlsKeyFile = mkOption {
290 description = "File containing x509 private key matching tlsCertFile.";
291 default = null;
292 type = nullOr path;
293 };
294
295 unschedulable = mkOption {
296 description = "Whether to set node taint to unschedulable=true as it is the case of node that has only master role.";
297 default = false;
298 type = bool;
299 };
300
301 verbosity = mkOption {
302 description = ''
303 Optional glog verbosity level for logging statements. See
304 <https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md>
305 '';
306 default = null;
307 type = nullOr int;
308 };
309
310 };
311
312 ###### implementation
313 config = mkMerge [
314 (mkIf cfg.enable {
315
316 environment.etc."cni/net.d".source = cniConfig;
317
318 services.kubernetes.kubelet.seedDockerImages = [ infraContainer ];
319
320 boot.kernel.sysctl = {
321 "net.bridge.bridge-nf-call-iptables" = 1;
322 "net.ipv4.ip_forward" = 1;
323 "net.bridge.bridge-nf-call-ip6tables" = 1;
324 };
325
326 systemd.services.kubelet = {
327 description = "Kubernetes Kubelet Service";
328 wantedBy = [ "kubernetes.target" ];
329 after = [
330 "containerd.service"
331 "network.target"
332 "kube-apiserver.service"
333 ];
334 path =
335 with pkgs;
336 [
337 gitMinimal
338 openssh
339 util-linux
340 iproute2
341 ethtool
342 thin-provisioning-tools
343 iptables
344 socat
345 ]
346 ++ lib.optional config.boot.zfs.enabled config.boot.zfs.package
347 ++ top.path;
348 preStart = ''
349 ${concatMapStrings (img: ''
350 echo "Seeding container image: ${img}"
351 ${
352 if (lib.hasSuffix "gz" img) then
353 ''${pkgs.gzip}/bin/zcat "${img}" | ${pkgs.containerd}/bin/ctr -n k8s.io image import -''
354 else
355 ''${pkgs.coreutils}/bin/cat "${img}" | ${pkgs.containerd}/bin/ctr -n k8s.io image import -''
356 }
357 '') cfg.seedDockerImages}
358
359 rm /opt/cni/bin/* || true
360 ${concatMapStrings (package: ''
361 echo "Linking cni package: ${package}"
362 ln -fs ${package}/bin/* /opt/cni/bin
363 '') cfg.cni.packages}
364 '';
365 serviceConfig = {
366 Slice = "kubernetes.slice";
367 CPUAccounting = true;
368 MemoryAccounting = true;
369 Restart = "on-failure";
370 RestartSec = "1000ms";
371 ExecStart = ''
372 ${top.package}/bin/kubelet \
373 --config=${kubeletConfig} \
374 --hostname-override=${cfg.hostname} \
375 --kubeconfig=${kubeconfig} \
376 ${optionalString (cfg.nodeIp != null) "--node-ip=${cfg.nodeIp}"} \
377 --pod-infra-container-image=pause \
378 ${optionalString (cfg.manifests != { }) "--pod-manifest-path=/etc/${manifestPath}"} \
379 ${optionalString (taints != "") "--register-with-taints=${taints}"} \
380 --root-dir=${top.dataDir} \
381 ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
382 ${cfg.extraOpts}
383 '';
384 WorkingDirectory = top.dataDir;
385 };
386 unitConfig = {
387 StartLimitIntervalSec = 0;
388 };
389 };
390
391 # Always include cni plugins
392 services.kubernetes.kubelet.cni.packages = [
393 pkgs.cni-plugins
394 pkgs.cni-plugin-flannel
395 ];
396
397 boot.kernelModules = [
398 "br_netfilter"
399 "overlay"
400 ];
401
402 services.kubernetes.kubelet.hostname = mkDefault (lib.toLower config.networking.fqdnOrHostName);
403
404 services.kubernetes.pki.certs = with top.lib; {
405 kubelet = mkCert {
406 name = "kubelet";
407 CN = top.kubelet.hostname;
408 action = "systemctl restart kubelet.service";
409
410 };
411 kubeletClient = mkCert {
412 name = "kubelet-client";
413 CN = "system:node:${top.kubelet.hostname}";
414 fields = {
415 O = "system:nodes";
416 };
417 action = "systemctl restart kubelet.service";
418 };
419 };
420
421 services.kubernetes.kubelet.kubeconfig.server = mkDefault top.apiserverAddress;
422 })
423
424 (mkIf (cfg.enable && cfg.manifests != { }) {
425 environment.etc = mapAttrs' (
426 name: manifest:
427 nameValuePair "${manifestPath}/${name}.json" {
428 text = builtins.toJSON manifest;
429 mode = "0755";
430 }
431 ) cfg.manifests;
432 })
433
434 (mkIf (cfg.unschedulable && cfg.enable) {
435 services.kubernetes.kubelet.taints.unschedulable = {
436 value = "true";
437 effect = "NoSchedule";
438 };
439 })
440
441 ];
442
443 meta.buildDocsInSandbox = false;
444}