pyrox.dev
1{ config, lib, options, pkgs, ... }:
2
3with lib;
4
5let
6 top = config.services.kubernetes;
7 otop = options.services.kubernetes;
8 cfg = top.kubelet;
9
10 cniConfig =
11 if cfg.cni.config != [] && cfg.cni.configDir != null then
12 throw "Verbatim CNI-config and CNI configDir cannot both be set."
13 else if cfg.cni.configDir != null then
14 cfg.cni.configDir
15 else
16 (pkgs.buildEnv {
17 name = "kubernetes-cni-config";
18 paths = imap (i: entry:
19 pkgs.writeTextDir "${toString (10+i)}-${entry.type}.conf" (builtins.toJSON entry)
20 ) cfg.cni.config;
21 });
22
23 infraContainer = pkgs.dockerTools.buildImage {
24 name = "pause";
25 tag = "latest";
26 copyToRoot = pkgs.buildEnv {
27 name = "image-root";
28 pathsToLink = [ "/bin" ];
29 paths = [ top.package.pause ];
30 };
31 config.Cmd = ["/bin/pause"];
32 };
33
34 kubeconfig = top.lib.mkKubeConfig "kubelet" cfg.kubeconfig;
35
36 # Flag based settings are deprecated, use the `--config` flag with a
37 # `KubeletConfiguration` struct.
38 # https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/
39 #
40 # NOTE: registerWithTaints requires a []core/v1.Taint, therefore requires
41 # additional work to be put in config format.
42 #
43 kubeletConfig = pkgs.writeText "kubelet-config" (builtins.toJSON ({
44 apiVersion = "kubelet.config.k8s.io/v1beta1";
45 kind = "KubeletConfiguration";
46 address = cfg.address;
47 port = cfg.port;
48 authentication = {
49 x509 = lib.optionalAttrs (cfg.clientCaFile != null) { clientCAFile = cfg.clientCaFile; };
50 webhook = {
51 enabled = true;
52 cacheTTL = "10s";
53 };
54 };
55 authorization = {
56 mode = "Webhook";
57 };
58 cgroupDriver = "systemd";
59 hairpinMode = "hairpin-veth";
60 registerNode = cfg.registerNode;
61 containerRuntimeEndpoint = cfg.containerRuntimeEndpoint;
62 healthzPort = cfg.healthz.port;
63 healthzBindAddress = cfg.healthz.bind;
64 } // lib.optionalAttrs (cfg.tlsCertFile != null) { tlsCertFile = cfg.tlsCertFile; }
65 // lib.optionalAttrs (cfg.tlsKeyFile != null) { tlsPrivateKeyFile = cfg.tlsKeyFile; }
66 // lib.optionalAttrs (cfg.clusterDomain != "") { clusterDomain = cfg.clusterDomain; }
67 // lib.optionalAttrs (cfg.clusterDns != "") { clusterDNS = [ cfg.clusterDns ] ; }
68 // lib.optionalAttrs (cfg.featureGates != []) { featureGates = cfg.featureGates; }
69 ));
70
71 manifestPath = "kubernetes/manifests";
72
73 taintOptions = with lib.types; { name, ... }: {
74 options = {
75 key = mkOption {
76 description = "Key of taint.";
77 default = name;
78 defaultText = literalMD "Name of this submodule.";
79 type = str;
80 };
81 value = mkOption {
82 description = "Value of taint.";
83 type = str;
84 };
85 effect = mkOption {
86 description = "Effect of taint.";
87 example = "NoSchedule";
88 type = enum ["NoSchedule" "PreferNoSchedule" "NoExecute"];
89 };
90 };
91 };
92
93 taints = concatMapStringsSep "," (v: "${v.key}=${v.value}:${v.effect}") (mapAttrsToList (n: v: v) cfg.taints);
94in
95{
96 imports = [
97 (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "applyManifests" ] "")
98 (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "cadvisorPort" ] "")
99 (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "allowPrivileged" ] "")
100 (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "networkPlugin" ] "")
101 (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "containerRuntime" ] "")
102 ];
103
104 ###### interface
105 options.services.kubernetes.kubelet = with lib.types; {
106
107 address = mkOption {
108 description = "Kubernetes kubelet info server listening address.";
109 default = "0.0.0.0";
110 type = str;
111 };
112
113 clusterDns = mkOption {
114 description = "Use alternative DNS.";
115 default = "10.1.0.1";
116 type = str;
117 };
118
119 clusterDomain = mkOption {
120 description = "Use alternative domain.";
121 default = config.services.kubernetes.addons.dns.clusterDomain;
122 defaultText = literalExpression "config.${options.services.kubernetes.addons.dns.clusterDomain}";
123 type = str;
124 };
125
126 clientCaFile = mkOption {
127 description = "Kubernetes apiserver CA file for client authentication.";
128 default = top.caFile;
129 defaultText = literalExpression "config.${otop.caFile}";
130 type = nullOr path;
131 };
132
133 cni = {
134 packages = mkOption {
135 description = "List of network plugin packages to install.";
136 type = listOf package;
137 default = [];
138 };
139
140 config = mkOption {
141 description = "Kubernetes CNI configuration.";
142 type = listOf attrs;
143 default = [];
144 example = literalExpression ''
145 [{
146 "cniVersion": "0.3.1",
147 "name": "mynet",
148 "type": "bridge",
149 "bridge": "cni0",
150 "isGateway": true,
151 "ipMasq": true,
152 "ipam": {
153 "type": "host-local",
154 "subnet": "10.22.0.0/16",
155 "routes": [
156 { "dst": "0.0.0.0/0" }
157 ]
158 }
159 } {
160 "cniVersion": "0.3.1",
161 "type": "loopback"
162 }]
163 '';
164 };
165
166 configDir = mkOption {
167 description = "Path to Kubernetes CNI configuration directory.";
168 type = nullOr path;
169 default = null;
170 };
171 };
172
173 containerRuntimeEndpoint = mkOption {
174 description = "Endpoint at which to find the container runtime api interface/socket";
175 type = str;
176 default = "unix:///run/containerd/containerd.sock";
177 };
178
179 enable = mkEnableOption "Kubernetes kubelet";
180
181 extraOpts = mkOption {
182 description = "Kubernetes kubelet extra command line options.";
183 default = "";
184 type = separatedString " ";
185 };
186
187 featureGates = mkOption {
188 description = "List set of feature gates";
189 default = top.featureGates;
190 defaultText = literalExpression "config.${otop.featureGates}";
191 type = listOf str;
192 };
193
194 healthz = {
195 bind = mkOption {
196 description = "Kubernetes kubelet healthz listening address.";
197 default = "127.0.0.1";
198 type = str;
199 };
200
201 port = mkOption {
202 description = "Kubernetes kubelet healthz port.";
203 default = 10248;
204 type = port;
205 };
206 };
207
208 hostname = mkOption {
209 description = "Kubernetes kubelet hostname override.";
210 defaultText = literalExpression "config.networking.fqdnOrHostName";
211 type = str;
212 };
213
214 kubeconfig = top.lib.mkKubeConfigOptions "Kubelet";
215
216 manifests = mkOption {
217 description = "List of manifests to bootstrap with kubelet (only pods can be created as manifest entry)";
218 type = attrsOf attrs;
219 default = {};
220 };
221
222 nodeIp = mkOption {
223 description = "IP address of the node. If set, kubelet will use this IP address for the node.";
224 default = null;
225 type = nullOr str;
226 };
227
228 registerNode = mkOption {
229 description = "Whether to auto register kubelet with API server.";
230 default = true;
231 type = bool;
232 };
233
234 port = mkOption {
235 description = "Kubernetes kubelet info server listening port.";
236 default = 10250;
237 type = port;
238 };
239
240 seedDockerImages = mkOption {
241 description = "List of docker images to preload on system";
242 default = [];
243 type = listOf package;
244 };
245
246 taints = mkOption {
247 description = "Node taints (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/).";
248 default = {};
249 type = attrsOf (submodule [ taintOptions ]);
250 };
251
252 tlsCertFile = mkOption {
253 description = "File containing x509 Certificate for HTTPS.";
254 default = null;
255 type = nullOr path;
256 };
257
258 tlsKeyFile = mkOption {
259 description = "File containing x509 private key matching tlsCertFile.";
260 default = null;
261 type = nullOr path;
262 };
263
264 unschedulable = mkOption {
265 description = "Whether to set node taint to unschedulable=true as it is the case of node that has only master role.";
266 default = false;
267 type = bool;
268 };
269
270 verbosity = mkOption {
271 description = ''
272 Optional glog verbosity level for logging statements. See
273 <https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md>
274 '';
275 default = null;
276 type = nullOr int;
277 };
278
279 };
280
281 ###### implementation
282 config = mkMerge [
283 (mkIf cfg.enable {
284
285 environment.etc."cni/net.d".source = cniConfig;
286
287 services.kubernetes.kubelet.seedDockerImages = [infraContainer];
288
289 boot.kernel.sysctl = {
290 "net.bridge.bridge-nf-call-iptables" = 1;
291 "net.ipv4.ip_forward" = 1;
292 "net.bridge.bridge-nf-call-ip6tables" = 1;
293 };
294
295 systemd.services.kubelet = {
296 description = "Kubernetes Kubelet Service";
297 wantedBy = [ "kubernetes.target" ];
298 after = [ "containerd.service" "network.target" "kube-apiserver.service" ];
299 path = with pkgs; [
300 gitMinimal
301 openssh
302 util-linux
303 iproute2
304 ethtool
305 thin-provisioning-tools
306 iptables
307 socat
308 ] ++ lib.optional config.boot.zfs.enabled config.boot.zfs.package ++ top.path;
309 preStart = ''
310 ${concatMapStrings (img: ''
311 echo "Seeding container image: ${img}"
312 ${if (lib.hasSuffix "gz" img) then
313 ''${pkgs.gzip}/bin/zcat "${img}" | ${pkgs.containerd}/bin/ctr -n k8s.io image import --all-platforms -''
314 else
315 ''${pkgs.coreutils}/bin/cat "${img}" | ${pkgs.containerd}/bin/ctr -n k8s.io image import --all-platforms -''
316 }
317 '') cfg.seedDockerImages}
318
319 rm /opt/cni/bin/* || true
320 ${concatMapStrings (package: ''
321 echo "Linking cni package: ${package}"
322 ln -fs ${package}/bin/* /opt/cni/bin
323 '') cfg.cni.packages}
324 '';
325 serviceConfig = {
326 Slice = "kubernetes.slice";
327 CPUAccounting = true;
328 MemoryAccounting = true;
329 Restart = "on-failure";
330 RestartSec = "1000ms";
331 ExecStart = ''${top.package}/bin/kubelet \
332 --config=${kubeletConfig} \
333 --hostname-override=${cfg.hostname} \
334 --kubeconfig=${kubeconfig} \
335 ${optionalString (cfg.nodeIp != null)
336 "--node-ip=${cfg.nodeIp}"} \
337 --pod-infra-container-image=pause \
338 ${optionalString (cfg.manifests != {})
339 "--pod-manifest-path=/etc/${manifestPath}"} \
340 ${optionalString (taints != "")
341 "--register-with-taints=${taints}"} \
342 --root-dir=${top.dataDir} \
343 ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
344 ${cfg.extraOpts}
345 '';
346 WorkingDirectory = top.dataDir;
347 };
348 unitConfig = {
349 StartLimitIntervalSec = 0;
350 };
351 };
352
353 # Always include cni plugins
354 services.kubernetes.kubelet.cni.packages = [pkgs.cni-plugins pkgs.cni-plugin-flannel];
355
356 boot.kernelModules = ["br_netfilter" "overlay"];
357
358 services.kubernetes.kubelet.hostname =
359 mkDefault config.networking.fqdnOrHostName;
360
361 services.kubernetes.pki.certs = with top.lib; {
362 kubelet = mkCert {
363 name = "kubelet";
364 CN = top.kubelet.hostname;
365 action = "systemctl restart kubelet.service";
366
367 };
368 kubeletClient = mkCert {
369 name = "kubelet-client";
370 CN = "system:node:${top.kubelet.hostname}";
371 fields = {
372 O = "system:nodes";
373 };
374 action = "systemctl restart kubelet.service";
375 };
376 };
377
378 services.kubernetes.kubelet.kubeconfig.server = mkDefault top.apiserverAddress;
379 })
380
381 (mkIf (cfg.enable && cfg.manifests != {}) {
382 environment.etc = mapAttrs' (name: manifest:
383 nameValuePair "${manifestPath}/${name}.json" {
384 text = builtins.toJSON manifest;
385 mode = "0755";
386 }
387 ) cfg.manifests;
388 })
389
390 (mkIf (cfg.unschedulable && cfg.enable) {
391 services.kubernetes.kubelet.taints.unschedulable = {
392 value = "true";
393 effect = "NoSchedule";
394 };
395 })
396
397 ];
398
399 meta.buildDocsInSandbox = false;
400}