pyrox.dev
1{ config, lib, options, pkgs, ... }:
2
3with lib;
4
5let
6 top = config.services.kubernetes;
7 otop = options.services.kubernetes;
8 cfg = top.kubelet;
9
10 cniConfig =
11 if cfg.cni.config != [] && cfg.cni.configDir != null then
12 throw "Verbatim CNI-config and CNI configDir cannot both be set."
13 else if cfg.cni.configDir != null then
14 cfg.cni.configDir
15 else
16 (pkgs.buildEnv {
17 name = "kubernetes-cni-config";
18 paths = imap (i: entry:
19 pkgs.writeTextDir "${toString (10+i)}-${entry.type}.conf" (builtins.toJSON entry)
20 ) cfg.cni.config;
21 });
22
23 infraContainer = pkgs.dockerTools.buildImage {
24 name = "pause";
25 tag = "latest";
26 copyToRoot = pkgs.buildEnv {
27 name = "image-root";
28 pathsToLink = [ "/bin" ];
29 paths = [ top.package.pause ];
30 };
31 config.Cmd = ["/bin/pause"];
32 };
33
34 kubeconfig = top.lib.mkKubeConfig "kubelet" cfg.kubeconfig;
35
36 manifestPath = "kubernetes/manifests";
37
38 taintOptions = with lib.types; { name, ... }: {
39 options = {
40 key = mkOption {
41 description = lib.mdDoc "Key of taint.";
42 default = name;
43 defaultText = literalMD "Name of this submodule.";
44 type = str;
45 };
46 value = mkOption {
47 description = lib.mdDoc "Value of taint.";
48 type = str;
49 };
50 effect = mkOption {
51 description = lib.mdDoc "Effect of taint.";
52 example = "NoSchedule";
53 type = enum ["NoSchedule" "PreferNoSchedule" "NoExecute"];
54 };
55 };
56 };
57
58 taints = concatMapStringsSep "," (v: "${v.key}=${v.value}:${v.effect}") (mapAttrsToList (n: v: v) cfg.taints);
59in
60{
61 imports = [
62 (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "applyManifests" ] "")
63 (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "cadvisorPort" ] "")
64 (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "allowPrivileged" ] "")
65 (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "networkPlugin" ] "")
66 (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "containerRuntime" ] "")
67 ];
68
69 ###### interface
70 options.services.kubernetes.kubelet = with lib.types; {
71
72 address = mkOption {
73 description = lib.mdDoc "Kubernetes kubelet info server listening address.";
74 default = "0.0.0.0";
75 type = str;
76 };
77
78 clusterDns = mkOption {
79 description = lib.mdDoc "Use alternative DNS.";
80 default = "10.1.0.1";
81 type = str;
82 };
83
84 clusterDomain = mkOption {
85 description = lib.mdDoc "Use alternative domain.";
86 default = config.services.kubernetes.addons.dns.clusterDomain;
87 defaultText = literalExpression "config.${options.services.kubernetes.addons.dns.clusterDomain}";
88 type = str;
89 };
90
91 clientCaFile = mkOption {
92 description = lib.mdDoc "Kubernetes apiserver CA file for client authentication.";
93 default = top.caFile;
94 defaultText = literalExpression "config.${otop.caFile}";
95 type = nullOr path;
96 };
97
98 cni = {
99 packages = mkOption {
100 description = lib.mdDoc "List of network plugin packages to install.";
101 type = listOf package;
102 default = [];
103 };
104
105 config = mkOption {
106 description = lib.mdDoc "Kubernetes CNI configuration.";
107 type = listOf attrs;
108 default = [];
109 example = literalExpression ''
110 [{
111 "cniVersion": "0.3.1",
112 "name": "mynet",
113 "type": "bridge",
114 "bridge": "cni0",
115 "isGateway": true,
116 "ipMasq": true,
117 "ipam": {
118 "type": "host-local",
119 "subnet": "10.22.0.0/16",
120 "routes": [
121 { "dst": "0.0.0.0/0" }
122 ]
123 }
124 } {
125 "cniVersion": "0.3.1",
126 "type": "loopback"
127 }]
128 '';
129 };
130
131 configDir = mkOption {
132 description = lib.mdDoc "Path to Kubernetes CNI configuration directory.";
133 type = nullOr path;
134 default = null;
135 };
136 };
137
138 containerRuntimeEndpoint = mkOption {
139 description = lib.mdDoc "Endpoint at which to find the container runtime api interface/socket";
140 type = str;
141 default = "unix:///run/containerd/containerd.sock";
142 };
143
144 enable = mkEnableOption (lib.mdDoc "Kubernetes kubelet");
145
146 extraOpts = mkOption {
147 description = lib.mdDoc "Kubernetes kubelet extra command line options.";
148 default = "";
149 type = separatedString " ";
150 };
151
152 featureGates = mkOption {
153 description = lib.mdDoc "List set of feature gates";
154 default = top.featureGates;
155 defaultText = literalExpression "config.${otop.featureGates}";
156 type = listOf str;
157 };
158
159 healthz = {
160 bind = mkOption {
161 description = lib.mdDoc "Kubernetes kubelet healthz listening address.";
162 default = "127.0.0.1";
163 type = str;
164 };
165
166 port = mkOption {
167 description = lib.mdDoc "Kubernetes kubelet healthz port.";
168 default = 10248;
169 type = port;
170 };
171 };
172
173 hostname = mkOption {
174 description = lib.mdDoc "Kubernetes kubelet hostname override.";
175 defaultText = literalExpression "config.networking.fqdnOrHostName";
176 type = str;
177 };
178
179 kubeconfig = top.lib.mkKubeConfigOptions "Kubelet";
180
181 manifests = mkOption {
182 description = lib.mdDoc "List of manifests to bootstrap with kubelet (only pods can be created as manifest entry)";
183 type = attrsOf attrs;
184 default = {};
185 };
186
187 nodeIp = mkOption {
188 description = lib.mdDoc "IP address of the node. If set, kubelet will use this IP address for the node.";
189 default = null;
190 type = nullOr str;
191 };
192
193 registerNode = mkOption {
194 description = lib.mdDoc "Whether to auto register kubelet with API server.";
195 default = true;
196 type = bool;
197 };
198
199 port = mkOption {
200 description = lib.mdDoc "Kubernetes kubelet info server listening port.";
201 default = 10250;
202 type = port;
203 };
204
205 seedDockerImages = mkOption {
206 description = lib.mdDoc "List of docker images to preload on system";
207 default = [];
208 type = listOf package;
209 };
210
211 taints = mkOption {
212 description = lib.mdDoc "Node taints (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/).";
213 default = {};
214 type = attrsOf (submodule [ taintOptions ]);
215 };
216
217 tlsCertFile = mkOption {
218 description = lib.mdDoc "File containing x509 Certificate for HTTPS.";
219 default = null;
220 type = nullOr path;
221 };
222
223 tlsKeyFile = mkOption {
224 description = lib.mdDoc "File containing x509 private key matching tlsCertFile.";
225 default = null;
226 type = nullOr path;
227 };
228
229 unschedulable = mkOption {
230 description = lib.mdDoc "Whether to set node taint to unschedulable=true as it is the case of node that has only master role.";
231 default = false;
232 type = bool;
233 };
234
235 verbosity = mkOption {
236 description = lib.mdDoc ''
237 Optional glog verbosity level for logging statements. See
238 <https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md>
239 '';
240 default = null;
241 type = nullOr int;
242 };
243
244 };
245
246 ###### implementation
247 config = mkMerge [
248 (mkIf cfg.enable {
249
250 environment.etc."cni/net.d".source = cniConfig;
251
252 services.kubernetes.kubelet.seedDockerImages = [infraContainer];
253
254 boot.kernel.sysctl = {
255 "net.bridge.bridge-nf-call-iptables" = 1;
256 "net.ipv4.ip_forward" = 1;
257 "net.bridge.bridge-nf-call-ip6tables" = 1;
258 };
259
260 systemd.services.kubelet = {
261 description = "Kubernetes Kubelet Service";
262 wantedBy = [ "kubernetes.target" ];
263 after = [ "containerd.service" "network.target" "kube-apiserver.service" ];
264 path = with pkgs; [
265 gitMinimal
266 openssh
267 util-linux
268 iproute2
269 ethtool
270 thin-provisioning-tools
271 iptables
272 socat
273 ] ++ lib.optional config.boot.zfs.enabled config.boot.zfs.package ++ top.path;
274 preStart = ''
275 ${concatMapStrings (img: ''
276 echo "Seeding container image: ${img}"
277 ${if (lib.hasSuffix "gz" img) then
278 ''${pkgs.gzip}/bin/zcat "${img}" | ${pkgs.containerd}/bin/ctr -n k8s.io image import --all-platforms -''
279 else
280 ''${pkgs.coreutils}/bin/cat "${img}" | ${pkgs.containerd}/bin/ctr -n k8s.io image import --all-platforms -''
281 }
282 '') cfg.seedDockerImages}
283
284 rm /opt/cni/bin/* || true
285 ${concatMapStrings (package: ''
286 echo "Linking cni package: ${package}"
287 ln -fs ${package}/bin/* /opt/cni/bin
288 '') cfg.cni.packages}
289 '';
290 serviceConfig = {
291 Slice = "kubernetes.slice";
292 CPUAccounting = true;
293 MemoryAccounting = true;
294 Restart = "on-failure";
295 RestartSec = "1000ms";
296 ExecStart = ''${top.package}/bin/kubelet \
297 --address=${cfg.address} \
298 --authentication-token-webhook \
299 --authentication-token-webhook-cache-ttl="10s" \
300 --authorization-mode=Webhook \
301 ${optionalString (cfg.clientCaFile != null)
302 "--client-ca-file=${cfg.clientCaFile}"} \
303 ${optionalString (cfg.clusterDns != "")
304 "--cluster-dns=${cfg.clusterDns}"} \
305 ${optionalString (cfg.clusterDomain != "")
306 "--cluster-domain=${cfg.clusterDomain}"} \
307 ${optionalString (cfg.featureGates != [])
308 "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \
309 --hairpin-mode=hairpin-veth \
310 --healthz-bind-address=${cfg.healthz.bind} \
311 --healthz-port=${toString cfg.healthz.port} \
312 --hostname-override=${cfg.hostname} \
313 --kubeconfig=${kubeconfig} \
314 ${optionalString (cfg.nodeIp != null)
315 "--node-ip=${cfg.nodeIp}"} \
316 --pod-infra-container-image=pause \
317 ${optionalString (cfg.manifests != {})
318 "--pod-manifest-path=/etc/${manifestPath}"} \
319 --port=${toString cfg.port} \
320 --register-node=${boolToString cfg.registerNode} \
321 ${optionalString (taints != "")
322 "--register-with-taints=${taints}"} \
323 --root-dir=${top.dataDir} \
324 ${optionalString (cfg.tlsCertFile != null)
325 "--tls-cert-file=${cfg.tlsCertFile}"} \
326 ${optionalString (cfg.tlsKeyFile != null)
327 "--tls-private-key-file=${cfg.tlsKeyFile}"} \
328 ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
329 --container-runtime-endpoint=${cfg.containerRuntimeEndpoint} \
330 --cgroup-driver=systemd \
331 ${cfg.extraOpts}
332 '';
333 WorkingDirectory = top.dataDir;
334 };
335 unitConfig = {
336 StartLimitIntervalSec = 0;
337 };
338 };
339
340 # Always include cni plugins
341 services.kubernetes.kubelet.cni.packages = [pkgs.cni-plugins pkgs.cni-plugin-flannel];
342
343 boot.kernelModules = ["br_netfilter" "overlay"];
344
345 services.kubernetes.kubelet.hostname =
346 mkDefault config.networking.fqdnOrHostName;
347
348 services.kubernetes.pki.certs = with top.lib; {
349 kubelet = mkCert {
350 name = "kubelet";
351 CN = top.kubelet.hostname;
352 action = "systemctl restart kubelet.service";
353
354 };
355 kubeletClient = mkCert {
356 name = "kubelet-client";
357 CN = "system:node:${top.kubelet.hostname}";
358 fields = {
359 O = "system:nodes";
360 };
361 action = "systemctl restart kubelet.service";
362 };
363 };
364
365 services.kubernetes.kubelet.kubeconfig.server = mkDefault top.apiserverAddress;
366 })
367
368 (mkIf (cfg.enable && cfg.manifests != {}) {
369 environment.etc = mapAttrs' (name: manifest:
370 nameValuePair "${manifestPath}/${name}.json" {
371 text = builtins.toJSON manifest;
372 mode = "0755";
373 }
374 ) cfg.manifests;
375 })
376
377 (mkIf (cfg.unschedulable && cfg.enable) {
378 services.kubernetes.kubelet.taints.unschedulable = {
379 value = "true";
380 effect = "NoSchedule";
381 };
382 })
383
384 ];
385
386 meta.buildDocsInSandbox = false;
387}