pyrox.dev
1{ config, lib, options, pkgs, ... }:
2
3with lib;
4
5let
6 top = config.services.kubernetes;
7 otop = options.services.kubernetes;
8 cfg = top.kubelet;
9
10 cniConfig =
11 if cfg.cni.config != [] && cfg.cni.configDir != null then
12 throw "Verbatim CNI-config and CNI configDir cannot both be set."
13 else if cfg.cni.configDir != null then
14 cfg.cni.configDir
15 else
16 (pkgs.buildEnv {
17 name = "kubernetes-cni-config";
18 paths = imap (i: entry:
19 pkgs.writeTextDir "${toString (10+i)}-${entry.type}.conf" (builtins.toJSON entry)
20 ) cfg.cni.config;
21 });
22
23 infraContainer = pkgs.dockerTools.buildImage {
24 name = "pause";
25 tag = "latest";
26 copyToRoot = pkgs.buildEnv {
27 name = "image-root";
28 pathsToLink = [ "/bin" ];
29 paths = [ top.package.pause ];
30 };
31 config.Cmd = ["/bin/pause"];
32 };
33
34 kubeconfig = top.lib.mkKubeConfig "kubelet" cfg.kubeconfig;
35
36 manifestPath = "kubernetes/manifests";
37
38 taintOptions = with lib.types; { name, ... }: {
39 options = {
40 key = mkOption {
41 description = lib.mdDoc "Key of taint.";
42 default = name;
43 defaultText = literalMD "Name of this submodule.";
44 type = str;
45 };
46 value = mkOption {
47 description = lib.mdDoc "Value of taint.";
48 type = str;
49 };
50 effect = mkOption {
51 description = lib.mdDoc "Effect of taint.";
52 example = "NoSchedule";
53 type = enum ["NoSchedule" "PreferNoSchedule" "NoExecute"];
54 };
55 };
56 };
57
58 taints = concatMapStringsSep "," (v: "${v.key}=${v.value}:${v.effect}") (mapAttrsToList (n: v: v) cfg.taints);
59in
60{
61 imports = [
62 (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "applyManifests" ] "")
63 (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "cadvisorPort" ] "")
64 (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "allowPrivileged" ] "")
65 (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "networkPlugin" ] "")
66 ];
67
68 ###### interface
69 options.services.kubernetes.kubelet = with lib.types; {
70
71 address = mkOption {
72 description = lib.mdDoc "Kubernetes kubelet info server listening address.";
73 default = "0.0.0.0";
74 type = str;
75 };
76
77 clusterDns = mkOption {
78 description = lib.mdDoc "Use alternative DNS.";
79 default = "10.1.0.1";
80 type = str;
81 };
82
83 clusterDomain = mkOption {
84 description = lib.mdDoc "Use alternative domain.";
85 default = config.services.kubernetes.addons.dns.clusterDomain;
86 defaultText = literalExpression "config.${options.services.kubernetes.addons.dns.clusterDomain}";
87 type = str;
88 };
89
90 clientCaFile = mkOption {
91 description = lib.mdDoc "Kubernetes apiserver CA file for client authentication.";
92 default = top.caFile;
93 defaultText = literalExpression "config.${otop.caFile}";
94 type = nullOr path;
95 };
96
97 cni = {
98 packages = mkOption {
99 description = lib.mdDoc "List of network plugin packages to install.";
100 type = listOf package;
101 default = [];
102 };
103
104 config = mkOption {
105 description = lib.mdDoc "Kubernetes CNI configuration.";
106 type = listOf attrs;
107 default = [];
108 example = literalExpression ''
109 [{
110 "cniVersion": "0.3.1",
111 "name": "mynet",
112 "type": "bridge",
113 "bridge": "cni0",
114 "isGateway": true,
115 "ipMasq": true,
116 "ipam": {
117 "type": "host-local",
118 "subnet": "10.22.0.0/16",
119 "routes": [
120 { "dst": "0.0.0.0/0" }
121 ]
122 }
123 } {
124 "cniVersion": "0.3.1",
125 "type": "loopback"
126 }]
127 '';
128 };
129
130 configDir = mkOption {
131 description = lib.mdDoc "Path to Kubernetes CNI configuration directory.";
132 type = nullOr path;
133 default = null;
134 };
135 };
136
137 containerRuntime = mkOption {
138 description = lib.mdDoc "Which container runtime type to use";
139 type = enum ["docker" "remote"];
140 default = "remote";
141 };
142
143 containerRuntimeEndpoint = mkOption {
144 description = lib.mdDoc "Endpoint at which to find the container runtime api interface/socket";
145 type = str;
146 default = "unix:///run/containerd/containerd.sock";
147 };
148
149 enable = mkEnableOption (lib.mdDoc "Kubernetes kubelet.");
150
151 extraOpts = mkOption {
152 description = lib.mdDoc "Kubernetes kubelet extra command line options.";
153 default = "";
154 type = separatedString " ";
155 };
156
157 featureGates = mkOption {
158 description = lib.mdDoc "List set of feature gates";
159 default = top.featureGates;
160 defaultText = literalExpression "config.${otop.featureGates}";
161 type = listOf str;
162 };
163
164 healthz = {
165 bind = mkOption {
166 description = lib.mdDoc "Kubernetes kubelet healthz listening address.";
167 default = "127.0.0.1";
168 type = str;
169 };
170
171 port = mkOption {
172 description = lib.mdDoc "Kubernetes kubelet healthz port.";
173 default = 10248;
174 type = int;
175 };
176 };
177
178 hostname = mkOption {
179 description = lib.mdDoc "Kubernetes kubelet hostname override.";
180 defaultText = literalExpression "config.networking.fqdnOrHostName";
181 type = str;
182 };
183
184 kubeconfig = top.lib.mkKubeConfigOptions "Kubelet";
185
186 manifests = mkOption {
187 description = lib.mdDoc "List of manifests to bootstrap with kubelet (only pods can be created as manifest entry)";
188 type = attrsOf attrs;
189 default = {};
190 };
191
192 nodeIp = mkOption {
193 description = lib.mdDoc "IP address of the node. If set, kubelet will use this IP address for the node.";
194 default = null;
195 type = nullOr str;
196 };
197
198 registerNode = mkOption {
199 description = lib.mdDoc "Whether to auto register kubelet with API server.";
200 default = true;
201 type = bool;
202 };
203
204 port = mkOption {
205 description = lib.mdDoc "Kubernetes kubelet info server listening port.";
206 default = 10250;
207 type = int;
208 };
209
210 seedDockerImages = mkOption {
211 description = lib.mdDoc "List of docker images to preload on system";
212 default = [];
213 type = listOf package;
214 };
215
216 taints = mkOption {
217 description = lib.mdDoc "Node taints (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/).";
218 default = {};
219 type = attrsOf (submodule [ taintOptions ]);
220 };
221
222 tlsCertFile = mkOption {
223 description = lib.mdDoc "File containing x509 Certificate for HTTPS.";
224 default = null;
225 type = nullOr path;
226 };
227
228 tlsKeyFile = mkOption {
229 description = lib.mdDoc "File containing x509 private key matching tlsCertFile.";
230 default = null;
231 type = nullOr path;
232 };
233
234 unschedulable = mkOption {
235 description = lib.mdDoc "Whether to set node taint to unschedulable=true as it is the case of node that has only master role.";
236 default = false;
237 type = bool;
238 };
239
240 verbosity = mkOption {
241 description = lib.mdDoc ''
242 Optional glog verbosity level for logging statements. See
243 <https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md>
244 '';
245 default = null;
246 type = nullOr int;
247 };
248
249 };
250
251 ###### implementation
252 config = mkMerge [
253 (mkIf cfg.enable {
254
255 environment.etc."cni/net.d".source = cniConfig;
256
257 services.kubernetes.kubelet.seedDockerImages = [infraContainer];
258
259 boot.kernel.sysctl = {
260 "net.bridge.bridge-nf-call-iptables" = 1;
261 "net.ipv4.ip_forward" = 1;
262 "net.bridge.bridge-nf-call-ip6tables" = 1;
263 };
264
265 systemd.services.kubelet = {
266 description = "Kubernetes Kubelet Service";
267 wantedBy = [ "kubernetes.target" ];
268 after = [ "containerd.service" "network.target" "kube-apiserver.service" ];
269 path = with pkgs; [
270 gitMinimal
271 openssh
272 util-linux
273 iproute2
274 ethtool
275 thin-provisioning-tools
276 iptables
277 socat
278 ] ++ lib.optional config.boot.zfs.enabled config.boot.zfs.package ++ top.path;
279 preStart = ''
280 ${concatMapStrings (img: ''
281 echo "Seeding container image: ${img}"
282 ${if (lib.hasSuffix "gz" img) then
283 ''${pkgs.gzip}/bin/zcat "${img}" | ${pkgs.containerd}/bin/ctr -n k8s.io image import --all-platforms -''
284 else
285 ''${pkgs.coreutils}/bin/cat "${img}" | ${pkgs.containerd}/bin/ctr -n k8s.io image import --all-platforms -''
286 }
287 '') cfg.seedDockerImages}
288
289 rm /opt/cni/bin/* || true
290 ${concatMapStrings (package: ''
291 echo "Linking cni package: ${package}"
292 ln -fs ${package}/bin/* /opt/cni/bin
293 '') cfg.cni.packages}
294 '';
295 serviceConfig = {
296 Slice = "kubernetes.slice";
297 CPUAccounting = true;
298 MemoryAccounting = true;
299 Restart = "on-failure";
300 RestartSec = "1000ms";
301 ExecStart = ''${top.package}/bin/kubelet \
302 --address=${cfg.address} \
303 --authentication-token-webhook \
304 --authentication-token-webhook-cache-ttl="10s" \
305 --authorization-mode=Webhook \
306 ${optionalString (cfg.clientCaFile != null)
307 "--client-ca-file=${cfg.clientCaFile}"} \
308 ${optionalString (cfg.clusterDns != "")
309 "--cluster-dns=${cfg.clusterDns}"} \
310 ${optionalString (cfg.clusterDomain != "")
311 "--cluster-domain=${cfg.clusterDomain}"} \
312 ${optionalString (cfg.featureGates != [])
313 "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \
314 --hairpin-mode=hairpin-veth \
315 --healthz-bind-address=${cfg.healthz.bind} \
316 --healthz-port=${toString cfg.healthz.port} \
317 --hostname-override=${cfg.hostname} \
318 --kubeconfig=${kubeconfig} \
319 ${optionalString (cfg.nodeIp != null)
320 "--node-ip=${cfg.nodeIp}"} \
321 --pod-infra-container-image=pause \
322 ${optionalString (cfg.manifests != {})
323 "--pod-manifest-path=/etc/${manifestPath}"} \
324 --port=${toString cfg.port} \
325 --register-node=${boolToString cfg.registerNode} \
326 ${optionalString (taints != "")
327 "--register-with-taints=${taints}"} \
328 --root-dir=${top.dataDir} \
329 ${optionalString (cfg.tlsCertFile != null)
330 "--tls-cert-file=${cfg.tlsCertFile}"} \
331 ${optionalString (cfg.tlsKeyFile != null)
332 "--tls-private-key-file=${cfg.tlsKeyFile}"} \
333 ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
334 --container-runtime=${cfg.containerRuntime} \
335 --container-runtime-endpoint=${cfg.containerRuntimeEndpoint} \
336 --cgroup-driver=systemd \
337 ${cfg.extraOpts}
338 '';
339 WorkingDirectory = top.dataDir;
340 };
341 unitConfig = {
342 StartLimitIntervalSec = 0;
343 };
344 };
345
346 # Allways include cni plugins
347 services.kubernetes.kubelet.cni.packages = [pkgs.cni-plugins pkgs.cni-plugin-flannel];
348
349 boot.kernelModules = ["br_netfilter" "overlay"];
350
351 services.kubernetes.kubelet.hostname =
352 mkDefault config.networking.fqdnOrHostName;
353
354 services.kubernetes.pki.certs = with top.lib; {
355 kubelet = mkCert {
356 name = "kubelet";
357 CN = top.kubelet.hostname;
358 action = "systemctl restart kubelet.service";
359
360 };
361 kubeletClient = mkCert {
362 name = "kubelet-client";
363 CN = "system:node:${top.kubelet.hostname}";
364 fields = {
365 O = "system:nodes";
366 };
367 action = "systemctl restart kubelet.service";
368 };
369 };
370
371 services.kubernetes.kubelet.kubeconfig.server = mkDefault top.apiserverAddress;
372 })
373
374 (mkIf (cfg.enable && cfg.manifests != {}) {
375 environment.etc = mapAttrs' (name: manifest:
376 nameValuePair "${manifestPath}/${name}.json" {
377 text = builtins.toJSON manifest;
378 mode = "0755";
379 }
380 ) cfg.manifests;
381 })
382
383 (mkIf (cfg.unschedulable && cfg.enable) {
384 services.kubernetes.kubelet.taints.unschedulable = {
385 value = "true";
386 effect = "NoSchedule";
387 };
388 })
389
390 ];
391
392 meta.buildDocsInSandbox = false;
393}