pyrox.dev
1{
2 config,
3 lib,
4 pkgs,
5 ...
6}:
7let
8 cfg = config.services.k3s;
9 removeOption =
10 config: instruction:
11 lib.mkRemovedOptionModule (
12 [
13 "services"
14 "k3s"
15 ]
16 ++ config
17 ) instruction;
18
19 manifestDir = "/var/lib/rancher/k3s/server/manifests";
20 chartDir = "/var/lib/rancher/k3s/server/static/charts";
21 imageDir = "/var/lib/rancher/k3s/agent/images";
22 containerdConfigTemplateFile = "/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl";
23 yamlFormat = pkgs.formats.yaml { };
24 yamlDocSeparator = builtins.toFile "yaml-doc-separator" "\n---\n";
25 # Manifests need a valid YAML suffix to be respected by k3s
26 mkManifestTarget =
27 name: if (lib.hasSuffix ".yaml" name || lib.hasSuffix ".yml" name) then name else name + ".yaml";
28 # Produces a list containing all duplicate manifest names
29 duplicateManifests = lib.intersectLists (builtins.attrNames cfg.autoDeployCharts) (
30 builtins.attrNames cfg.manifests
31 );
32 # Produces a list containing all duplicate chart names
33 duplicateCharts = lib.intersectLists (builtins.attrNames cfg.autoDeployCharts) (
34 builtins.attrNames cfg.charts
35 );
36
37 # Converts YAML -> JSON -> Nix
38 fromYaml =
39 path:
40 builtins.fromJSON (
41 builtins.readFile (
42 pkgs.runCommand "${path}-converted.json" { nativeBuildInputs = [ pkgs.yq-go ]; } ''
43 yq --no-colors --output-format json ${path} > $out
44 ''
45 )
46 );
47
48 # Replace prefixes and characters that are problematic in file names
49 cleanHelmChartName =
50 name:
51 let
52 woPrefix = lib.removePrefix "https://" (lib.removePrefix "oci://" name);
53 in
54 lib.replaceStrings
55 [
56 "/"
57 ":"
58 ]
59 [
60 "-"
61 "-"
62 ]
63 woPrefix;
64
65 # Fetch a Helm chart from a public registry. This only supports a basic Helm pull.
66 fetchHelm =
67 {
68 name,
69 repo,
70 version,
71 hash ? lib.fakeHash,
72 }:
73 let
74 isOci = lib.hasPrefix "oci://" repo;
75 pullCmd = if isOci then repo else "--repo ${repo} ${name}";
76 name' = if isOci then "${repo}-${version}" else "${repo}-${name}-${version}";
77 in
78 pkgs.runCommand (cleanHelmChartName "${name'}.tgz")
79 {
80 inherit (lib.fetchers.normalizeHash { } { inherit hash; }) outputHash outputHashAlgo;
81 impureEnvVars = lib.fetchers.proxyImpureEnvVars;
82 nativeBuildInputs = with pkgs; [
83 kubernetes-helm
84 cacert
85 # Helm requires HOME to refer to a writable dir
86 writableTmpDirAsHomeHook
87 ];
88 }
89 ''
90 helm pull ${pullCmd} --version ${version}
91 mv ./*.tgz $out
92 '';
93
94 # Returns the path to a YAML manifest file
95 mkExtraDeployManifest =
96 x:
97 # x is a derivation that provides a YAML file
98 if lib.isDerivation x then
99 x.outPath
100 # x is an attribute set that needs to be converted to a YAML file
101 else if builtins.isAttrs x then
102 (yamlFormat.generate "extra-deploy-chart-manifest" x)
103 # assume x is a path to a YAML file
104 else
105 x;
106
107 # Generate a HelmChart custom resource.
108 mkHelmChartCR =
109 name: value:
110 let
111 chartValues = if (lib.isPath value.values) then fromYaml value.values else value.values;
112 # use JSON for values as it's a subset of YAML and understood by the k3s Helm controller
113 valuesContent = builtins.toJSON chartValues;
114 in
115 # merge with extraFieldDefinitions to allow setting advanced values and overwrite generated
116 # values
117 lib.recursiveUpdate {
118 apiVersion = "helm.cattle.io/v1";
119 kind = "HelmChart";
120 metadata = {
121 inherit name;
122 namespace = "kube-system";
123 };
124 spec = {
125 inherit valuesContent;
126 inherit (value) targetNamespace createNamespace;
127 chart = "https://%{KUBERNETES_API}%/static/charts/${name}.tgz";
128 };
129 } value.extraFieldDefinitions;
130
131 # Generate a HelmChart custom resource together with extraDeploy manifests. This
132 # generates possibly a multi document YAML file that the auto deploy mechanism of k3s
133 # deploys.
134 mkAutoDeployChartManifest = name: value: {
135 # target is the final name of the link created for the manifest file
136 target = mkManifestTarget name;
137 inherit (value) enable package;
138 # source is a store path containing the complete manifest file
139 source = pkgs.concatText "auto-deploy-chart-${name}.yaml" (
140 [
141 (yamlFormat.generate "helm-chart-manifest-${name}.yaml" (mkHelmChartCR name value))
142 ]
143 # alternate the YAML doc separator (---) and extraDeploy manifests to create
144 # multi document YAMLs
145 ++ (lib.concatMap (x: [
146 yamlDocSeparator
147 (mkExtraDeployManifest x)
148 ]) value.extraDeploy)
149 );
150 };
151
152 autoDeployChartsModule = lib.types.submodule (
153 { config, ... }:
154 {
155 options = {
156 enable = lib.mkOption {
157 type = lib.types.bool;
158 default = true;
159 example = false;
160 description = ''
161 Whether to enable the installation of this Helm chart. Note that setting
162 this option to `false` will not uninstall the chart from the cluster, if
163 it was previously installed. Please use the the `--disable` flag or `.skip`
164 files to delete/disable Helm charts, as mentioned in the
165 [docs](https://docs.k3s.io/installation/packaged-components#disabling-manifests).
166 '';
167 };
168
169 repo = lib.mkOption {
170 type = lib.types.nonEmptyStr;
171 example = "https://kubernetes.github.io/ingress-nginx";
172 description = ''
173 The repo of the Helm chart. Only has an effect if `package` is not set.
174 The Helm chart is fetched during build time and placed as a `.tgz` archive on the
175 filesystem.
176 '';
177 };
178
179 name = lib.mkOption {
180 type = lib.types.nonEmptyStr;
181 example = "ingress-nginx";
182 description = ''
183 The name of the Helm chart. Only has an effect if `package` is not set.
184 The Helm chart is fetched during build time and placed as a `.tgz` archive on the
185 filesystem.
186 '';
187 };
188
189 version = lib.mkOption {
190 type = lib.types.nonEmptyStr;
191 example = "4.7.0";
192 description = ''
193 The version of the Helm chart. Only has an effect if `package` is not set.
194 The Helm chart is fetched during build time and placed as a `.tgz` archive on the
195 filesystem.
196 '';
197 };
198
199 hash = lib.mkOption {
200 type = lib.types.str;
201 example = "sha256-ej+vpPNdiOoXsaj1jyRpWLisJgWo8EqX+Z5VbpSjsPA=";
202 default = "";
203 description = ''
204 The hash of the packaged Helm chart. Only has an effect if `package` is not set.
205 The Helm chart is fetched during build time and placed as a `.tgz` archive on the
206 filesystem.
207 '';
208 };
209
210 package = lib.mkOption {
211 type = with lib.types; either path package;
212 example = lib.literalExpression "../my-helm-chart.tgz";
213 description = ''
214 The packaged Helm chart. Overwrites the options `repo`, `name`, `version`
215 and `hash` in case of conflicts.
216 '';
217 };
218
219 targetNamespace = lib.mkOption {
220 type = lib.types.nonEmptyStr;
221 default = "default";
222 example = "kube-system";
223 description = "The namespace in which the Helm chart gets installed.";
224 };
225
226 createNamespace = lib.mkOption {
227 type = lib.types.bool;
228 default = false;
229 example = true;
230 description = "Whether to create the target namespace if not present.";
231 };
232
233 values = lib.mkOption {
234 type = with lib.types; either path attrs;
235 default = { };
236 example = {
237 replicaCount = 3;
238 hostName = "my-host";
239 server = {
240 name = "nginx";
241 port = 80;
242 };
243 };
244 description = ''
245 Override default chart values via Nix expressions. This is equivalent to setting
246 values in a `values.yaml` file.
247
248 WARNING: The values (including secrets!) specified here are exposed unencrypted
249 in the world-readable nix store.
250 '';
251 };
252
253 extraDeploy = lib.mkOption {
254 type = with lib.types; listOf (either path attrs);
255 default = [ ];
256 example = lib.literalExpression ''
257 [
258 ../manifests/my-extra-deployment.yaml
259 {
260 apiVersion = "v1";
261 kind = "Service";
262 metadata = {
263 name = "app-service";
264 };
265 spec = {
266 selector = {
267 "app.kubernetes.io/name" = "MyApp";
268 };
269 ports = [
270 {
271 name = "name-of-service-port";
272 protocol = "TCP";
273 port = 80;
274 targetPort = "http-web-svc";
275 }
276 ];
277 };
278 }
279 ];
280 '';
281 description = "List of extra Kubernetes manifests to deploy with this Helm chart.";
282 };
283
284 extraFieldDefinitions = lib.mkOption {
285 inherit (yamlFormat) type;
286 default = { };
287 example = {
288 spec = {
289 bootstrap = true;
290 helmVersion = "v2";
291 backOffLimit = 3;
292 jobImage = "custom-helm-controller:v0.0.1";
293 };
294 };
295 description = ''
296 Extra HelmChart field definitions that are merged with the rest of the HelmChart
297 custom resource. This can be used to set advanced fields or to overwrite
298 generated fields. See <https://docs.k3s.io/helm#helmchart-field-definitions>
299 for possible fields.
300 '';
301 };
302 };
303
304 config.package = lib.mkDefault (fetchHelm {
305 inherit (config)
306 repo
307 name
308 version
309 hash
310 ;
311 });
312 }
313 );
314
315 manifestModule = lib.types.submodule (
316 {
317 name,
318 config,
319 options,
320 ...
321 }:
322 {
323 options = {
324 enable = lib.mkOption {
325 type = lib.types.bool;
326 default = true;
327 description = "Whether this manifest file should be generated.";
328 };
329
330 target = lib.mkOption {
331 type = lib.types.nonEmptyStr;
332 example = "manifest.yaml";
333 description = ''
334 Name of the symlink (relative to {file}`${manifestDir}`).
335 Defaults to the attribute name.
336 '';
337 };
338
339 content = lib.mkOption {
340 type = with lib.types; nullOr (either attrs (listOf attrs));
341 default = null;
342 description = ''
343 Content of the manifest file. A single attribute set will
344 generate a single document YAML file. A list of attribute sets
345 will generate multiple documents separated by `---` in a single
346 YAML file.
347 '';
348 };
349
350 source = lib.mkOption {
351 type = lib.types.path;
352 example = lib.literalExpression "./manifests/app.yaml";
353 description = ''
354 Path of the source `.yaml` file.
355 '';
356 };
357 };
358
359 config = {
360 target = lib.mkDefault (mkManifestTarget name);
361 source = lib.mkIf (config.content != null) (
362 let
363 name' = "k3s-manifest-" + builtins.baseNameOf name;
364 docName = "k3s-manifest-doc-" + builtins.baseNameOf name;
365 mkSource =
366 value:
367 if builtins.isList value then
368 pkgs.concatText name' (
369 lib.concatMap (x: [
370 yamlDocSeparator
371 (yamlFormat.generate docName x)
372 ]) value
373 )
374 else
375 yamlFormat.generate name' value;
376 in
377 lib.mkDerivedConfig options.content mkSource
378 );
379 };
380 }
381 );
382in
383{
384 imports = [ (removeOption [ "docker" ] "k3s docker option is no longer supported.") ];
385
386 # interface
387 options.services.k3s = {
388 enable = lib.mkEnableOption "k3s";
389
390 package = lib.mkPackageOption pkgs "k3s" { };
391
392 role = lib.mkOption {
393 description = ''
394 Whether k3s should run as a server or agent.
395
396 If it's a server:
397
398 - By default it also runs workloads as an agent.
399 - Starts by default as a standalone server using an embedded sqlite datastore.
400 - Configure `clusterInit = true` to switch over to embedded etcd datastore and enable HA mode.
401 - Configure `serverAddr` to join an already-initialized HA cluster.
402
403 If it's an agent:
404
405 - `serverAddr` is required.
406 '';
407 default = "server";
408 type = lib.types.enum [
409 "server"
410 "agent"
411 ];
412 };
413
414 serverAddr = lib.mkOption {
415 type = lib.types.str;
416 description = ''
417 The k3s server to connect to.
418
419 Servers and agents need to communicate each other. Read
420 [the networking docs](https://rancher.com/docs/k3s/latest/en/installation/installation-requirements/#networking)
421 to know how to configure the firewall.
422 '';
423 example = "https://10.0.0.10:6443";
424 default = "";
425 };
426
427 clusterInit = lib.mkOption {
428 type = lib.types.bool;
429 default = false;
430 description = ''
431 Initialize HA cluster using an embedded etcd datastore.
432
433 If this option is `false` and `role` is `server`
434
435 On a server that was using the default embedded sqlite backend,
436 enabling this option will migrate to an embedded etcd DB.
437
438 If an HA cluster using the embedded etcd datastore was already initialized,
439 this option has no effect.
440
441 This option only makes sense in a server that is not connecting to another server.
442
443 If you are configuring an HA cluster with an embedded etcd,
444 the 1st server must have `clusterInit = true`
445 and other servers must connect to it using `serverAddr`.
446 '';
447 };
448
449 token = lib.mkOption {
450 type = lib.types.str;
451 description = ''
452 The k3s token to use when connecting to a server.
453
454 WARNING: This option will expose store your token unencrypted world-readable in the nix store.
455 If this is undesired use the tokenFile option instead.
456 '';
457 default = "";
458 };
459
460 tokenFile = lib.mkOption {
461 type = lib.types.nullOr lib.types.path;
462 description = "File path containing k3s token to use when connecting to the server.";
463 default = null;
464 };
465
466 extraFlags = lib.mkOption {
467 description = "Extra flags to pass to the k3s command.";
468 type = with lib.types; either str (listOf str);
469 default = [ ];
470 example = [
471 "--disable traefik"
472 "--cluster-cidr 10.24.0.0/16"
473 ];
474 };
475
476 disableAgent = lib.mkOption {
477 type = lib.types.bool;
478 default = false;
479 description = "Only run the server. This option only makes sense for a server.";
480 };
481
482 environmentFile = lib.mkOption {
483 type = lib.types.nullOr lib.types.path;
484 description = ''
485 File path containing environment variables for configuring the k3s service in the format of an EnvironmentFile. See {manpage}`systemd.exec(5)`.
486 '';
487 default = null;
488 };
489
490 configPath = lib.mkOption {
491 type = lib.types.nullOr lib.types.path;
492 default = null;
493 description = "File path containing the k3s YAML config. This is useful when the config is generated (for example on boot).";
494 };
495
496 manifests = lib.mkOption {
497 type = lib.types.attrsOf manifestModule;
498 default = { };
499 example = lib.literalExpression ''
500 {
501 deployment.source = ../manifests/deployment.yaml;
502 my-service = {
503 enable = false;
504 target = "app-service.yaml";
505 content = {
506 apiVersion = "v1";
507 kind = "Service";
508 metadata = {
509 name = "app-service";
510 };
511 spec = {
512 selector = {
513 "app.kubernetes.io/name" = "MyApp";
514 };
515 ports = [
516 {
517 name = "name-of-service-port";
518 protocol = "TCP";
519 port = 80;
520 targetPort = "http-web-svc";
521 }
522 ];
523 };
524 };
525 };
526
527 nginx.content = [
528 {
529 apiVersion = "v1";
530 kind = "Pod";
531 metadata = {
532 name = "nginx";
533 labels = {
534 "app.kubernetes.io/name" = "MyApp";
535 };
536 };
537 spec = {
538 containers = [
539 {
540 name = "nginx";
541 image = "nginx:1.14.2";
542 ports = [
543 {
544 containerPort = 80;
545 name = "http-web-svc";
546 }
547 ];
548 }
549 ];
550 };
551 }
552 {
553 apiVersion = "v1";
554 kind = "Service";
555 metadata = {
556 name = "nginx-service";
557 };
558 spec = {
559 selector = {
560 "app.kubernetes.io/name" = "MyApp";
561 };
562 ports = [
563 {
564 name = "name-of-service-port";
565 protocol = "TCP";
566 port = 80;
567 targetPort = "http-web-svc";
568 }
569 ];
570 };
571 }
572 ];
573 };
574 '';
575 description = ''
576 Auto-deploying manifests that are linked to {file}`${manifestDir}` before k3s starts.
577 Note that deleting manifest files will not remove or otherwise modify the resources
578 it created. Please use the the `--disable` flag or `.skip` files to delete/disable AddOns,
579 as mentioned in the [docs](https://docs.k3s.io/installation/packaged-components#disabling-manifests).
580 This option only makes sense on server nodes (`role = server`).
581 Read the [auto-deploying manifests docs](https://docs.k3s.io/installation/packaged-components#auto-deploying-manifests-addons)
582 for further information.
583 '';
584 };
585
586 charts = lib.mkOption {
587 type = with lib.types; attrsOf (either path package);
588 default = { };
589 example = lib.literalExpression ''
590 nginx = ../charts/my-nginx-chart.tgz;
591 redis = ../charts/my-redis-chart.tgz;
592 '';
593 description = ''
594 Packaged Helm charts that are linked to {file}`${chartDir}` before k3s starts.
595 The attribute name will be used as the link target (relative to {file}`${chartDir}`).
596 The specified charts will only be placed on the file system and made available to the
597 Kubernetes APIServer from within the cluster. See the [](#opt-services.k3s.autoDeployCharts)
598 option and the [k3s Helm controller docs](https://docs.k3s.io/helm#using-the-helm-controller)
599 to deploy Helm charts. This option only makes sense on server nodes (`role = server`).
600 '';
601 };
602
603 containerdConfigTemplate = lib.mkOption {
604 type = lib.types.nullOr lib.types.str;
605 default = null;
606 example = lib.literalExpression ''
607 # Base K3s config
608 {{ template "base" . }}
609
610 # Add a custom runtime
611 [plugins."io.containerd.grpc.v1.cri".containerd.runtimes."custom"]
612 runtime_type = "io.containerd.runc.v2"
613 [plugins."io.containerd.grpc.v1.cri".containerd.runtimes."custom".options]
614 BinaryName = "/path/to/custom-container-runtime"
615 '';
616 description = ''
617 Config template for containerd, to be placed at
618 `/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl`.
619 See the K3s docs on [configuring containerd](https://docs.k3s.io/advanced#configuring-containerd).
620 '';
621 };
622
623 images = lib.mkOption {
624 type = with lib.types; listOf package;
625 default = [ ];
626 example = lib.literalExpression ''
627 [
628 (pkgs.dockerTools.pullImage {
629 imageName = "docker.io/bitnami/keycloak";
630 imageDigest = "sha256:714dfadc66a8e3adea6609bda350345bd3711657b7ef3cf2e8015b526bac2d6b";
631 hash = "sha256-IM2BLZ0EdKIZcRWOtuFY9TogZJXCpKtPZnMnPsGlq0Y=";
632 finalImageTag = "21.1.2-debian-11-r0";
633 })
634
635 config.services.k3s.package.airgap-images
636 ]
637 '';
638 description = ''
639 List of derivations that provide container images.
640 All images are linked to {file}`${imageDir}` before k3s starts and consequently imported
641 by the k3s agent. Consider importing the k3s airgap images archive of the k3s package in
642 use, if you want to pre-provision this node with all k3s container images. This option
643 only makes sense on nodes with an enabled agent.
644 '';
645 };
646
647 gracefulNodeShutdown = {
648 enable = lib.mkEnableOption ''
649 graceful node shutdowns where the kubelet attempts to detect
650 node system shutdown and terminates pods running on the node. See the
651 [documentation](https://kubernetes.io/docs/concepts/cluster-administration/node-shutdown/#graceful-node-shutdown)
652 for further information.
653 '';
654
655 shutdownGracePeriod = lib.mkOption {
656 type = lib.types.nonEmptyStr;
657 default = "30s";
658 example = "1m30s";
659 description = ''
660 Specifies the total duration that the node should delay the shutdown by. This is the total
661 grace period for pod termination for both regular and critical pods.
662 '';
663 };
664
665 shutdownGracePeriodCriticalPods = lib.mkOption {
666 type = lib.types.nonEmptyStr;
667 default = "10s";
668 example = "15s";
669 description = ''
670 Specifies the duration used to terminate critical pods during a node shutdown. This should be
671 less than `shutdownGracePeriod`.
672 '';
673 };
674 };
675
676 extraKubeletConfig = lib.mkOption {
677 type = with lib.types; attrsOf anything;
678 default = { };
679 example = {
680 podsPerCore = 3;
681 memoryThrottlingFactor = 0.69;
682 containerLogMaxSize = "5Mi";
683 };
684 description = ''
685 Extra configuration to add to the kubelet's configuration file. The subset of the kubelet's
686 configuration that can be configured via a file is defined by the
687 [KubeletConfiguration](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/)
688 struct. See the
689 [documentation](https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/)
690 for further information.
691 '';
692 };
693
694 extraKubeProxyConfig = lib.mkOption {
695 type = with lib.types; attrsOf anything;
696 default = { };
697 example = {
698 mode = "nftables";
699 clientConnection.kubeconfig = "/var/lib/rancher/k3s/agent/kubeproxy.kubeconfig";
700 };
701 description = ''
702 Extra configuration to add to the kube-proxy's configuration file. The subset of the kube-proxy's
703 configuration that can be configured via a file is defined by the
704 [KubeProxyConfiguration](https://kubernetes.io/docs/reference/config-api/kube-proxy-config.v1alpha1/)
705 struct. Note that the kubeconfig param will be override by `clientConnection.kubeconfig`, so you must
706 set the `clientConnection.kubeconfig` if you want to use `extraKubeProxyConfig`.
707 '';
708 };
709
710 autoDeployCharts = lib.mkOption {
711 type = lib.types.attrsOf autoDeployChartsModule;
712 apply = lib.mapAttrs mkAutoDeployChartManifest;
713 default = { };
714 example = lib.literalExpression ''
715 {
716 harbor = {
717 name = "harbor";
718 repo = "https://helm.goharbor.io";
719 version = "1.14.0";
720 hash = "sha256-fMP7q1MIbvzPGS9My91vbQ1d3OJMjwc+o8YE/BXZaYU=";
721 values = {
722 existingSecretAdminPassword = "harbor-admin";
723 expose = {
724 tls = {
725 enabled = true;
726 certSource = "secret";
727 secret.secretName = "my-tls-secret";
728 };
729 ingress = {
730 hosts.core = "example.com";
731 className = "nginx";
732 };
733 };
734 };
735 };
736 nginx = {
737 repo = "oci://registry-1.docker.io/bitnamicharts/nginx";
738 version = "20.0.0";
739 hash = "sha256-sy+tzB+i9jIl/tqOMzzuhVhTU4EZVsoSBtPznxF/36c=";
740 };
741 custom-chart = {
742 package = ../charts/my-chart.tgz;
743 values = ../values/my-values.yaml;
744 extraFieldDefinitions = {
745 spec.timeout = "60s";
746 };
747 };
748 }
749 '';
750 description = ''
751 Auto deploying Helm charts that are installed by the k3s Helm controller. Avoid to use
752 attribute names that are also used in the [](#opt-services.k3s.manifests) and
753 [](#opt-services.k3s.charts) options. Manifests with the same name will override
754 auto deploying charts with the same name. Similiarly, charts with the same name will
755 overwrite the Helm chart contained in auto deploying charts. This option only makes
756 sense on server nodes (`role = server`). See the
757 [k3s Helm documentation](https://docs.k3s.io/helm) for further information.
758 '';
759 };
760 };
761
762 # implementation
763
764 config = lib.mkIf cfg.enable {
765 warnings =
766 (lib.optional (cfg.role != "server" && cfg.manifests != { })
767 "k3s: Auto deploying manifests are only installed on server nodes (role == server), they will be ignored by this node."
768 )
769 ++ (lib.optional (cfg.role != "server" && cfg.charts != { })
770 "k3s: Helm charts are only made available to the cluster on server nodes (role == server), they will be ignored by this node."
771 )
772 ++ (lib.optional (cfg.role != "server" && cfg.autoDeployCharts != { })
773 "k3s: Auto deploying Helm charts are only installed on server nodes (role == server), they will be ignored by this node."
774 )
775 ++ (lib.optional (duplicateManifests != [ ])
776 "k3s: The following auto deploying charts are overriden by manifests of the same name: ${toString duplicateManifests}."
777 )
778 ++ (lib.optional (duplicateCharts != [ ])
779 "k3s: The following auto deploying charts are overriden by charts of the same name: ${toString duplicateCharts}."
780 )
781 ++ (lib.optional (
782 cfg.disableAgent && cfg.images != [ ]
783 ) "k3s: Images are only imported on nodes with an enabled agent, they will be ignored by this node")
784 ++ (lib.optional (
785 cfg.role == "agent" && cfg.configPath == null && cfg.serverAddr == ""
786 ) "k3s: serverAddr or configPath (with 'server' key) should be set if role is 'agent'")
787 ++ (lib.optional
788 (cfg.role == "agent" && cfg.configPath == null && cfg.tokenFile == null && cfg.token == "")
789 "k3s: Token or tokenFile or configPath (with 'token' or 'token-file' keys) should be set if role is 'agent'"
790 );
791
792 assertions = [
793 {
794 assertion = cfg.role == "agent" -> !cfg.disableAgent;
795 message = "k3s: disableAgent must be false if role is 'agent'";
796 }
797 {
798 assertion = cfg.role == "agent" -> !cfg.clusterInit;
799 message = "k3s: clusterInit must be false if role is 'agent'";
800 }
801 ];
802
803 environment.systemPackages = [ config.services.k3s.package ];
804
805 # Use systemd-tmpfiles to activate k3s content
806 systemd.tmpfiles.settings."10-k3s" =
807 let
808 # Merge manifest with manifests generated from auto deploying charts, keep only enabled manifests
809 enabledManifests = lib.filterAttrs (_: v: v.enable) (cfg.autoDeployCharts // cfg.manifests);
810 # Merge charts with charts contained in enabled auto deploying charts
811 helmCharts =
812 (lib.concatMapAttrs (n: v: { ${n} = v.package; }) (
813 lib.filterAttrs (_: v: v.enable) cfg.autoDeployCharts
814 ))
815 // cfg.charts;
816 # Make a systemd-tmpfiles rule for a manifest
817 mkManifestRule = manifest: {
818 name = "${manifestDir}/${manifest.target}";
819 value = {
820 "L+".argument = "${manifest.source}";
821 };
822 };
823 # Ensure that all chart targets have a .tgz suffix
824 mkChartTarget = name: if (lib.hasSuffix ".tgz" name) then name else name + ".tgz";
825 # Make a systemd-tmpfiles rule for a chart
826 mkChartRule = target: source: {
827 name = "${chartDir}/${mkChartTarget target}";
828 value = {
829 "L+".argument = "${source}";
830 };
831 };
832 # Make a systemd-tmpfiles rule for a container image
833 mkImageRule = image: {
834 name = "${imageDir}/${image.name}";
835 value = {
836 "L+".argument = "${image}";
837 };
838 };
839 in
840 (lib.mapAttrs' (_: v: mkManifestRule v) enabledManifests)
841 // (lib.mapAttrs' (n: v: mkChartRule n v) helmCharts)
842 // (builtins.listToAttrs (map mkImageRule cfg.images))
843 // (lib.optionalAttrs (cfg.containerdConfigTemplate != null) {
844 ${containerdConfigTemplateFile} = {
845 "L+".argument = "${pkgs.writeText "config.toml.tmpl" cfg.containerdConfigTemplate}";
846 };
847 });
848
849 systemd.services.k3s =
850 let
851 kubeletParams =
852 (lib.optionalAttrs (cfg.gracefulNodeShutdown.enable) {
853 inherit (cfg.gracefulNodeShutdown) shutdownGracePeriod shutdownGracePeriodCriticalPods;
854 })
855 // cfg.extraKubeletConfig;
856 kubeletConfig = (pkgs.formats.yaml { }).generate "k3s-kubelet-config" (
857 {
858 apiVersion = "kubelet.config.k8s.io/v1beta1";
859 kind = "KubeletConfiguration";
860 }
861 // kubeletParams
862 );
863
864 kubeProxyConfig = (pkgs.formats.yaml { }).generate "k3s-kubeProxy-config" (
865 {
866 apiVersion = "kubeproxy.config.k8s.io/v1alpha1";
867 kind = "KubeProxyConfiguration";
868 }
869 // cfg.extraKubeProxyConfig
870 );
871 in
872 {
873 description = "k3s service";
874 after = [
875 "firewall.service"
876 "network-online.target"
877 ];
878 wants = [
879 "firewall.service"
880 "network-online.target"
881 ];
882 wantedBy = [ "multi-user.target" ];
883 path = lib.optional config.boot.zfs.enabled config.boot.zfs.package;
884 serviceConfig = {
885 # See: https://github.com/rancher/k3s/blob/dddbd16305284ae4bd14c0aade892412310d7edc/install.sh#L197
886 Type = if cfg.role == "agent" then "exec" else "notify";
887 KillMode = "process";
888 Delegate = "yes";
889 Restart = "always";
890 RestartSec = "5s";
891 LimitNOFILE = 1048576;
892 LimitNPROC = "infinity";
893 LimitCORE = "infinity";
894 TasksMax = "infinity";
895 EnvironmentFile = cfg.environmentFile;
896 ExecStart = lib.concatStringsSep " \\\n " (
897 [ "${cfg.package}/bin/k3s ${cfg.role}" ]
898 ++ (lib.optional cfg.clusterInit "--cluster-init")
899 ++ (lib.optional cfg.disableAgent "--disable-agent")
900 ++ (lib.optional (cfg.serverAddr != "") "--server ${cfg.serverAddr}")
901 ++ (lib.optional (cfg.token != "") "--token ${cfg.token}")
902 ++ (lib.optional (cfg.tokenFile != null) "--token-file ${cfg.tokenFile}")
903 ++ (lib.optional (cfg.configPath != null) "--config ${cfg.configPath}")
904 ++ (lib.optional (kubeletParams != { }) "--kubelet-arg=config=${kubeletConfig}")
905 ++ (lib.optional (cfg.extraKubeProxyConfig != { }) "--kube-proxy-arg=config=${kubeProxyConfig}")
906 ++ (lib.flatten cfg.extraFlags)
907 );
908 };
909 };
910 };
911
912 meta.maintainers = lib.teams.k3s.members;
913}