pyrox.dev
1{
2 config,
3 lib,
4 pkgs,
5 ...
6}:
7let
8 cfg = config.services.k3s;
9 removeOption =
10 config: instruction:
11 lib.mkRemovedOptionModule (
12 [
13 "services"
14 "k3s"
15 ]
16 ++ config
17 ) instruction;
18
19 manifestDir = "/var/lib/rancher/k3s/server/manifests";
20 chartDir = "/var/lib/rancher/k3s/server/static/charts";
21 imageDir = "/var/lib/rancher/k3s/agent/images";
22 containerdConfigTemplateFile = "/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl";
23 yamlFormat = pkgs.formats.yaml { };
24 yamlDocSeparator = builtins.toFile "yaml-doc-separator" "\n---\n";
25 # Manifests need a valid YAML suffix to be respected by k3s
26 mkManifestTarget =
27 name: if (lib.hasSuffix ".yaml" name || lib.hasSuffix ".yml" name) then name else name + ".yaml";
28 # Produces a list containing all duplicate manifest names
29 duplicateManifests =
30 with builtins;
31 lib.intersectLists (attrNames cfg.autoDeployCharts) (attrNames cfg.manifests);
32 # Produces a list containing all duplicate chart names
33 duplicateCharts =
34 with builtins;
35 lib.intersectLists (attrNames cfg.autoDeployCharts) (attrNames cfg.charts);
36
37 # Converts YAML -> JSON -> Nix
38 fromYaml =
39 path:
40 with builtins;
41 fromJSON (
42 readFile (
43 pkgs.runCommand "${path}-converted.json" { nativeBuildInputs = [ yq-go ]; } ''
44 yq --no-colors --output-format json ${path} > $out
45 ''
46 )
47 );
48
49 # Replace characters that are problematic in file names
50 cleanHelmChartName =
51 lib.replaceStrings
52 [
53 "/"
54 ":"
55 ]
56 [
57 "-"
58 "-"
59 ];
60
61 # Fetch a Helm chart from a public registry. This only supports a basic Helm pull.
62 fetchHelm =
63 {
64 name,
65 repo,
66 version,
67 hash ? lib.fakeHash,
68 }:
69 pkgs.runCommand (cleanHelmChartName "${lib.removePrefix "https://" repo}-${name}-${version}.tgz")
70 {
71 inherit (lib.fetchers.normalizeHash { } { inherit hash; }) outputHash outputHashAlgo;
72 impureEnvVars = lib.fetchers.proxyImpureEnvVars;
73 nativeBuildInputs = with pkgs; [
74 kubernetes-helm
75 cacert
76 ];
77 }
78 ''
79 export HOME="$PWD"
80 helm repo add repository ${repo}
81 helm pull repository/${name} --version ${version}
82 mv ./*.tgz $out
83 '';
84
85 # Returns the path to a YAML manifest file
86 mkExtraDeployManifest =
87 x:
88 # x is a derivation that provides a YAML file
89 if lib.isDerivation x then
90 x.outPath
91 # x is an attribute set that needs to be converted to a YAML file
92 else if builtins.isAttrs x then
93 (yamlFormat.generate "extra-deploy-chart-manifest" x)
94 # assume x is a path to a YAML file
95 else
96 x;
97
98 # Generate a HelmChart custom resource.
99 mkHelmChartCR =
100 name: value:
101 let
102 chartValues = if (lib.isPath value.values) then fromYaml value.values else value.values;
103 # use JSON for values as it's a subset of YAML and understood by the k3s Helm controller
104 valuesContent = builtins.toJSON chartValues;
105 in
106 # merge with extraFieldDefinitions to allow setting advanced values and overwrite generated
107 # values
108 lib.recursiveUpdate {
109 apiVersion = "helm.cattle.io/v1";
110 kind = "HelmChart";
111 metadata = {
112 inherit name;
113 namespace = "kube-system";
114 };
115 spec = {
116 inherit valuesContent;
117 inherit (value) targetNamespace createNamespace;
118 chart = "https://%{KUBERNETES_API}%/static/charts/${name}.tgz";
119 };
120 } value.extraFieldDefinitions;
121
122 # Generate a HelmChart custom resource together with extraDeploy manifests. This
123 # generates possibly a multi document YAML file that the auto deploy mechanism of k3s
124 # deploys.
125 mkAutoDeployChartManifest = name: value: {
126 # target is the final name of the link created for the manifest file
127 target = mkManifestTarget name;
128 inherit (value) enable package;
129 # source is a store path containing the complete manifest file
130 source = pkgs.concatText "auto-deploy-chart-${name}.yaml" (
131 [
132 (yamlFormat.generate "helm-chart-manifest-${name}.yaml" (mkHelmChartCR name value))
133 ]
134 # alternate the YAML doc seperator (---) and extraDeploy manifests to create
135 # multi document YAMLs
136 ++ (lib.concatMap (x: [
137 yamlDocSeparator
138 (mkExtraDeployManifest x)
139 ]) value.extraDeploy)
140 );
141 };
142
143 autoDeployChartsModule = lib.types.submodule (
144 { config, ... }:
145 {
146 options = {
147 enable = lib.mkOption {
148 type = lib.types.bool;
149 default = true;
150 example = false;
151 description = ''
152 Whether to enable the installation of this Helm chart. Note that setting
153 this option to `false` will not uninstall the chart from the cluster, if
154 it was previously installed. Please use the the `--disable` flag or `.skip`
155 files to delete/disable Helm charts, as mentioned in the
156 [docs](https://docs.k3s.io/installation/packaged-components#disabling-manifests).
157 '';
158 };
159
160 repo = lib.mkOption {
161 type = lib.types.nonEmptyStr;
162 example = "https://kubernetes.github.io/ingress-nginx";
163 description = ''
164 The repo of the Helm chart. Only has an effect if `package` is not set.
165 The Helm chart is fetched during build time and placed as a `.tgz` archive on the
166 filesystem.
167 '';
168 };
169
170 name = lib.mkOption {
171 type = lib.types.nonEmptyStr;
172 example = "ingress-nginx";
173 description = ''
174 The name of the Helm chart. Only has an effect if `package` is not set.
175 The Helm chart is fetched during build time and placed as a `.tgz` archive on the
176 filesystem.
177 '';
178 };
179
180 version = lib.mkOption {
181 type = lib.types.nonEmptyStr;
182 example = "4.7.0";
183 description = ''
184 The version of the Helm chart. Only has an effect if `package` is not set.
185 The Helm chart is fetched during build time and placed as a `.tgz` archive on the
186 filesystem.
187 '';
188 };
189
190 hash = lib.mkOption {
191 type = lib.types.str;
192 example = "sha256-ej+vpPNdiOoXsaj1jyRpWLisJgWo8EqX+Z5VbpSjsPA=";
193 default = "";
194 description = ''
195 The hash of the packaged Helm chart. Only has an effect if `package` is not set.
196 The Helm chart is fetched during build time and placed as a `.tgz` archive on the
197 filesystem.
198 '';
199 };
200
201 package = lib.mkOption {
202 type = with lib.types; either path package;
203 example = lib.literalExpression "../my-helm-chart.tgz";
204 description = ''
205 The packaged Helm chart. Overwrites the options `repo`, `name`, `version`
206 and `hash` in case of conflicts.
207 '';
208 };
209
210 targetNamespace = lib.mkOption {
211 type = lib.types.nonEmptyStr;
212 default = "default";
213 example = "kube-system";
214 description = "The namespace in which the Helm chart gets installed.";
215 };
216
217 createNamespace = lib.mkOption {
218 type = lib.types.bool;
219 default = false;
220 example = true;
221 description = "Whether to create the target namespace if not present.";
222 };
223
224 values = lib.mkOption {
225 type = with lib.types; either path attrs;
226 default = { };
227 example = {
228 replicaCount = 3;
229 hostName = "my-host";
230 server = {
231 name = "nginx";
232 port = 80;
233 };
234 };
235 description = ''
236 Override default chart values via Nix expressions. This is equivalent to setting
237 values in a `values.yaml` file.
238
239 WARNING: The values (including secrets!) specified here are exposed unencrypted
240 in the world-readable nix store.
241 '';
242 };
243
244 extraDeploy = lib.mkOption {
245 type = with lib.types; listOf (either path attrs);
246 default = [ ];
247 example = lib.literalExpression ''
248 [
249 ../manifests/my-extra-deployment.yaml
250 {
251 apiVersion = "v1";
252 kind = "Service";
253 metadata = {
254 name = "app-service";
255 };
256 spec = {
257 selector = {
258 "app.kubernetes.io/name" = "MyApp";
259 };
260 ports = [
261 {
262 name = "name-of-service-port";
263 protocol = "TCP";
264 port = 80;
265 targetPort = "http-web-svc";
266 }
267 ];
268 };
269 }
270 ];
271 '';
272 description = "List of extra Kubernetes manifests to deploy with this Helm chart.";
273 };
274
275 extraFieldDefinitions = lib.mkOption {
276 inherit (yamlFormat) type;
277 default = { };
278 example = {
279 spec = {
280 bootstrap = true;
281 helmVersion = "v2";
282 backOffLimit = 3;
283 jobImage = "custom-helm-controller:v0.0.1";
284 };
285 };
286 description = ''
287 Extra HelmChart field definitions that are merged with the rest of the HelmChart
288 custom resource. This can be used to set advanced fields or to overwrite
289 generated fields. See https://docs.k3s.io/helm#helmchart-field-definitions
290 for possible fields.
291 '';
292 };
293 };
294
295 config.package = lib.mkDefault (fetchHelm {
296 inherit (config)
297 repo
298 name
299 version
300 hash
301 ;
302 });
303 }
304 );
305
306 manifestModule = lib.types.submodule (
307 {
308 name,
309 config,
310 options,
311 ...
312 }:
313 {
314 options = {
315 enable = lib.mkOption {
316 type = lib.types.bool;
317 default = true;
318 description = "Whether this manifest file should be generated.";
319 };
320
321 target = lib.mkOption {
322 type = lib.types.nonEmptyStr;
323 example = "manifest.yaml";
324 description = ''
325 Name of the symlink (relative to {file}`${manifestDir}`).
326 Defaults to the attribute name.
327 '';
328 };
329
330 content = lib.mkOption {
331 type = with lib.types; nullOr (either attrs (listOf attrs));
332 default = null;
333 description = ''
334 Content of the manifest file. A single attribute set will
335 generate a single document YAML file. A list of attribute sets
336 will generate multiple documents separated by `---` in a single
337 YAML file.
338 '';
339 };
340
341 source = lib.mkOption {
342 type = lib.types.path;
343 example = lib.literalExpression "./manifests/app.yaml";
344 description = ''
345 Path of the source `.yaml` file.
346 '';
347 };
348 };
349
350 config = {
351 target = lib.mkDefault (mkManifestTarget name);
352 source = lib.mkIf (config.content != null) (
353 let
354 name' = "k3s-manifest-" + builtins.baseNameOf name;
355 docName = "k3s-manifest-doc-" + builtins.baseNameOf name;
356 mkSource =
357 value:
358 if builtins.isList value then
359 pkgs.concatText name' (
360 lib.concatMap (x: [
361 yamlDocSeparator
362 (yamlFormat.generate docName x)
363 ]) value
364 )
365 else
366 yamlFormat.generate name' value;
367 in
368 lib.mkDerivedConfig options.content mkSource
369 );
370 };
371 }
372 );
373in
374{
375 imports = [ (removeOption [ "docker" ] "k3s docker option is no longer supported.") ];
376
377 # interface
378 options.services.k3s = {
379 enable = lib.mkEnableOption "k3s";
380
381 package = lib.mkPackageOption pkgs "k3s" { };
382
383 role = lib.mkOption {
384 description = ''
385 Whether k3s should run as a server or agent.
386
387 If it's a server:
388
389 - By default it also runs workloads as an agent.
390 - Starts by default as a standalone server using an embedded sqlite datastore.
391 - Configure `clusterInit = true` to switch over to embedded etcd datastore and enable HA mode.
392 - Configure `serverAddr` to join an already-initialized HA cluster.
393
394 If it's an agent:
395
396 - `serverAddr` is required.
397 '';
398 default = "server";
399 type = lib.types.enum [
400 "server"
401 "agent"
402 ];
403 };
404
405 serverAddr = lib.mkOption {
406 type = lib.types.str;
407 description = ''
408 The k3s server to connect to.
409
410 Servers and agents need to communicate each other. Read
411 [the networking docs](https://rancher.com/docs/k3s/latest/en/installation/installation-requirements/#networking)
412 to know how to configure the firewall.
413 '';
414 example = "https://10.0.0.10:6443";
415 default = "";
416 };
417
418 clusterInit = lib.mkOption {
419 type = lib.types.bool;
420 default = false;
421 description = ''
422 Initialize HA cluster using an embedded etcd datastore.
423
424 If this option is `false` and `role` is `server`
425
426 On a server that was using the default embedded sqlite backend,
427 enabling this option will migrate to an embedded etcd DB.
428
429 If an HA cluster using the embedded etcd datastore was already initialized,
430 this option has no effect.
431
432 This option only makes sense in a server that is not connecting to another server.
433
434 If you are configuring an HA cluster with an embedded etcd,
435 the 1st server must have `clusterInit = true`
436 and other servers must connect to it using `serverAddr`.
437 '';
438 };
439
440 token = lib.mkOption {
441 type = lib.types.str;
442 description = ''
443 The k3s token to use when connecting to a server.
444
445 WARNING: This option will expose store your token unencrypted world-readable in the nix store.
446 If this is undesired use the tokenFile option instead.
447 '';
448 default = "";
449 };
450
451 tokenFile = lib.mkOption {
452 type = lib.types.nullOr lib.types.path;
453 description = "File path containing k3s token to use when connecting to the server.";
454 default = null;
455 };
456
457 extraFlags = lib.mkOption {
458 description = "Extra flags to pass to the k3s command.";
459 type = with lib.types; either str (listOf str);
460 default = [ ];
461 example = [
462 "--disable traefik"
463 "--cluster-cidr 10.24.0.0/16"
464 ];
465 };
466
467 disableAgent = lib.mkOption {
468 type = lib.types.bool;
469 default = false;
470 description = "Only run the server. This option only makes sense for a server.";
471 };
472
473 environmentFile = lib.mkOption {
474 type = lib.types.nullOr lib.types.path;
475 description = ''
476 File path containing environment variables for configuring the k3s service in the format of an EnvironmentFile. See {manpage}`systemd.exec(5)`.
477 '';
478 default = null;
479 };
480
481 configPath = lib.mkOption {
482 type = lib.types.nullOr lib.types.path;
483 default = null;
484 description = "File path containing the k3s YAML config. This is useful when the config is generated (for example on boot).";
485 };
486
487 manifests = lib.mkOption {
488 type = lib.types.attrsOf manifestModule;
489 default = { };
490 example = lib.literalExpression ''
491 {
492 deployment.source = ../manifests/deployment.yaml;
493 my-service = {
494 enable = false;
495 target = "app-service.yaml";
496 content = {
497 apiVersion = "v1";
498 kind = "Service";
499 metadata = {
500 name = "app-service";
501 };
502 spec = {
503 selector = {
504 "app.kubernetes.io/name" = "MyApp";
505 };
506 ports = [
507 {
508 name = "name-of-service-port";
509 protocol = "TCP";
510 port = 80;
511 targetPort = "http-web-svc";
512 }
513 ];
514 };
515 };
516 };
517
518 nginx.content = [
519 {
520 apiVersion = "v1";
521 kind = "Pod";
522 metadata = {
523 name = "nginx";
524 labels = {
525 "app.kubernetes.io/name" = "MyApp";
526 };
527 };
528 spec = {
529 containers = [
530 {
531 name = "nginx";
532 image = "nginx:1.14.2";
533 ports = [
534 {
535 containerPort = 80;
536 name = "http-web-svc";
537 }
538 ];
539 }
540 ];
541 };
542 }
543 {
544 apiVersion = "v1";
545 kind = "Service";
546 metadata = {
547 name = "nginx-service";
548 };
549 spec = {
550 selector = {
551 "app.kubernetes.io/name" = "MyApp";
552 };
553 ports = [
554 {
555 name = "name-of-service-port";
556 protocol = "TCP";
557 port = 80;
558 targetPort = "http-web-svc";
559 }
560 ];
561 };
562 }
563 ];
564 };
565 '';
566 description = ''
567 Auto-deploying manifests that are linked to {file}`${manifestDir}` before k3s starts.
568 Note that deleting manifest files will not remove or otherwise modify the resources
569 it created. Please use the the `--disable` flag or `.skip` files to delete/disable AddOns,
570 as mentioned in the [docs](https://docs.k3s.io/installation/packaged-components#disabling-manifests).
571 This option only makes sense on server nodes (`role = server`).
572 Read the [auto-deploying manifests docs](https://docs.k3s.io/installation/packaged-components#auto-deploying-manifests-addons)
573 for further information.
574 '';
575 };
576
577 charts = lib.mkOption {
578 type = with lib.types; attrsOf (either path package);
579 default = { };
580 example = lib.literalExpression ''
581 nginx = ../charts/my-nginx-chart.tgz;
582 redis = ../charts/my-redis-chart.tgz;
583 '';
584 description = ''
585 Packaged Helm charts that are linked to {file}`${chartDir}` before k3s starts.
586 The attribute name will be used as the link target (relative to {file}`${chartDir}`).
587 The specified charts will only be placed on the file system and made available to the
588 Kubernetes APIServer from within the cluster. See the [](#opt-services.k3s.autoDeployCharts)
589 option and the [k3s Helm controller docs](https://docs.k3s.io/helm#using-the-helm-controller)
590 to deploy Helm charts. This option only makes sense on server nodes (`role = server`).
591 '';
592 };
593
594 containerdConfigTemplate = lib.mkOption {
595 type = lib.types.nullOr lib.types.str;
596 default = null;
597 example = lib.literalExpression ''
598 # Base K3s config
599 {{ template "base" . }}
600
601 # Add a custom runtime
602 [plugins."io.containerd.grpc.v1.cri".containerd.runtimes."custom"]
603 runtime_type = "io.containerd.runc.v2"
604 [plugins."io.containerd.grpc.v1.cri".containerd.runtimes."custom".options]
605 BinaryName = "/path/to/custom-container-runtime"
606 '';
607 description = ''
608 Config template for containerd, to be placed at
609 `/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl`.
610 See the K3s docs on [configuring containerd](https://docs.k3s.io/advanced#configuring-containerd).
611 '';
612 };
613
614 images = lib.mkOption {
615 type = with lib.types; listOf package;
616 default = [ ];
617 example = lib.literalExpression ''
618 [
619 (pkgs.dockerTools.pullImage {
620 imageName = "docker.io/bitnami/keycloak";
621 imageDigest = "sha256:714dfadc66a8e3adea6609bda350345bd3711657b7ef3cf2e8015b526bac2d6b";
622 hash = "sha256-IM2BLZ0EdKIZcRWOtuFY9TogZJXCpKtPZnMnPsGlq0Y=";
623 finalImageTag = "21.1.2-debian-11-r0";
624 })
625
626 config.services.k3s.package.airgapImages
627 ]
628 '';
629 description = ''
630 List of derivations that provide container images.
631 All images are linked to {file}`${imageDir}` before k3s starts and consequently imported
632 by the k3s agent. Consider importing the k3s airgap images archive of the k3s package in
633 use, if you want to pre-provision this node with all k3s container images. This option
634 only makes sense on nodes with an enabled agent.
635 '';
636 };
637
638 gracefulNodeShutdown = {
639 enable = lib.mkEnableOption ''
640 graceful node shutdowns where the kubelet attempts to detect
641 node system shutdown and terminates pods running on the node. See the
642 [documentation](https://kubernetes.io/docs/concepts/cluster-administration/node-shutdown/#graceful-node-shutdown)
643 for further information.
644 '';
645
646 shutdownGracePeriod = lib.mkOption {
647 type = lib.types.nonEmptyStr;
648 default = "30s";
649 example = "1m30s";
650 description = ''
651 Specifies the total duration that the node should delay the shutdown by. This is the total
652 grace period for pod termination for both regular and critical pods.
653 '';
654 };
655
656 shutdownGracePeriodCriticalPods = lib.mkOption {
657 type = lib.types.nonEmptyStr;
658 default = "10s";
659 example = "15s";
660 description = ''
661 Specifies the duration used to terminate critical pods during a node shutdown. This should be
662 less than `shutdownGracePeriod`.
663 '';
664 };
665 };
666
667 extraKubeletConfig = lib.mkOption {
668 type = with lib.types; attrsOf anything;
669 default = { };
670 example = {
671 podsPerCore = 3;
672 memoryThrottlingFactor = 0.69;
673 containerLogMaxSize = "5Mi";
674 };
675 description = ''
676 Extra configuration to add to the kubelet's configuration file. The subset of the kubelet's
677 configuration that can be configured via a file is defined by the
678 [KubeletConfiguration](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/)
679 struct. See the
680 [documentation](https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/)
681 for further information.
682 '';
683 };
684
685 extraKubeProxyConfig = lib.mkOption {
686 type = with lib.types; attrsOf anything;
687 default = { };
688 example = {
689 mode = "nftables";
690 clientConnection.kubeconfig = "/var/lib/rancher/k3s/agent/kubeproxy.kubeconfig";
691 };
692 description = ''
693 Extra configuration to add to the kube-proxy's configuration file. The subset of the kube-proxy's
694 configuration that can be configured via a file is defined by the
695 [KubeProxyConfiguration](https://kubernetes.io/docs/reference/config-api/kube-proxy-config.v1alpha1/)
696 struct. Note that the kubeconfig param will be override by `clientConnection.kubeconfig`, so you must
697 set the `clientConnection.kubeconfig` if you want to use `extraKubeProxyConfig`.
698 '';
699 };
700
701 autoDeployCharts = lib.mkOption {
702 type = lib.types.attrsOf autoDeployChartsModule;
703 apply = lib.mapAttrs mkAutoDeployChartManifest;
704 default = { };
705 example = lib.literalExpression ''
706 {
707 harbor = {
708 name = "harbor";
709 repo = "https://helm.goharbor.io";
710 version = "1.14.0";
711 hash = "sha256-fMP7q1MIbvzPGS9My91vbQ1d3OJMjwc+o8YE/BXZaYU=";
712 values = {
713 existingSecretAdminPassword = "harbor-admin";
714 expose = {
715 tls = {
716 enabled = true;
717 certSource = "secret";
718 secret.secretName = "my-tls-secret";
719 };
720 ingress = {
721 hosts.core = "example.com";
722 className = "nginx";
723 };
724 };
725 };
726 };
727
728 custom-chart = {
729 package = ../charts/my-chart.tgz;
730 values = ../values/my-values.yaml;
731 extraFieldDefinitions = {
732 spec.timeout = "60s";
733 };
734 };
735 }
736 '';
737 description = ''
738 Auto deploying Helm charts that are installed by the k3s Helm controller. Avoid to use
739 attribute names that are also used in the [](#opt-services.k3s.manifests) and
740 [](#opt-services.k3s.charts) options. Manifests with the same name will override
741 auto deploying charts with the same name. Similiarly, charts with the same name will
742 overwrite the Helm chart contained in auto deploying charts. This option only makes
743 sense on server nodes (`role = server`). See the
744 [k3s Helm documentation](https://docs.k3s.io/helm) for further information.
745 '';
746 };
747 };
748
749 # implementation
750
751 config = lib.mkIf cfg.enable {
752 warnings =
753 (lib.optional (cfg.role != "server" && cfg.manifests != { })
754 "k3s: Auto deploying manifests are only installed on server nodes (role == server), they will be ignored by this node."
755 )
756 ++ (lib.optional (cfg.role != "server" && cfg.charts != { })
757 "k3s: Helm charts are only made available to the cluster on server nodes (role == server), they will be ignored by this node."
758 )
759 ++ (lib.optional (cfg.role != "server" && cfg.autoDeployCharts != { })
760 "k3s: Auto deploying Helm charts are only installed on server nodes (role == server), they will be ignored by this node."
761 )
762 ++ (lib.optional (duplicateManifests != [ ])
763 "k3s: The following auto deploying charts are overriden by manifests of the same name: ${toString duplicateManifests}."
764 )
765 ++ (lib.optional (duplicateCharts != [ ])
766 "k3s: The following auto deploying charts are overriden by charts of the same name: ${toString duplicateCharts}."
767 )
768 ++ (lib.optional (
769 cfg.disableAgent && cfg.images != [ ]
770 ) "k3s: Images are only imported on nodes with an enabled agent, they will be ignored by this node")
771 ++ (lib.optional (
772 cfg.role == "agent" && cfg.configPath == null && cfg.serverAddr == ""
773 ) "k3s: ServerAddr or configPath (with 'server' key) should be set if role is 'agent'")
774 ++ (lib.optional
775 (cfg.role == "agent" && cfg.configPath == null && cfg.tokenFile == null && cfg.token == "")
776 "k3s: Token or tokenFile or configPath (with 'token' or 'token-file' keys) should be set if role is 'agent'"
777 );
778
779 assertions = [
780 {
781 assertion = cfg.role == "agent" -> !cfg.disableAgent;
782 message = "k3s: disableAgent must be false if role is 'agent'";
783 }
784 {
785 assertion = cfg.role == "agent" -> !cfg.clusterInit;
786 message = "k3s: clusterInit must be false if role is 'agent'";
787 }
788 ];
789
790 environment.systemPackages = [ config.services.k3s.package ];
791
792 # Use systemd-tmpfiles to activate k3s content
793 systemd.tmpfiles.settings."10-k3s" =
794 let
795 # Merge manifest with manifests generated from auto deploying charts, keep only enabled manifests
796 enabledManifests = lib.filterAttrs (_: v: v.enable) (cfg.autoDeployCharts // cfg.manifests);
797 # Merge charts with charts contained in enabled auto deploying charts
798 helmCharts =
799 (lib.concatMapAttrs (n: v: { ${n} = v.package; }) (
800 lib.filterAttrs (_: v: v.enable) cfg.autoDeployCharts
801 ))
802 // cfg.charts;
803 # Make a systemd-tmpfiles rule for a manifest
804 mkManifestRule = manifest: {
805 name = "${manifestDir}/${manifest.target}";
806 value = {
807 "L+".argument = "${manifest.source}";
808 };
809 };
810 # Ensure that all chart targets have a .tgz suffix
811 mkChartTarget = name: if (lib.hasSuffix ".tgz" name) then name else name + ".tgz";
812 # Make a systemd-tmpfiles rule for a chart
813 mkChartRule = target: source: {
814 name = "${chartDir}/${mkChartTarget target}";
815 value = {
816 "L+".argument = "${source}";
817 };
818 };
819 # Make a systemd-tmpfiles rule for a container image
820 mkImageRule = image: {
821 name = "${imageDir}/${image.name}";
822 value = {
823 "L+".argument = "${image}";
824 };
825 };
826 in
827 (lib.mapAttrs' (_: v: mkManifestRule v) enabledManifests)
828 // (lib.mapAttrs' (n: v: mkChartRule n v) helmCharts)
829 // (builtins.listToAttrs (map mkImageRule cfg.images))
830 // (lib.optionalAttrs (cfg.containerdConfigTemplate != null) {
831 ${containerdConfigTemplateFile} = {
832 "L+".argument = "${pkgs.writeText "config.toml.tmpl" cfg.containerdConfigTemplate}";
833 };
834 });
835
836 systemd.services.k3s =
837 let
838 kubeletParams =
839 (lib.optionalAttrs (cfg.gracefulNodeShutdown.enable) {
840 inherit (cfg.gracefulNodeShutdown) shutdownGracePeriod shutdownGracePeriodCriticalPods;
841 })
842 // cfg.extraKubeletConfig;
843 kubeletConfig = (pkgs.formats.yaml { }).generate "k3s-kubelet-config" (
844 {
845 apiVersion = "kubelet.config.k8s.io/v1beta1";
846 kind = "KubeletConfiguration";
847 }
848 // kubeletParams
849 );
850
851 kubeProxyConfig = (pkgs.formats.yaml { }).generate "k3s-kubeProxy-config" (
852 {
853 apiVersion = "kubeproxy.config.k8s.io/v1alpha1";
854 kind = "KubeProxyConfiguration";
855 }
856 // cfg.extraKubeProxyConfig
857 );
858 in
859 {
860 description = "k3s service";
861 after = [
862 "firewall.service"
863 "network-online.target"
864 ];
865 wants = [
866 "firewall.service"
867 "network-online.target"
868 ];
869 wantedBy = [ "multi-user.target" ];
870 path = lib.optional config.boot.zfs.enabled config.boot.zfs.package;
871 serviceConfig = {
872 # See: https://github.com/rancher/k3s/blob/dddbd16305284ae4bd14c0aade892412310d7edc/install.sh#L197
873 Type = if cfg.role == "agent" then "exec" else "notify";
874 KillMode = "process";
875 Delegate = "yes";
876 Restart = "always";
877 RestartSec = "5s";
878 LimitNOFILE = 1048576;
879 LimitNPROC = "infinity";
880 LimitCORE = "infinity";
881 TasksMax = "infinity";
882 EnvironmentFile = cfg.environmentFile;
883 ExecStart = lib.concatStringsSep " \\\n " (
884 [ "${cfg.package}/bin/k3s ${cfg.role}" ]
885 ++ (lib.optional cfg.clusterInit "--cluster-init")
886 ++ (lib.optional cfg.disableAgent "--disable-agent")
887 ++ (lib.optional (cfg.serverAddr != "") "--server ${cfg.serverAddr}")
888 ++ (lib.optional (cfg.token != "") "--token ${cfg.token}")
889 ++ (lib.optional (cfg.tokenFile != null) "--token-file ${cfg.tokenFile}")
890 ++ (lib.optional (cfg.configPath != null) "--config ${cfg.configPath}")
891 ++ (lib.optional (kubeletParams != { }) "--kubelet-arg=config=${kubeletConfig}")
892 ++ (lib.optional (cfg.extraKubeProxyConfig != { }) "--kube-proxy-arg=config=${kubeProxyConfig}")
893 ++ (lib.flatten cfg.extraFlags)
894 );
895 };
896 };
897 };
898
899 meta.maintainers = lib.teams.k3s.members;
900}