pyrox.dev
1# Tests whether container images are imported and auto deploying Helm charts,
2# including the bundled traefik, work
3import ../make-test-python.nix (
4 {
5 k3s,
6 lib,
7 pkgs,
8 ...
9 }:
10 let
11 testImageEnv = pkgs.buildEnv {
12 name = "k3s-pause-image-env";
13 paths = with pkgs; [
14 busybox
15 hello
16 ];
17 };
18 testImage = pkgs.dockerTools.buildImage {
19 name = "test.local/test";
20 tag = "local";
21 # Slightly reduces the time needed to import image
22 compressor = "zstd";
23 copyToRoot = testImageEnv;
24 };
25 # pack the test helm chart as a .tgz archive
26 package =
27 pkgs.runCommand "k3s-test-chart.tgz"
28 {
29 nativeBuildInputs = [ pkgs.kubernetes-helm ];
30 }
31 ''
32 helm package ${./k3s-test-chart}
33 mv ./*.tgz $out
34 '';
35 # The common Helm chart that is used in this test
36 testChart = {
37 inherit package;
38 values = {
39 runCommand = "hello";
40 image = {
41 repository = testImage.imageName;
42 tag = testImage.imageTag;
43 };
44 };
45 };
46 in
47 {
48 name = "${k3s.name}-auto-deploy-helm";
49 meta.maintainers = lib.teams.k3s.members;
50 nodes.machine =
51 { pkgs, ... }:
52 {
53 # k3s uses enough resources the default vm fails.
54 virtualisation = {
55 memorySize = 1536;
56 diskSize = 4096;
57 };
58 environment.systemPackages = [ pkgs.yq-go ];
59 services.k3s = {
60 enable = true;
61 package = k3s;
62 # Slightly reduce resource usage
63 extraFlags = [
64 "--disable coredns"
65 "--disable local-storage"
66 "--disable metrics-server"
67 "--disable servicelb"
68 ];
69 images = [
70 # Provides the k3s Helm controller
71 k3s.airgap-images
72 testImage
73 ];
74 autoDeployCharts = {
75 # regular test chart that should get installed
76 hello = testChart;
77 # disabled chart that should not get installed
78 disabled = testChart // {
79 enable = false;
80 };
81 # chart with values set via YAML file
82 values-file = testChart // {
83 # Remove unsafeDiscardStringContext workaround when Nix can convert a string to a path
84 # https://github.com/NixOS/nix/issues/12407
85 values = /.
86 + builtins.unsafeDiscardStringContext (
87 builtins.toFile "k3s-test-chart-values.yaml" ''
88 runCommand: "echo 'Hello, file!'"
89 image:
90 repository: test.local/test
91 tag: local
92 ''
93 );
94 };
95 # advanced chart that should get installed in the "test" namespace with a custom
96 # timeout and overridden values
97 advanced = testChart // {
98 # create the "test" namespace via extraDeploy for testing
99 extraDeploy = [
100 {
101 apiVersion = "v1";
102 kind = "Namespace";
103 metadata.name = "test";
104 }
105 ];
106 extraFieldDefinitions = {
107 spec = {
108 # overwrite chart values
109 valuesContent = ''
110 runCommand: "echo 'advanced hello'"
111 image:
112 repository: ${testImage.imageName}
113 tag: ${testImage.imageTag}
114 '';
115 # overwrite the chart namespace
116 targetNamespace = "test";
117 # set a custom timeout
118 timeout = "69s";
119 };
120 };
121 };
122 };
123 };
124 };
125
126 testScript = # python
127 ''
128 import json
129
130 machine.wait_for_unit("k3s")
131 # check existence/absence of chart manifest files
132 machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/hello.yaml")
133 machine.succeed("test ! -e /var/lib/rancher/k3s/server/manifests/disabled.yaml")
134 machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/values-file.yaml")
135 machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/advanced.yaml")
136 # check that the timeout is set correctly, select only the first doc in advanced.yaml
137 advancedManifest = json.loads(machine.succeed("yq -o json 'select(di == 0)' /var/lib/rancher/k3s/server/manifests/advanced.yaml"))
138 t.assertEqual(advancedManifest["spec"]["timeout"], "69s", "unexpected value for spec.timeout")
139 # wait for test jobs to complete
140 machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello", timeout=180)
141 machine.wait_until_succeeds("kubectl wait --for=condition=complete job/values-file", timeout=180)
142 machine.wait_until_succeeds("kubectl -n test wait --for=condition=complete job/advanced", timeout=180)
143 # check output of test jobs
144 hello_output = machine.succeed("kubectl logs -l batch.kubernetes.io/job-name=hello")
145 values_file_output = machine.succeed("kubectl logs -l batch.kubernetes.io/job-name=values-file")
146 advanced_output = machine.succeed("kubectl -n test logs -l batch.kubernetes.io/job-name=advanced")
147 # strip the output to remove trailing whitespaces
148 t.assertEqual(hello_output.rstrip(), "Hello, world!", "unexpected output of hello job")
149 t.assertEqual(values_file_output.rstrip(), "Hello, file!", "unexpected output of values file job")
150 t.assertEqual(advanced_output.rstrip(), "advanced hello", "unexpected output of advanced job")
151 # wait for bundled traefik deployment
152 machine.wait_until_succeeds("kubectl -n kube-system rollout status deployment traefik", timeout=180)
153 '';
154 }
155)