pyrox.dev
1import ../make-test-python.nix (
2 {
3 pkgs,
4 lib,
5 rke2,
6 ...
7 }:
8 let
9 throwSystem = throw "RKE2: Unsupported system: ${pkgs.stdenv.hostPlatform.system}";
10 coreImages =
11 {
12 aarch64-linux = rke2.images-core-linux-arm64-tar-zst;
13 x86_64-linux = rke2.images-core-linux-amd64-tar-zst;
14 }
15 .${pkgs.stdenv.hostPlatform.system} or throwSystem;
16 canalImages =
17 {
18 aarch64-linux = rke2.images-canal-linux-arm64-tar-zst;
19 x86_64-linux = rke2.images-canal-linux-amd64-tar-zst;
20 }
21 .${pkgs.stdenv.hostPlatform.system} or throwSystem;
22 helloImage = pkgs.dockerTools.buildImage {
23 name = "test.local/hello";
24 tag = "local";
25 compressor = "zstd";
26 copyToRoot = pkgs.buildEnv {
27 name = "rke2-hello-image-env";
28 paths = with pkgs; [
29 coreutils
30 socat
31 ];
32 };
33 };
34 # A daemonset that responds 'hello' on port 8000
35 networkTestDaemonset = pkgs.writeText "test.yml" ''
36 apiVersion: apps/v1
37 kind: DaemonSet
38 metadata:
39 name: test
40 labels:
41 name: test
42 spec:
43 selector:
44 matchLabels:
45 name: test
46 template:
47 metadata:
48 labels:
49 name: test
50 spec:
51 containers:
52 - name: test
53 image: test.local/hello:local
54 imagePullPolicy: Never
55 resources:
56 limits:
57 memory: 20Mi
58 command: ["socat", "TCP4-LISTEN:8000,fork", "EXEC:echo hello"]
59 '';
60 tokenFile = pkgs.writeText "token" "p@s$w0rd";
61 agentTokenFile = pkgs.writeText "agent-token" "agentP@s$w0rd";
62 # Let flannel use eth1 to enable inter-node communication in tests
63 canalConfig = pkgs.writeText "rke2-canal-config.yaml" ''
64 apiVersion: helm.cattle.io/v1
65 kind: HelmChartConfig
66 metadata:
67 name: rke2-canal
68 namespace: kube-system
69 spec:
70 valuesContent: |-
71 flannel:
72 iface: "eth1"
73 '';
74 in
75 {
76 name = "${rke2.name}-multi-node";
77 meta.maintainers = rke2.meta.maintainers;
78
79 nodes = {
80 server =
81 {
82 config,
83 nodes,
84 pkgs,
85 ...
86 }:
87 {
88 # Setup image archives to be imported by rke2
89 systemd.tmpfiles.settings."10-rke2" = {
90 "/var/lib/rancher/rke2/agent/images/rke2-images-core.tar.zst" = {
91 "L+".argument = "${coreImages}";
92 };
93 "/var/lib/rancher/rke2/agent/images/rke2-images-canal.tar.zst" = {
94 "L+".argument = "${canalImages}";
95 };
96 "/var/lib/rancher/rke2/agent/images/hello.tar.zst" = {
97 "L+".argument = "${helloImage}";
98 };
99 # Copy the canal config so that rke2 can write the remaining default values to it
100 "/var/lib/rancher/rke2/server/manifests/rke2-canal-config.yaml" = {
101 "C".argument = "${canalConfig}";
102 };
103 };
104
105 # Canal CNI with VXLAN
106 networking.firewall.allowedUDPPorts = [ 8472 ];
107 networking.firewall.allowedTCPPorts = [
108 # Kubernetes API
109 6443
110 # Canal CNI health checks
111 9099
112 # RKE2 supervisor API
113 9345
114 ];
115
116 # RKE2 needs more resources than the default
117 virtualisation.cores = 4;
118 virtualisation.memorySize = 4096;
119 virtualisation.diskSize = 8092;
120
121 services.rke2 = {
122 enable = true;
123 role = "server";
124 package = rke2;
125 inherit tokenFile;
126 inherit agentTokenFile;
127 # Without nodeIP the apiserver starts with the wrong service IP family
128 nodeIP = config.networking.primaryIPAddress;
129 disable = [
130 "rke2-coredns"
131 "rke2-metrics-server"
132 "rke2-ingress-nginx"
133 "rke2-snapshot-controller"
134 "rke2-snapshot-controller-crd"
135 "rke2-snapshot-validation-webhook"
136 ];
137 };
138 };
139
140 agent =
141 {
142 config,
143 nodes,
144 pkgs,
145 ...
146 }:
147 {
148 # Setup image archives to be imported by rke2
149 systemd.tmpfiles.settings."10-rke2" = {
150 "/var/lib/rancher/rke2/agent/images/rke2-images-core.linux-amd64.tar.zst" = {
151 "L+".argument = "${coreImages}";
152 };
153 "/var/lib/rancher/rke2/agent/images/rke2-images-canal.linux-amd64.tar.zst" = {
154 "L+".argument = "${canalImages}";
155 };
156 "/var/lib/rancher/rke2/agent/images/hello.tar.zst" = {
157 "L+".argument = "${helloImage}";
158 };
159 "/var/lib/rancher/rke2/server/manifests/rke2-canal-config.yaml" = {
160 "C".argument = "${canalConfig}";
161 };
162 };
163
164 # Canal CNI health checks
165 networking.firewall.allowedTCPPorts = [ 9099 ];
166 # Canal CNI with VXLAN
167 networking.firewall.allowedUDPPorts = [ 8472 ];
168
169 # The agent node can work with less resources
170 virtualisation.memorySize = 2048;
171 virtualisation.diskSize = 8092;
172
173 services.rke2 = {
174 enable = true;
175 role = "agent";
176 package = rke2;
177 tokenFile = agentTokenFile;
178 serverAddr = "https://${nodes.server.networking.primaryIPAddress}:9345";
179 nodeIP = config.networking.primaryIPAddress;
180 };
181 };
182 };
183
184 testScript =
185 let
186 kubectl = "${pkgs.kubectl}/bin/kubectl --kubeconfig=/etc/rancher/rke2/rke2.yaml";
187 jq = "${pkgs.jq}/bin/jq";
188 in
189 # python
190 ''
191 start_all()
192
193 server.wait_for_unit("rke2-server")
194 agent.wait_for_unit("rke2-agent")
195
196 # Wait for the agent to be ready
197 server.wait_until_succeeds(r"""${kubectl} wait --for='jsonpath={.status.conditions[?(@.type=="Ready")].status}=True' nodes/agent""")
198
199 server.succeed("${kubectl} cluster-info")
200 server.wait_until_succeeds("${kubectl} get serviceaccount default")
201
202 # Now create a pod on each node via a daemonset and verify they can talk to each other.
203 server.succeed("${kubectl} apply -f ${networkTestDaemonset}")
204 server.wait_until_succeeds(
205 f'[ "$(${kubectl} get ds test -o json | ${jq} .status.numberReady)" -eq {len(machines)} ]'
206 )
207
208 # Get pod IPs
209 pods = server.succeed("${kubectl} get po -o json | ${jq} '.items[].metadata.name' -r").splitlines()
210 pod_ips = [
211 server.succeed(f"${kubectl} get po {n} -o json | ${jq} '.status.podIP' -cr").strip() for n in pods
212 ]
213
214 # Verify each node can ping each pod ip
215 for pod_ip in pod_ips:
216 # The CNI sometimes needs a little time
217 server.wait_until_succeeds(f"ping -c 1 {pod_ip}", timeout=5)
218 agent.wait_until_succeeds(f"ping -c 1 {pod_ip}", timeout=5)
219 # Verify the server can exec into the pod
220 # for pod in pods:
221 # resp = server.succeed(f"${kubectl} exec {pod} -- socat TCP:{pod_ip}:8000 -")
222 # assert resp.strip() == "hello", f"Unexpected response from hello daemonset: {resp.strip()}"
223 '';
224 }
225)