pyrox.dev
1import ../make-test-python.nix (
2 {
3 pkgs,
4 lib,
5 k3s,
6 ...
7 }:
8 let
9 imageEnv = pkgs.buildEnv {
10 name = "k3s-pause-image-env";
11 paths = with pkgs; [
12 tini
13 bashInteractive
14 coreutils
15 socat
16 ];
17 };
18 pauseImage = pkgs.dockerTools.streamLayeredImage {
19 name = "test.local/pause";
20 tag = "local";
21 contents = imageEnv;
22 config.Entrypoint = [
23 "/bin/tini"
24 "--"
25 "/bin/sleep"
26 "inf"
27 ];
28 };
29 # A daemonset that responds 'server' on port 8000
30 networkTestDaemonset = pkgs.writeText "test.yml" ''
31 apiVersion: apps/v1
32 kind: DaemonSet
33 metadata:
34 name: test
35 labels:
36 name: test
37 spec:
38 selector:
39 matchLabels:
40 name: test
41 template:
42 metadata:
43 labels:
44 name: test
45 spec:
46 containers:
47 - name: test
48 image: test.local/pause:local
49 imagePullPolicy: Never
50 resources:
51 limits:
52 memory: 20Mi
53 command: ["socat", "TCP4-LISTEN:8000,fork", "EXEC:echo server"]
54 '';
55 tokenFile = pkgs.writeText "token" "p@s$w0rd";
56 in
57 {
58 name = "${k3s.name}-multi-node";
59
60 nodes = {
61 server =
62 { pkgs, ... }:
63 {
64 environment.systemPackages = with pkgs; [
65 gzip
66 jq
67 ];
68 # k3s uses enough resources the default vm fails.
69 virtualisation.memorySize = 1536;
70 virtualisation.diskSize = 4096;
71
72 services.k3s = {
73 inherit tokenFile;
74 enable = true;
75 role = "server";
76 package = k3s;
77 clusterInit = true;
78 extraFlags = builtins.toString [
79 "--disable"
80 "coredns"
81 "--disable"
82 "local-storage"
83 "--disable"
84 "metrics-server"
85 "--disable"
86 "servicelb"
87 "--disable"
88 "traefik"
89 "--node-ip"
90 "192.168.1.1"
91 "--pause-image"
92 "test.local/pause:local"
93 ];
94 };
95 networking.firewall.allowedTCPPorts = [
96 2379
97 2380
98 6443
99 ];
100 networking.firewall.allowedUDPPorts = [ 8472 ];
101 networking.firewall.trustedInterfaces = [ "flannel.1" ];
102 networking.useDHCP = false;
103 networking.defaultGateway = "192.168.1.1";
104 networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
105 {
106 address = "192.168.1.1";
107 prefixLength = 24;
108 }
109 ];
110 };
111
112 server2 =
113 { pkgs, ... }:
114 {
115 environment.systemPackages = with pkgs; [
116 gzip
117 jq
118 ];
119 virtualisation.memorySize = 1536;
120 virtualisation.diskSize = 4096;
121
122 services.k3s = {
123 inherit tokenFile;
124 enable = true;
125 serverAddr = "https://192.168.1.1:6443";
126 clusterInit = false;
127 extraFlags = builtins.toString [
128 "--disable"
129 "coredns"
130 "--disable"
131 "local-storage"
132 "--disable"
133 "metrics-server"
134 "--disable"
135 "servicelb"
136 "--disable"
137 "traefik"
138 "--node-ip"
139 "192.168.1.3"
140 "--pause-image"
141 "test.local/pause:local"
142 ];
143 };
144 networking.firewall.allowedTCPPorts = [
145 2379
146 2380
147 6443
148 ];
149 networking.firewall.allowedUDPPorts = [ 8472 ];
150 networking.firewall.trustedInterfaces = [ "flannel.1" ];
151 networking.useDHCP = false;
152 networking.defaultGateway = "192.168.1.3";
153 networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
154 {
155 address = "192.168.1.3";
156 prefixLength = 24;
157 }
158 ];
159 };
160
161 agent =
162 { pkgs, ... }:
163 {
164 virtualisation.memorySize = 1024;
165 virtualisation.diskSize = 2048;
166 services.k3s = {
167 inherit tokenFile;
168 enable = true;
169 role = "agent";
170 serverAddr = "https://192.168.1.3:6443";
171 extraFlags = lib.concatStringsSep " " [
172 "--pause-image"
173 "test.local/pause:local"
174 "--node-ip"
175 "192.168.1.2"
176 ];
177 };
178 networking.firewall.allowedTCPPorts = [ 6443 ];
179 networking.firewall.allowedUDPPorts = [ 8472 ];
180 networking.firewall.trustedInterfaces = [ "flannel.1" ];
181 networking.useDHCP = false;
182 networking.defaultGateway = "192.168.1.2";
183 networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
184 {
185 address = "192.168.1.2";
186 prefixLength = 24;
187 }
188 ];
189 };
190 };
191
192 meta.maintainers = k3s.meta.maintainers;
193
194 testScript = ''
195 machines = [server, server2, agent]
196 for m in machines:
197 m.start()
198 m.wait_for_unit("k3s")
199
200 is_aarch64 = "${toString pkgs.stdenv.isAarch64}" == "1"
201
202 # wait for the agent to show up
203 server.wait_until_succeeds("k3s kubectl get node agent")
204
205 for m in machines:
206 # Fix-Me: Tests fail for 'aarch64-linux' as: "CONFIG_CGROUP_FREEZER: missing (fail)"
207 if not is_aarch64:
208 m.succeed("k3s check-config")
209 m.succeed(
210 "${pauseImage} | k3s ctr image import -"
211 )
212
213 server.succeed("k3s kubectl cluster-info")
214 # Also wait for our service account to show up; it takes a sec
215 server.wait_until_succeeds("k3s kubectl get serviceaccount default")
216
217 # Now create a pod on each node via a daemonset and verify they can talk to each other.
218 server.succeed("k3s kubectl apply -f ${networkTestDaemonset}")
219 server.wait_until_succeeds(f'[ "$(k3s kubectl get ds test -o json | jq .status.numberReady)" -eq {len(machines)} ]')
220
221 # Get pod IPs
222 pods = server.succeed("k3s kubectl get po -o json | jq '.items[].metadata.name' -r").splitlines()
223 pod_ips = [server.succeed(f"k3s kubectl get po {name} -o json | jq '.status.podIP' -cr").strip() for name in pods]
224
225 # Verify each server can ping each pod ip
226 for pod_ip in pod_ips:
227 server.succeed(f"ping -c 1 {pod_ip}")
228 agent.succeed(f"ping -c 1 {pod_ip}")
229
230 # Verify the pods can talk to each other
231 resp = server.wait_until_succeeds(f"k3s kubectl exec {pods[0]} -- socat TCP:{pod_ips[1]}:8000 -")
232 assert resp.strip() == "server"
233 resp = server.wait_until_succeeds(f"k3s kubectl exec {pods[1]} -- socat TCP:{pod_ips[0]}:8000 -")
234 assert resp.strip() == "server"
235
236 # Cleanup
237 server.succeed("k3s kubectl delete -f ${networkTestDaemonset}")
238
239 for m in machines:
240 m.shutdown()
241 '';
242 }
243)