pyrox.dev
1# A test that runs a multi-node k3s cluster and verify pod networking works across nodes
2import ../make-test-python.nix (
3 {
4 pkgs,
5 lib,
6 k3s,
7 ...
8 }:
9 let
10 imageEnv = pkgs.buildEnv {
11 name = "k3s-pause-image-env";
12 paths = with pkgs; [
13 tini
14 bashInteractive
15 coreutils
16 socat
17 ];
18 };
19 pauseImage = pkgs.dockerTools.buildImage {
20 name = "test.local/pause";
21 tag = "local";
22 copyToRoot = imageEnv;
23 config.Entrypoint = [
24 "/bin/tini"
25 "--"
26 "/bin/sleep"
27 "inf"
28 ];
29 };
30 # A daemonset that responds 'server' on port 8000
31 networkTestDaemonset = pkgs.writeText "test.yml" ''
32 apiVersion: apps/v1
33 kind: DaemonSet
34 metadata:
35 name: test
36 labels:
37 name: test
38 spec:
39 selector:
40 matchLabels:
41 name: test
42 template:
43 metadata:
44 labels:
45 name: test
46 spec:
47 containers:
48 - name: test
49 image: test.local/pause:local
50 imagePullPolicy: Never
51 resources:
52 limits:
53 memory: 20Mi
54 command: ["socat", "TCP4-LISTEN:8000,fork", "EXEC:echo server"]
55 '';
56 tokenFile = pkgs.writeText "token" "p@s$w0rd";
57 in
58 {
59 name = "${k3s.name}-multi-node";
60
61 nodes = {
62 server =
63 { nodes, pkgs, ... }:
64 {
65 environment.systemPackages = with pkgs; [
66 gzip
67 jq
68 ];
69 # k3s uses enough resources the default vm fails.
70 virtualisation.memorySize = 1536;
71 virtualisation.diskSize = 4096;
72
73 services.k3s = {
74 inherit tokenFile;
75 enable = true;
76 role = "server";
77 package = k3s;
78 images = [ pauseImage ];
79 clusterInit = true;
80 extraFlags = [
81 "--disable coredns"
82 "--disable local-storage"
83 "--disable metrics-server"
84 "--disable servicelb"
85 "--disable traefik"
86 "--pause-image test.local/pause:local"
87 "--node-ip ${nodes.server.networking.primaryIPAddress}"
88 # The interface selection logic of flannel would normally use eth0, as the nixos
89 # testing driver sets a default route via dev eth0. However, in test setups we
90 # have to use eth1 for inter-node communication.
91 "--flannel-iface eth1"
92 ];
93 };
94 networking.firewall.allowedTCPPorts = [
95 2379
96 2380
97 6443
98 ];
99 networking.firewall.allowedUDPPorts = [ 8472 ];
100 };
101
102 server2 =
103 { nodes, pkgs, ... }:
104 {
105 environment.systemPackages = with pkgs; [
106 gzip
107 jq
108 ];
109 virtualisation.memorySize = 1536;
110 virtualisation.diskSize = 4096;
111
112 services.k3s = {
113 inherit tokenFile;
114 enable = true;
115 package = k3s;
116 images = [ pauseImage ];
117 serverAddr = "https://${nodes.server.networking.primaryIPAddress}:6443";
118 clusterInit = false;
119 extraFlags = [
120 "--disable coredns"
121 "--disable local-storage"
122 "--disable metrics-server"
123 "--disable servicelb"
124 "--disable traefik"
125 "--pause-image test.local/pause:local"
126 "--node-ip ${nodes.server2.networking.primaryIPAddress}"
127 "--flannel-iface eth1"
128 ];
129 };
130 networking.firewall.allowedTCPPorts = [
131 2379
132 2380
133 6443
134 ];
135 networking.firewall.allowedUDPPorts = [ 8472 ];
136 };
137
138 agent =
139 { nodes, pkgs, ... }:
140 {
141 virtualisation.memorySize = 1024;
142 virtualisation.diskSize = 2048;
143 services.k3s = {
144 inherit tokenFile;
145 enable = true;
146 role = "agent";
147 package = k3s;
148 images = [ pauseImage ];
149 serverAddr = "https://${nodes.server2.networking.primaryIPAddress}:6443";
150 extraFlags = [
151 "--pause-image test.local/pause:local"
152 "--node-ip ${nodes.agent.networking.primaryIPAddress}"
153 "--flannel-iface eth1"
154 ];
155 };
156 networking.firewall.allowedTCPPorts = [ 6443 ];
157 networking.firewall.allowedUDPPorts = [ 8472 ];
158 };
159 };
160
161 testScript = # python
162 ''
163 start_all()
164
165 machines = [server, server2, agent]
166 for m in machines:
167 m.wait_for_unit("k3s")
168
169 # wait for the agent to show up
170 server.wait_until_succeeds("k3s kubectl get node agent")
171
172 for m in machines:
173 m.succeed("k3s check-config")
174
175 server.succeed("k3s kubectl cluster-info")
176 # Also wait for our service account to show up; it takes a sec
177 server.wait_until_succeeds("k3s kubectl get serviceaccount default")
178
179 # Now create a pod on each node via a daemonset and verify they can talk to each other.
180 server.succeed("k3s kubectl apply -f ${networkTestDaemonset}")
181 server.wait_until_succeeds(f'[ "$(k3s kubectl get ds test -o json | jq .status.numberReady)" -eq {len(machines)} ]')
182
183 # Get pod IPs
184 pods = server.succeed("k3s kubectl get po -o json | jq '.items[].metadata.name' -r").splitlines()
185 pod_ips = [server.succeed(f"k3s kubectl get po {name} -o json | jq '.status.podIP' -cr").strip() for name in pods]
186
187 # Verify each server can ping each pod ip
188 for pod_ip in pod_ips:
189 server.succeed(f"ping -c 1 {pod_ip}")
190 server2.succeed(f"ping -c 1 {pod_ip}")
191 agent.succeed(f"ping -c 1 {pod_ip}")
192 # Verify the pods can talk to each other
193 for pod in pods:
194 resp = server.succeed(f"k3s kubectl exec {pod} -- socat TCP:{pod_ip}:8000 -")
195 t.assertEqual(resp.strip(), "server")
196 '';
197
198 meta.maintainers = lib.teams.k3s.members;
199 }
200)