···
9
-
pauseImage = pkgs.dockerTools.streamLayeredImage {
10
-
name = "test.local/pause";
9
+
throwSystem = throw "RKE2: Unsupported system: ${pkgs.stdenv.hostPlatform.system}";
12
+
aarch64-linux = rke2.images-core-linux-arm64-tar-zst;
13
+
x86_64-linux = rke2.images-core-linux-amd64-tar-zst;
15
+
.${pkgs.stdenv.hostPlatform.system} or throwSystem;
18
+
aarch64-linux = rke2.images-canal-linux-arm64-tar-zst;
19
+
x86_64-linux = rke2.images-canal-linux-amd64-tar-zst;
21
+
.${pkgs.stdenv.hostPlatform.system} or throwSystem;
22
+
helloImage = pkgs.dockerTools.buildImage {
23
+
name = "test.local/hello";
12
-
contents = pkgs.buildEnv {
13
-
name = "rke2-pause-image-env";
25
+
compressor = "zstd";
26
+
copyToRoot = pkgs.buildEnv {
27
+
name = "rke2-hello-image-env";
21
-
config.Entrypoint = [
28
-
# A daemonset that responds 'server' on port 8000
34
+
# A daemonset that responds 'hello' on port 8000
networkTestDaemonset = pkgs.writeText "test.yml" ''
···
47
-
image: test.local/pause:local
53
+
image: test.local/hello:local
52
-
command: ["socat", "TCP4-LISTEN:8000,fork", "EXEC:echo server"]
58
+
command: ["socat", "TCP4-LISTEN:8000,fork", "EXEC:echo hello"]
tokenFile = pkgs.writeText "token" "p@s$w0rd";
55
-
agentTokenFile = pkgs.writeText "agent-token" "p@s$w0rd";
61
+
agentTokenFile = pkgs.writeText "agent-token" "agentP@s$w0rd";
62
+
# Let flannel use eth1 to enable inter-node communication in tests
63
+
canalConfig = pkgs.writeText "rke2-canal-config.yaml" ''
64
+
apiVersion: helm.cattle.io/v1
65
+
kind: HelmChartConfig
68
+
namespace: kube-system
name = "${rke2.name}-multi-node";
meta.maintainers = rke2.meta.maintainers;
65
-
networking.firewall.enable = false;
66
-
networking.useDHCP = false;
67
-
networking.defaultGateway = "192.168.1.1";
68
-
networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
70
-
address = "192.168.1.1";
88
+
# Setup image archives to be imported by rke2
89
+
systemd.tmpfiles.settings."10-rke2" = {
90
+
"/var/lib/rancher/rke2/agent/images/rke2-images-core.tar.zst" = {
91
+
"L+".argument = "${coreImages}";
93
+
"/var/lib/rancher/rke2/agent/images/rke2-images-canal.tar.zst" = {
94
+
"L+".argument = "${canalImages}";
96
+
"/var/lib/rancher/rke2/agent/images/hello.tar.zst" = {
97
+
"L+".argument = "${helloImage}";
99
+
# Copy the canal config so that rke2 can write the remaining default values to it
100
+
"/var/lib/rancher/rke2/server/manifests/rke2-canal-config.yaml" = {
101
+
"C".argument = "${canalConfig}";
105
+
# Canal CNI with VXLAN
106
+
networking.firewall.allowedUDPPorts = [ 8472 ];
107
+
networking.firewall.allowedTCPPorts = [
110
+
# Canal CNI health checks
112
+
# RKE2 supervisor API
75
-
virtualisation.memorySize = 1536;
76
-
virtualisation.diskSize = 4096;
116
+
# RKE2 needs more resources than the default
117
+
virtualisation.cores = 4;
118
+
virtualisation.memorySize = 4096;
119
+
virtualisation.diskSize = 8092;
83
-
nodeName = "${rke2.name}-server1";
85
-
nodeIP = "192.168.1.1";
127
+
# Without nodeIP the apiserver starts with the wrong service IP family
128
+
nodeIP = config.networking.primaryIPAddress;
133
+
"rke2-snapshot-controller"
134
+
"rke2-snapshot-controller-crd"
135
+
"rke2-snapshot-validation-webhook"
100
-
networking.firewall.enable = false;
101
-
networking.useDHCP = false;
102
-
networking.defaultGateway = "192.168.1.2";
103
-
networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
105
-
address = "192.168.1.2";
110
-
virtualisation.memorySize = 1536;
111
-
virtualisation.diskSize = 4096;
116
-
serverAddr = "https://192.168.1.1:6443";
118
-
inherit agentTokenFile;
119
-
nodeName = "${rke2.name}-server2";
121
-
nodeIP = "192.168.1.2";
124
-
"rke2-metrics-server"
125
-
"rke2-ingress-nginx"
148
+
# Setup image archives to be imported by rke2
149
+
systemd.tmpfiles.settings."10-rke2" = {
150
+
"/var/lib/rancher/rke2/agent/images/rke2-images-core.linux-amd64.tar.zst" = {
151
+
"L+".argument = "${coreImages}";
153
+
"/var/lib/rancher/rke2/agent/images/rke2-images-canal.linux-amd64.tar.zst" = {
154
+
"L+".argument = "${canalImages}";
156
+
"/var/lib/rancher/rke2/agent/images/hello.tar.zst" = {
157
+
"L+".argument = "${helloImage}";
159
+
"/var/lib/rancher/rke2/server/manifests/rke2-canal-config.yaml" = {
160
+
"C".argument = "${canalConfig}";
133
-
networking.firewall.enable = false;
134
-
networking.useDHCP = false;
135
-
networking.defaultGateway = "192.168.1.3";
136
-
networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
138
-
address = "192.168.1.3";
164
+
# Canal CNI health checks
165
+
networking.firewall.allowedTCPPorts = [ 9099 ];
166
+
# Canal CNI with VXLAN
167
+
networking.firewall.allowedUDPPorts = [ 8472 ];
143
-
virtualisation.memorySize = 1536;
144
-
virtualisation.diskSize = 4096;
169
+
# The agent node can work with less resources
170
+
virtualisation.memorySize = 2048;
171
+
virtualisation.diskSize = 8092;
149
-
tokenFile = agentTokenFile;
150
-
serverAddr = "https://192.168.1.2:6443";
151
-
nodeName = "${rke2.name}-agent1";
153
-
nodeIP = "192.168.1.3";
177
+
tokenFile = agentTokenFile;
178
+
serverAddr = "https://${nodes.server.networking.primaryIPAddress}:9345";
179
+
nodeIP = config.networking.primaryIPAddress;
···
kubectl = "${pkgs.kubectl}/bin/kubectl --kubeconfig=/etc/rancher/rke2/rke2.yaml";
161
-
ctr = "${pkgs.containerd}/bin/ctr -a /run/k3s/containerd/containerd.sock";
jq = "${pkgs.jq}/bin/jq";
163
-
ping = "${pkgs.iputils}/bin/ping";
166
-
machines = [server1, server2, agent1]
168
-
for machine in machines:
170
-
machine.wait_for_unit("rke2")
172
-
# wait for the agent to show up
173
-
server1.succeed("${kubectl} get node ${rke2.name}-agent1")
193
+
server.wait_for_unit("rke2-server")
194
+
agent.wait_for_unit("rke2-agent")
175
-
for machine in machines:
176
-
machine.succeed("${pauseImage} | ${ctr} image import -")
196
+
# Wait for the agent to be ready
197
+
server.wait_until_succeeds(r"""${kubectl} wait --for='jsonpath={.status.conditions[?(@.type=="Ready")].status}=True' nodes/agent""")
178
-
server1.succeed("${kubectl} cluster-info")
179
-
server1.wait_until_succeeds("${kubectl} get serviceaccount default")
199
+
server.succeed("${kubectl} cluster-info")
200
+
server.wait_until_succeeds("${kubectl} get serviceaccount default")
# Now create a pod on each node via a daemonset and verify they can talk to each other.
182
-
server1.succeed("${kubectl} apply -f ${networkTestDaemonset}")
183
-
server1.wait_until_succeeds(
203
+
server.succeed("${kubectl} apply -f ${networkTestDaemonset}")
204
+
server.wait_until_succeeds(
f'[ "$(${kubectl} get ds test -o json | ${jq} .status.numberReady)" -eq {len(machines)} ]'
188
-
pods = server1.succeed("${kubectl} get po -o json | ${jq} '.items[].metadata.name' -r").splitlines()
209
+
pods = server.succeed("${kubectl} get po -o json | ${jq} '.items[].metadata.name' -r").splitlines()
190
-
server1.succeed(f"${kubectl} get po {n} -o json | ${jq} '.status.podIP' -cr").strip() for n in pods
211
+
server.succeed(f"${kubectl} get po {n} -o json | ${jq} '.status.podIP' -cr").strip() for n in pods
193
-
# Verify each server can ping each pod ip
214
+
# Verify each node can ping each pod ip
195
-
server1.succeed(f"${ping} -c 1 {pod_ip}")
196
-
agent1.succeed(f"${ping} -c 1 {pod_ip}")
198
-
# Verify the pods can talk to each other
199
-
resp = server1.wait_until_succeeds(f"${kubectl} exec {pods[0]} -- socat TCP:{pod_ips[1]}:8000 -")
200
-
assert resp.strip() == "server"
201
-
resp = server1.wait_until_succeeds(f"${kubectl} exec {pods[1]} -- socat TCP:{pod_ips[0]}:8000 -")
202
-
assert resp.strip() == "server"
205
-
server1.succeed("${kubectl} delete -f ${networkTestDaemonset}")
206
-
for machine in machines:
216
+
# The CNI sometimes needs a little time
217
+
server.wait_until_succeeds(f"ping -c 1 {pod_ip}", timeout=5)
218
+
agent.wait_until_succeeds(f"ping -c 1 {pod_ip}", timeout=5)
219
+
# Verify the server can exec into the pod
221
+
# resp = server.succeed(f"${kubectl} exec {pod} -- socat TCP:{pod_ip}:8000 -")
222
+
# assert resp.strip() == "hello", f"Unexpected response from hello daemonset: {resp.strip()}"
pyrox.dev