pyrox.dev
1import ../make-test-python.nix ({ pkgs, lib, k3s, ... }:
2 let
3 imageEnv = pkgs.buildEnv {
4 name = "k3s-pause-image-env";
5 paths = with pkgs; [ tini bashInteractive coreutils socat ];
6 };
7 pauseImage = pkgs.dockerTools.streamLayeredImage {
8 name = "test.local/pause";
9 tag = "local";
10 contents = imageEnv;
11 config.Entrypoint = [ "/bin/tini" "--" "/bin/sleep" "inf" ];
12 };
13 # A daemonset that responds 'server' on port 8000
14 networkTestDaemonset = pkgs.writeText "test.yml" ''
15 apiVersion: apps/v1
16 kind: DaemonSet
17 metadata:
18 name: test
19 labels:
20 name: test
21 spec:
22 selector:
23 matchLabels:
24 name: test
25 template:
26 metadata:
27 labels:
28 name: test
29 spec:
30 containers:
31 - name: test
32 image: test.local/pause:local
33 imagePullPolicy: Never
34 resources:
35 limits:
36 memory: 20Mi
37 command: ["socat", "TCP4-LISTEN:8000,fork", "EXEC:echo server"]
38 '';
39 tokenFile = pkgs.writeText "token" "p@s$w0rd";
40 in
41 {
42 name = "${k3s.name}-multi-node";
43
44 nodes = {
45 server = { pkgs, ... }: {
46 environment.systemPackages = with pkgs; [ gzip jq ];
47 # k3s uses enough resources the default vm fails.
48 virtualisation.memorySize = 1536;
49 virtualisation.diskSize = 4096;
50
51 services.k3s = {
52 inherit tokenFile;
53 enable = true;
54 role = "server";
55 package = k3s;
56 clusterInit = true;
57 extraFlags = builtins.toString [
58 "--disable" "coredns"
59 "--disable" "local-storage"
60 "--disable" "metrics-server"
61 "--disable" "servicelb"
62 "--disable" "traefik"
63 "--node-ip" "192.168.1.1"
64 "--pause-image" "test.local/pause:local"
65 ];
66 };
67 networking.firewall.allowedTCPPorts = [ 2379 2380 6443 ];
68 networking.firewall.allowedUDPPorts = [ 8472 ];
69 networking.firewall.trustedInterfaces = [ "flannel.1" ];
70 networking.useDHCP = false;
71 networking.defaultGateway = "192.168.1.1";
72 networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
73 { address = "192.168.1.1"; prefixLength = 24; }
74 ];
75 };
76
77 server2 = { pkgs, ... }: {
78 environment.systemPackages = with pkgs; [ gzip jq ];
79 virtualisation.memorySize = 1536;
80 virtualisation.diskSize = 4096;
81
82 services.k3s = {
83 inherit tokenFile;
84 enable = true;
85 serverAddr = "https://192.168.1.1:6443";
86 clusterInit = false;
87 extraFlags = builtins.toString [
88 "--disable" "coredns"
89 "--disable" "local-storage"
90 "--disable" "metrics-server"
91 "--disable" "servicelb"
92 "--disable" "traefik"
93 "--node-ip" "192.168.1.3"
94 "--pause-image" "test.local/pause:local"
95 ];
96 };
97 networking.firewall.allowedTCPPorts = [ 2379 2380 6443 ];
98 networking.firewall.allowedUDPPorts = [ 8472 ];
99 networking.firewall.trustedInterfaces = [ "flannel.1" ];
100 networking.useDHCP = false;
101 networking.defaultGateway = "192.168.1.3";
102 networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
103 { address = "192.168.1.3"; prefixLength = 24; }
104 ];
105 };
106
107 agent = { pkgs, ... }: {
108 virtualisation.memorySize = 1024;
109 virtualisation.diskSize = 2048;
110 services.k3s = {
111 inherit tokenFile;
112 enable = true;
113 role = "agent";
114 serverAddr = "https://192.168.1.3:6443";
115 extraFlags = lib.concatStringsSep " " [
116 "--pause-image" "test.local/pause:local"
117 "--node-ip" "192.168.1.2"
118 ];
119 };
120 networking.firewall.allowedTCPPorts = [ 6443 ];
121 networking.firewall.allowedUDPPorts = [ 8472 ];
122 networking.firewall.trustedInterfaces = [ "flannel.1" ];
123 networking.useDHCP = false;
124 networking.defaultGateway = "192.168.1.2";
125 networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
126 { address = "192.168.1.2"; prefixLength = 24; }
127 ];
128 };
129 };
130
131 meta = with pkgs.lib.maintainers; {
132 maintainers = [ euank ];
133 };
134
135 testScript = ''
136 machines = [server, server2, agent]
137 for m in machines:
138 m.start()
139 m.wait_for_unit("k3s")
140
141 is_aarch64 = "${toString pkgs.stdenv.isAarch64}" == "1"
142
143 # wait for the agent to show up
144 server.wait_until_succeeds("k3s kubectl get node agent")
145
146 for m in machines:
147 # Fix-Me: Tests fail for 'aarch64-linux' as: "CONFIG_CGROUP_FREEZER: missing (fail)"
148 if not is_aarch64:
149 m.succeed("k3s check-config")
150 m.succeed(
151 "${pauseImage} | k3s ctr image import -"
152 )
153
154 server.succeed("k3s kubectl cluster-info")
155 # Also wait for our service account to show up; it takes a sec
156 server.wait_until_succeeds("k3s kubectl get serviceaccount default")
157
158 # Now create a pod on each node via a daemonset and verify they can talk to each other.
159 server.succeed("k3s kubectl apply -f ${networkTestDaemonset}")
160 server.wait_until_succeeds(f'[ "$(k3s kubectl get ds test -o json | jq .status.numberReady)" -eq {len(machines)} ]')
161
162 # Get pod IPs
163 pods = server.succeed("k3s kubectl get po -o json | jq '.items[].metadata.name' -r").splitlines()
164 pod_ips = [server.succeed(f"k3s kubectl get po {name} -o json | jq '.status.podIP' -cr").strip() for name in pods]
165
166 # Verify each server can ping each pod ip
167 for pod_ip in pod_ips:
168 server.succeed(f"ping -c 1 {pod_ip}")
169 agent.succeed(f"ping -c 1 {pod_ip}")
170
171 # Verify the pods can talk to each other
172 resp = server.wait_until_succeeds(f"k3s kubectl exec {pods[0]} -- socat TCP:{pod_ips[1]}:8000 -")
173 assert resp.strip() == "server"
174 resp = server.wait_until_succeeds(f"k3s kubectl exec {pods[1]} -- socat TCP:{pod_ips[0]}:8000 -")
175 assert resp.strip() == "server"
176
177 # Cleanup
178 server.succeed("k3s kubectl delete -f ${networkTestDaemonset}")
179
180 for m in machines:
181 m.shutdown()
182 '';
183 })