pyrox.dev
1# Tests K3s with Etcd backend
2import ../make-test-python.nix (
3 {
4 pkgs,
5 lib,
6 k3s,
7 etcd,
8 ...
9 }:
10
11 {
12 name = "${k3s.name}-etcd";
13
14 nodes = {
15
16 etcd =
17 { ... }:
18 {
19 services.etcd = {
20 enable = true;
21 openFirewall = true;
22 listenClientUrls = [
23 "http://192.168.1.1:2379"
24 "http://127.0.0.1:2379"
25 ];
26 listenPeerUrls = [ "http://192.168.1.1:2380" ];
27 initialAdvertisePeerUrls = [ "http://192.168.1.1:2380" ];
28 initialCluster = [ "etcd=http://192.168.1.1:2380" ];
29 };
30 networking = {
31 useDHCP = false;
32 defaultGateway = "192.168.1.1";
33 interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
34 {
35 address = "192.168.1.1";
36 prefixLength = 24;
37 }
38 ];
39 };
40 };
41
42 k3s =
43 { pkgs, ... }:
44 {
45 environment.systemPackages = with pkgs; [ jq ];
46 # k3s uses enough resources the default vm fails.
47 virtualisation.memorySize = 1536;
48 virtualisation.diskSize = 4096;
49
50 services.k3s = {
51 enable = true;
52 role = "server";
53 extraFlags = [
54 "--datastore-endpoint=\"http://192.168.1.1:2379\""
55 "--disable coredns"
56 "--disable local-storage"
57 "--disable metrics-server"
58 "--disable servicelb"
59 "--disable traefik"
60 "--node-ip 192.168.1.2"
61 ];
62 };
63
64 networking = {
65 firewall = {
66 allowedTCPPorts = [
67 2379
68 2380
69 6443
70 ];
71 allowedUDPPorts = [ 8472 ];
72 };
73 useDHCP = false;
74 defaultGateway = "192.168.1.2";
75 interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
76 {
77 address = "192.168.1.2";
78 prefixLength = 24;
79 }
80 ];
81 };
82 };
83 };
84
85 testScript = ''
86 with subtest("should start etcd"):
87 etcd.start()
88 etcd.wait_for_unit("etcd.service")
89
90 with subtest("should wait for etcdctl endpoint status to succeed"):
91 etcd.wait_until_succeeds("etcdctl endpoint status")
92
93 with subtest("should wait for etcdctl endpoint health to succeed"):
94 etcd.wait_until_succeeds("etcdctl endpoint health")
95
96 with subtest("should start k3s"):
97 k3s.start()
98 k3s.wait_for_unit("k3s")
99
100 with subtest("should test if kubectl works"):
101 k3s.wait_until_succeeds("k3s kubectl get node")
102
103 with subtest("should wait for service account to show up; takes a sec"):
104 k3s.wait_until_succeeds("k3s kubectl get serviceaccount default")
105
106 with subtest("should create a sample secret object"):
107 k3s.succeed("k3s kubectl create secret generic nixossecret --from-literal thesecret=abacadabra")
108
109 with subtest("should check if secret is correct"):
110 k3s.wait_until_succeeds("[[ $(kubectl get secrets nixossecret -o json | jq -r .data.thesecret | base64 -d) == abacadabra ]]")
111
112 with subtest("should have a secret in database"):
113 etcd.wait_until_succeeds("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]")
114
115 with subtest("should delete the secret"):
116 k3s.succeed("k3s kubectl delete secret nixossecret")
117
118 with subtest("should not have a secret in database"):
119 etcd.wait_until_fails("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]")
120
121 with subtest("should shutdown k3s and etcd"):
122 k3s.shutdown()
123 etcd.shutdown()
124 '';
125
126 meta.maintainers = etcd.meta.maintainers ++ lib.teams.k3s.members;
127 }
128)