pyrox.dev
1# Tests K3s with Etcd backend
2import ../make-test-python.nix (
3 {
4 pkgs,
5 lib,
6 k3s,
7 etcd,
8 ...
9 }:
10
11 {
12 name = "${k3s.name}-etcd";
13
14 nodes = {
15
16 etcd =
17 { ... }:
18 {
19 services.etcd = {
20 enable = true;
21 openFirewall = true;
22 listenClientUrls = [
23 "http://192.168.1.1:2379"
24 "http://127.0.0.1:2379"
25 ];
26 listenPeerUrls = [ "http://192.168.1.1:2380" ];
27 initialAdvertisePeerUrls = [ "http://192.168.1.1:2380" ];
28 initialCluster = [ "etcd=http://192.168.1.1:2380" ];
29 };
30 networking = {
31 useDHCP = false;
32 defaultGateway = "192.168.1.1";
33 interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
34 {
35 address = "192.168.1.1";
36 prefixLength = 24;
37 }
38 ];
39 };
40 };
41
42 k3s =
43 { pkgs, ... }:
44 {
45 environment.systemPackages = with pkgs; [ jq ];
46 # k3s uses enough resources the default vm fails.
47 virtualisation.memorySize = 1536;
48 virtualisation.diskSize = 4096;
49
50 services.k3s = {
51 enable = true;
52 role = "server";
53 package = k3s;
54 extraFlags = [
55 "--datastore-endpoint=\"http://192.168.1.1:2379\""
56 "--disable coredns"
57 "--disable local-storage"
58 "--disable metrics-server"
59 "--disable servicelb"
60 "--disable traefik"
61 "--node-ip 192.168.1.2"
62 ];
63 };
64
65 networking = {
66 firewall = {
67 allowedTCPPorts = [
68 2379
69 2380
70 6443
71 ];
72 allowedUDPPorts = [ 8472 ];
73 };
74 useDHCP = false;
75 defaultGateway = "192.168.1.2";
76 interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
77 {
78 address = "192.168.1.2";
79 prefixLength = 24;
80 }
81 ];
82 };
83 };
84 };
85
86 testScript = # python
87 ''
88 with subtest("should start etcd"):
89 etcd.start()
90 etcd.wait_for_unit("etcd.service")
91
92 with subtest("should wait for etcdctl endpoint status to succeed"):
93 etcd.wait_until_succeeds("etcdctl endpoint status")
94
95 with subtest("should wait for etcdctl endpoint health to succeed"):
96 etcd.wait_until_succeeds("etcdctl endpoint health")
97
98 with subtest("should start k3s"):
99 k3s.start()
100 k3s.wait_for_unit("k3s")
101
102 with subtest("should test if kubectl works"):
103 k3s.wait_until_succeeds("k3s kubectl get node")
104
105 with subtest("should wait for service account to show up; takes a sec"):
106 k3s.wait_until_succeeds("k3s kubectl get serviceaccount default")
107
108 with subtest("should create a sample secret object"):
109 k3s.succeed("k3s kubectl create secret generic nixossecret --from-literal thesecret=abacadabra")
110
111 with subtest("should check if secret is correct"):
112 k3s.wait_until_succeeds("[[ $(kubectl get secrets nixossecret -o json | jq -r .data.thesecret | base64 -d) == abacadabra ]]")
113
114 with subtest("should have a secret in database"):
115 etcd.wait_until_succeeds("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]")
116
117 with subtest("should delete the secret"):
118 k3s.succeed("k3s kubectl delete secret nixossecret")
119
120 with subtest("should not have a secret in database"):
121 etcd.wait_until_fails("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]")
122 '';
123
124 meta.maintainers = etcd.meta.maintainers ++ lib.teams.k3s.members;
125 }
126)