pyrox.dev
1{ system ? builtins.currentSystem, pkgs ? import ../../.. { inherit system; } }:
2with import ./base.nix { inherit system; };
3let
4
5 roServiceAccount = pkgs.writeText "ro-service-account.json" (builtins.toJSON {
6 kind = "ServiceAccount";
7 apiVersion = "v1";
8 metadata = {
9 name = "read-only";
10 namespace = "default";
11 };
12 });
13
14 roRoleBinding = pkgs.writeText "ro-role-binding.json" (builtins.toJSON {
15 apiVersion = "rbac.authorization.k8s.io/v1";
16 kind = "RoleBinding";
17 metadata = {
18 name = "read-pods";
19 namespace = "default";
20 };
21 roleRef = {
22 apiGroup = "rbac.authorization.k8s.io";
23 kind = "Role";
24 name = "pod-reader";
25 };
26 subjects = [{
27 kind = "ServiceAccount";
28 name = "read-only";
29 namespace = "default";
30 }];
31 });
32
33 roRole = pkgs.writeText "ro-role.json" (builtins.toJSON {
34 apiVersion = "rbac.authorization.k8s.io/v1";
35 kind = "Role";
36 metadata = {
37 name = "pod-reader";
38 namespace = "default";
39 };
40 rules = [{
41 apiGroups = [""];
42 resources = ["pods"];
43 verbs = ["get" "list" "watch"];
44 }];
45 });
46
47 kubectlPod = pkgs.writeText "kubectl-pod.json" (builtins.toJSON {
48 kind = "Pod";
49 apiVersion = "v1";
50 metadata.name = "kubectl";
51 metadata.namespace = "default";
52 metadata.labels.name = "kubectl";
53 spec.serviceAccountName = "read-only";
54 spec.containers = [{
55 name = "kubectl";
56 image = "kubectl:latest";
57 command = ["/bin/tail" "-f"];
58 imagePullPolicy = "Never";
59 tty = true;
60 }];
61 });
62
63 kubectlPod2 = pkgs.writeTextDir "kubectl-pod-2.json" (builtins.toJSON {
64 kind = "Pod";
65 apiVersion = "v1";
66 metadata.name = "kubectl-2";
67 metadata.namespace = "default";
68 metadata.labels.name = "kubectl-2";
69 spec.serviceAccountName = "read-only";
70 spec.containers = [{
71 name = "kubectl-2";
72 image = "kubectl:latest";
73 command = ["/bin/tail" "-f"];
74 imagePullPolicy = "Never";
75 tty = true;
76 }];
77 });
78
79 copyKubectl = pkgs.runCommand "copy-kubectl" { } ''
80 mkdir -p $out/bin
81 cp ${pkgs.kubernetes}/bin/kubectl $out/bin/kubectl
82 '';
83
84 kubectlImage = pkgs.dockerTools.buildImage {
85 name = "kubectl";
86 tag = "latest";
87 copyToRoot = pkgs.buildEnv {
88 name = "image-root";
89 pathsToLink = [ "/bin" ];
90 paths = [ copyKubectl pkgs.busybox kubectlPod2 ];
91 };
92 config.Entrypoint = ["/bin/sh"];
93 };
94
95 base = {
96 name = "rbac";
97 };
98
99 singlenode = base // {
100 test = ''
101 machine1.wait_until_succeeds("kubectl get node machine1.my.zyx | grep -w Ready")
102
103 machine1.wait_until_succeeds(
104 "${pkgs.gzip}/bin/zcat ${kubectlImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
105 )
106
107 machine1.wait_until_succeeds(
108 "kubectl apply -f ${roServiceAccount}"
109 )
110 machine1.wait_until_succeeds(
111 "kubectl apply -f ${roRole}"
112 )
113 machine1.wait_until_succeeds(
114 "kubectl apply -f ${roRoleBinding}"
115 )
116 machine1.wait_until_succeeds(
117 "kubectl create -f ${kubectlPod}"
118 )
119
120 machine1.wait_until_succeeds("kubectl get pod kubectl | grep Running")
121
122 machine1.wait_until_succeeds("kubectl exec kubectl -- kubectl get pods")
123 machine1.fail("kubectl exec kubectl -- kubectl create -f /kubectl-pod-2.json")
124 machine1.fail("kubectl exec kubectl -- kubectl delete pods -l name=kubectl")
125 '';
126 };
127
128 multinode = base // {
129 test = ''
130 # Node token exchange
131 machine1.wait_until_succeeds(
132 "cp -f /var/lib/cfssl/apitoken.secret /tmp/shared/apitoken.secret"
133 )
134 machine2.wait_until_succeeds(
135 "cat /tmp/shared/apitoken.secret | nixos-kubernetes-node-join"
136 )
137
138 machine1.wait_until_succeeds("kubectl get node machine2.my.zyx | grep -w Ready")
139
140 machine2.wait_until_succeeds(
141 "${pkgs.gzip}/bin/zcat ${kubectlImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
142 )
143
144 machine1.wait_until_succeeds(
145 "kubectl apply -f ${roServiceAccount}"
146 )
147 machine1.wait_until_succeeds(
148 "kubectl apply -f ${roRole}"
149 )
150 machine1.wait_until_succeeds(
151 "kubectl apply -f ${roRoleBinding}"
152 )
153 machine1.wait_until_succeeds(
154 "kubectl create -f ${kubectlPod}"
155 )
156
157 machine1.wait_until_succeeds("kubectl get pod kubectl | grep Running")
158
159 machine1.wait_until_succeeds("kubectl exec kubectl -- kubectl get pods")
160 machine1.fail("kubectl exec kubectl -- kubectl create -f /kubectl-pod-2.json")
161 machine1.fail("kubectl exec kubectl -- kubectl delete pods -l name=kubectl")
162 '';
163 };
164
165in {
166 singlenode = mkKubernetesSingleNodeTest singlenode;
167 multinode = mkKubernetesMultiNodeTest multinode;
168}