pyrox.dev
1{
2 system ? builtins.currentSystem,
3 pkgs ? import ../../.. { inherit system; },
4}:
5with import ./base.nix { inherit system; };
6let
7 domain = "my.zyx";
8
9 redisPod = pkgs.writeText "redis-pod.json" (
10 builtins.toJSON {
11 kind = "Pod";
12 apiVersion = "v1";
13 metadata.name = "redis";
14 metadata.labels.name = "redis";
15 spec.containers = [
16 {
17 name = "redis";
18 image = "redis";
19 args = [
20 "--bind"
21 "0.0.0.0"
22 ];
23 imagePullPolicy = "Never";
24 ports = [
25 {
26 name = "redis-server";
27 containerPort = 6379;
28 }
29 ];
30 }
31 ];
32 }
33 );
34
35 redisService = pkgs.writeText "redis-service.json" (
36 builtins.toJSON {
37 kind = "Service";
38 apiVersion = "v1";
39 metadata.name = "redis";
40 spec = {
41 ports = [
42 {
43 port = 6379;
44 targetPort = 6379;
45 }
46 ];
47 selector = {
48 name = "redis";
49 };
50 };
51 }
52 );
53
54 redisImage = pkgs.dockerTools.buildImage {
55 name = "redis";
56 tag = "latest";
57 copyToRoot = pkgs.buildEnv {
58 name = "image-root";
59 pathsToLink = [ "/bin" ];
60 paths = [
61 pkgs.redis
62 pkgs.bind.host
63 ];
64 };
65 config.Entrypoint = [ "/bin/redis-server" ];
66 };
67
68 probePod = pkgs.writeText "probe-pod.json" (
69 builtins.toJSON {
70 kind = "Pod";
71 apiVersion = "v1";
72 metadata.name = "probe";
73 metadata.labels.name = "probe";
74 spec.containers = [
75 {
76 name = "probe";
77 image = "probe";
78 args = [ "-f" ];
79 tty = true;
80 imagePullPolicy = "Never";
81 }
82 ];
83 }
84 );
85
86 probeImage = pkgs.dockerTools.buildImage {
87 name = "probe";
88 tag = "latest";
89 copyToRoot = pkgs.buildEnv {
90 name = "image-root";
91 pathsToLink = [ "/bin" ];
92 paths = [
93 pkgs.bind.host
94 pkgs.busybox
95 ];
96 };
97 config.Entrypoint = [ "/bin/tail" ];
98 };
99
100 extraConfiguration =
101 {
102 config,
103 pkgs,
104 lib,
105 ...
106 }:
107 {
108 environment.systemPackages = [ pkgs.bind.host ];
109 services.dnsmasq.enable = true;
110 services.dnsmasq.settings.server = [
111 "/cluster.local/${config.services.kubernetes.addons.dns.clusterIp}#53"
112 ];
113 };
114
115 base = {
116 name = "dns";
117 inherit domain extraConfiguration;
118 };
119
120 singleNodeTest = {
121 test = ''
122 # prepare machine1 for test
123 machine1.wait_until_succeeds("kubectl get node machine1.${domain} | grep -w Ready")
124 machine1.wait_until_succeeds(
125 "${pkgs.gzip}/bin/zcat ${redisImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
126 )
127 machine1.wait_until_succeeds(
128 "kubectl create -f ${redisPod}"
129 )
130 machine1.wait_until_succeeds(
131 "kubectl create -f ${redisService}"
132 )
133 machine1.wait_until_succeeds(
134 "${pkgs.gzip}/bin/zcat ${probeImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
135 )
136 machine1.wait_until_succeeds(
137 "kubectl create -f ${probePod}"
138 )
139
140 # check if pods are running
141 machine1.wait_until_succeeds("kubectl get pod redis | grep Running")
142 machine1.wait_until_succeeds("kubectl get pod probe | grep Running")
143 machine1.wait_until_succeeds("kubectl get pods -n kube-system | grep 'coredns.*1/1'")
144
145 # check dns on host (dnsmasq)
146 machine1.succeed("host redis.default.svc.cluster.local")
147
148 # check dns inside the container
149 machine1.succeed("kubectl exec probe -- /bin/host redis.default.svc.cluster.local")
150 '';
151 };
152
153 multiNodeTest = {
154 test = ''
155 # Node token exchange
156 machine1.wait_until_succeeds(
157 "cp -f /var/lib/cfssl/apitoken.secret /tmp/shared/apitoken.secret"
158 )
159 machine2.wait_until_succeeds(
160 "cat /tmp/shared/apitoken.secret | nixos-kubernetes-node-join"
161 )
162
163 # prepare machines for test
164 machine1.wait_until_succeeds("kubectl get node machine2.${domain} | grep -w Ready")
165 machine2.wait_until_succeeds(
166 "${pkgs.gzip}/bin/zcat ${redisImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
167 )
168 machine1.wait_until_succeeds(
169 "kubectl create -f ${redisPod}"
170 )
171 machine1.wait_until_succeeds(
172 "kubectl create -f ${redisService}"
173 )
174 machine2.wait_until_succeeds(
175 "${pkgs.gzip}/bin/zcat ${probeImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
176 )
177 machine1.wait_until_succeeds(
178 "kubectl create -f ${probePod}"
179 )
180
181 # check if pods are running
182 machine1.wait_until_succeeds("kubectl get pod redis | grep Running")
183 machine1.wait_until_succeeds("kubectl get pod probe | grep Running")
184 machine1.wait_until_succeeds("kubectl get pods -n kube-system | grep 'coredns.*1/1'")
185
186 # check dns on hosts (dnsmasq)
187 machine1.succeed("host redis.default.svc.cluster.local")
188 machine2.succeed("host redis.default.svc.cluster.local")
189
190 # check dns inside the container
191 machine1.succeed("kubectl exec probe -- /bin/host redis.default.svc.cluster.local")
192 '';
193 };
194in
195{
196 singlenode = mkKubernetesSingleNodeTest (base // singleNodeTest);
197 multinode = mkKubernetesMultiNodeTest (base // multiNodeTest);
198}