pyrox.dev
1{ system ? builtins.currentSystem, pkgs ? import ../../.. { inherit system; } }:
2with import ./base.nix { inherit system; };
3let
4 domain = "my.zyx";
5
6 redisPod = pkgs.writeText "redis-pod.json" (builtins.toJSON {
7 kind = "Pod";
8 apiVersion = "v1";
9 metadata.name = "redis";
10 metadata.labels.name = "redis";
11 spec.containers = [{
12 name = "redis";
13 image = "redis";
14 args = ["--bind" "0.0.0.0"];
15 imagePullPolicy = "Never";
16 ports = [{
17 name = "redis-server";
18 containerPort = 6379;
19 }];
20 }];
21 });
22
23 redisService = pkgs.writeText "redis-service.json" (builtins.toJSON {
24 kind = "Service";
25 apiVersion = "v1";
26 metadata.name = "redis";
27 spec = {
28 ports = [{port = 6379; targetPort = 6379;}];
29 selector = {name = "redis";};
30 };
31 });
32
33 redisImage = pkgs.dockerTools.buildImage {
34 name = "redis";
35 tag = "latest";
36 copyToRoot = pkgs.buildEnv {
37 name = "image-root";
38 pathsToLink = [ "/bin" ];
39 paths = [ pkgs.redis pkgs.bind.host ];
40 };
41 config.Entrypoint = ["/bin/redis-server"];
42 };
43
44 probePod = pkgs.writeText "probe-pod.json" (builtins.toJSON {
45 kind = "Pod";
46 apiVersion = "v1";
47 metadata.name = "probe";
48 metadata.labels.name = "probe";
49 spec.containers = [{
50 name = "probe";
51 image = "probe";
52 args = [ "-f" ];
53 tty = true;
54 imagePullPolicy = "Never";
55 }];
56 });
57
58 probeImage = pkgs.dockerTools.buildImage {
59 name = "probe";
60 tag = "latest";
61 copyToRoot = pkgs.buildEnv {
62 name = "image-root";
63 pathsToLink = [ "/bin" ];
64 paths = [ pkgs.bind.host pkgs.busybox ];
65 };
66 config.Entrypoint = ["/bin/tail"];
67 };
68
69 extraConfiguration = { config, pkgs, lib, ... }: {
70 environment.systemPackages = [ pkgs.bind.host ];
71 services.dnsmasq.enable = true;
72 services.dnsmasq.servers = [
73 "/cluster.local/${config.services.kubernetes.addons.dns.clusterIp}#53"
74 ];
75 };
76
77 base = {
78 name = "dns";
79 inherit domain extraConfiguration;
80 };
81
82 singleNodeTest = {
83 test = ''
84 # prepare machine1 for test
85 machine1.wait_until_succeeds("kubectl get node machine1.${domain} | grep -w Ready")
86 machine1.wait_until_succeeds(
87 "${pkgs.gzip}/bin/zcat ${redisImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
88 )
89 machine1.wait_until_succeeds(
90 "kubectl create -f ${redisPod}"
91 )
92 machine1.wait_until_succeeds(
93 "kubectl create -f ${redisService}"
94 )
95 machine1.wait_until_succeeds(
96 "${pkgs.gzip}/bin/zcat ${probeImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
97 )
98 machine1.wait_until_succeeds(
99 "kubectl create -f ${probePod}"
100 )
101
102 # check if pods are running
103 machine1.wait_until_succeeds("kubectl get pod redis | grep Running")
104 machine1.wait_until_succeeds("kubectl get pod probe | grep Running")
105 machine1.wait_until_succeeds("kubectl get pods -n kube-system | grep 'coredns.*1/1'")
106
107 # check dns on host (dnsmasq)
108 machine1.succeed("host redis.default.svc.cluster.local")
109
110 # check dns inside the container
111 machine1.succeed("kubectl exec probe -- /bin/host redis.default.svc.cluster.local")
112 '';
113 };
114
115 multiNodeTest = {
116 test = ''
117 # Node token exchange
118 machine1.wait_until_succeeds(
119 "cp -f /var/lib/cfssl/apitoken.secret /tmp/shared/apitoken.secret"
120 )
121 machine2.wait_until_succeeds(
122 "cat /tmp/shared/apitoken.secret | nixos-kubernetes-node-join"
123 )
124
125 # prepare machines for test
126 machine1.wait_until_succeeds("kubectl get node machine2.${domain} | grep -w Ready")
127 machine2.wait_until_succeeds(
128 "${pkgs.gzip}/bin/zcat ${redisImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
129 )
130 machine1.wait_until_succeeds(
131 "kubectl create -f ${redisPod}"
132 )
133 machine1.wait_until_succeeds(
134 "kubectl create -f ${redisService}"
135 )
136 machine2.wait_until_succeeds(
137 "${pkgs.gzip}/bin/zcat ${probeImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
138 )
139 machine1.wait_until_succeeds(
140 "kubectl create -f ${probePod}"
141 )
142
143 # check if pods are running
144 machine1.wait_until_succeeds("kubectl get pod redis | grep Running")
145 machine1.wait_until_succeeds("kubectl get pod probe | grep Running")
146 machine1.wait_until_succeeds("kubectl get pods -n kube-system | grep 'coredns.*1/1'")
147
148 # check dns on hosts (dnsmasq)
149 machine1.succeed("host redis.default.svc.cluster.local")
150 machine2.succeed("host redis.default.svc.cluster.local")
151
152 # check dns inside the container
153 machine1.succeed("kubectl exec probe -- /bin/host redis.default.svc.cluster.local")
154 '';
155 };
156in {
157 singlenode = mkKubernetesSingleNodeTest (base // singleNodeTest);
158 multinode = mkKubernetesMultiNodeTest (base // multiNodeTest);
159}