pyrox.dev
1{ system ? builtins.currentSystem, pkgs ? import <nixpkgs> { inherit system; } }:
2with import ./base.nix { inherit system; };
3let
4 domain = "my.zyx";
5
6 redisPod = pkgs.writeText "redis-pod.json" (builtins.toJSON {
7 kind = "Pod";
8 apiVersion = "v1";
9 metadata.name = "redis";
10 metadata.labels.name = "redis";
11 spec.containers = [{
12 name = "redis";
13 image = "redis";
14 args = ["--bind" "0.0.0.0"];
15 imagePullPolicy = "Never";
16 ports = [{
17 name = "redis-server";
18 containerPort = 6379;
19 }];
20 }];
21 });
22
23 redisService = pkgs.writeText "redis-service.json" (builtins.toJSON {
24 kind = "Service";
25 apiVersion = "v1";
26 metadata.name = "redis";
27 spec = {
28 ports = [{port = 6379; targetPort = 6379;}];
29 selector = {name = "redis";};
30 };
31 });
32
33 redisImage = pkgs.dockerTools.buildImage {
34 name = "redis";
35 tag = "latest";
36 contents = [ pkgs.redis pkgs.bind.host ];
37 config.Entrypoint = ["/bin/redis-server"];
38 };
39
40 probePod = pkgs.writeText "probe-pod.json" (builtins.toJSON {
41 kind = "Pod";
42 apiVersion = "v1";
43 metadata.name = "probe";
44 metadata.labels.name = "probe";
45 spec.containers = [{
46 name = "probe";
47 image = "probe";
48 args = [ "-f" ];
49 tty = true;
50 imagePullPolicy = "Never";
51 }];
52 });
53
54 probeImage = pkgs.dockerTools.buildImage {
55 name = "probe";
56 tag = "latest";
57 contents = [ pkgs.bind.host pkgs.busybox ];
58 config.Entrypoint = ["/bin/tail"];
59 };
60
61 extraConfiguration = { config, pkgs, lib, ... }: {
62 environment.systemPackages = [ pkgs.bind.host ];
63 services.dnsmasq.enable = true;
64 services.dnsmasq.servers = [
65 "/cluster.local/${config.services.kubernetes.addons.dns.clusterIp}#53"
66 ];
67 };
68
69 base = {
70 name = "dns";
71 inherit domain extraConfiguration;
72 };
73
74 singleNodeTest = {
75 test = ''
76 # prepare machine1 for test
77 machine1.wait_until_succeeds("kubectl get node machine1.${domain} | grep -w Ready")
78 machine1.wait_until_succeeds(
79 "${pkgs.gzip}/bin/zcat ${redisImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
80 )
81 machine1.wait_until_succeeds(
82 "kubectl create -f ${redisPod}"
83 )
84 machine1.wait_until_succeeds(
85 "kubectl create -f ${redisService}"
86 )
87 machine1.wait_until_succeeds(
88 "${pkgs.gzip}/bin/zcat ${probeImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
89 )
90 machine1.wait_until_succeeds(
91 "kubectl create -f ${probePod}"
92 )
93
94 # check if pods are running
95 machine1.wait_until_succeeds("kubectl get pod redis | grep Running")
96 machine1.wait_until_succeeds("kubectl get pod probe | grep Running")
97 machine1.wait_until_succeeds("kubectl get pods -n kube-system | grep 'coredns.*1/1'")
98
99 # check dns on host (dnsmasq)
100 machine1.succeed("host redis.default.svc.cluster.local")
101
102 # check dns inside the container
103 machine1.succeed("kubectl exec -ti probe -- /bin/host redis.default.svc.cluster.local")
104 '';
105 };
106
107 multiNodeTest = {
108 test = ''
109 # Node token exchange
110 machine1.wait_until_succeeds(
111 "cp -f /var/lib/cfssl/apitoken.secret /tmp/shared/apitoken.secret"
112 )
113 machine2.wait_until_succeeds(
114 "cat /tmp/shared/apitoken.secret | nixos-kubernetes-node-join"
115 )
116
117 # prepare machines for test
118 machine1.wait_until_succeeds("kubectl get node machine2.${domain} | grep -w Ready")
119 machine2.wait_until_succeeds(
120 "${pkgs.gzip}/bin/zcat ${redisImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
121 )
122 machine1.wait_until_succeeds(
123 "kubectl create -f ${redisPod}"
124 )
125 machine1.wait_until_succeeds(
126 "kubectl create -f ${redisService}"
127 )
128 machine2.wait_until_succeeds(
129 "${pkgs.gzip}/bin/zcat ${probeImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
130 )
131 machine1.wait_until_succeeds(
132 "kubectl create -f ${probePod}"
133 )
134
135 # check if pods are running
136 machine1.wait_until_succeeds("kubectl get pod redis | grep Running")
137 machine1.wait_until_succeeds("kubectl get pod probe | grep Running")
138 machine1.wait_until_succeeds("kubectl get pods -n kube-system | grep 'coredns.*1/1'")
139
140 # check dns on hosts (dnsmasq)
141 machine1.succeed("host redis.default.svc.cluster.local")
142 machine2.succeed("host redis.default.svc.cluster.local")
143
144 # check dns inside the container
145 machine1.succeed("kubectl exec -ti probe -- /bin/host redis.default.svc.cluster.local")
146 '';
147 };
148in {
149 singlenode = mkKubernetesSingleNodeTest (base // singleNodeTest);
150 multinode = mkKubernetesMultiNodeTest (base // multiNodeTest);
151}