pyrox.dev
1{ system ? builtins.currentSystem, pkgs ? import <nixpkgs> { inherit system; } }:
2with import ./base.nix { inherit system; };
3let
4 domain = "my.zyx";
5
6 certs = import ./certs.nix { externalDomain = domain; kubelets = [ "machine1" "machine2" ]; };
7
8 redisPod = pkgs.writeText "redis-pod.json" (builtins.toJSON {
9 kind = "Pod";
10 apiVersion = "v1";
11 metadata.name = "redis";
12 metadata.labels.name = "redis";
13 spec.containers = [{
14 name = "redis";
15 image = "redis";
16 args = ["--bind" "0.0.0.0"];
17 imagePullPolicy = "Never";
18 ports = [{
19 name = "redis-server";
20 containerPort = 6379;
21 }];
22 }];
23 });
24
25 redisService = pkgs.writeText "redis-service.json" (builtins.toJSON {
26 kind = "Service";
27 apiVersion = "v1";
28 metadata.name = "redis";
29 spec = {
30 ports = [{port = 6379; targetPort = 6379;}];
31 selector = {name = "redis";};
32 };
33 });
34
35 redisImage = pkgs.dockerTools.buildImage {
36 name = "redis";
37 tag = "latest";
38 contents = [ pkgs.redis pkgs.bind.host ];
39 config.Entrypoint = "/bin/redis-server";
40 };
41
42 probePod = pkgs.writeText "probe-pod.json" (builtins.toJSON {
43 kind = "Pod";
44 apiVersion = "v1";
45 metadata.name = "probe";
46 metadata.labels.name = "probe";
47 spec.containers = [{
48 name = "probe";
49 image = "probe";
50 args = [ "-f" ];
51 tty = true;
52 imagePullPolicy = "Never";
53 }];
54 });
55
56 probeImage = pkgs.dockerTools.buildImage {
57 name = "probe";
58 tag = "latest";
59 contents = [ pkgs.bind.host pkgs.busybox ];
60 config.Entrypoint = "/bin/tail";
61 };
62
63 extraConfiguration = { config, pkgs, lib, nodes, ... }: {
64 environment.systemPackages = [ pkgs.bind.host ];
65 # virtualisation.docker.extraOptions = "--dns=${config.services.kubernetes.addons.dns.clusterIp}";
66 services.dnsmasq.enable = true;
67 services.dnsmasq.servers = [
68 "/cluster.local/${config.services.kubernetes.addons.dns.clusterIp}#53"
69 ];
70 };
71
72 base = {
73 name = "dns";
74 inherit domain certs extraConfiguration;
75 };
76
77 singleNodeTest = {
78 test = ''
79 # prepare machine1 for test
80 $machine1->waitUntilSucceeds("kubectl get node machine1.${domain} | grep -w Ready");
81 $machine1->execute("docker load < ${redisImage}");
82 $machine1->waitUntilSucceeds("kubectl create -f ${redisPod}");
83 $machine1->waitUntilSucceeds("kubectl create -f ${redisService}");
84 $machine1->execute("docker load < ${probeImage}");
85 $machine1->waitUntilSucceeds("kubectl create -f ${probePod}");
86
87 # check if pods are running
88 $machine1->waitUntilSucceeds("kubectl get pod redis | grep Running");
89 $machine1->waitUntilSucceeds("kubectl get pod probe | grep Running");
90 $machine1->waitUntilSucceeds("kubectl get pods -n kube-system | grep 'kube-dns.*3/3'");
91
92 # check dns on host (dnsmasq)
93 $machine1->succeed("host redis.default.svc.cluster.local");
94
95 # check dns inside the container
96 $machine1->succeed("kubectl exec -ti probe -- /bin/host redis.default.svc.cluster.local");
97 '';
98 };
99
100 multiNodeTest = {
101 test = ''
102 # prepare machines for test
103 $machine1->waitUntilSucceeds("kubectl get node machine1.${domain} | grep -w Ready");
104 $machine1->waitUntilSucceeds("kubectl get node machine2.${domain} | grep -w Ready");
105 $machine2->execute("docker load < ${redisImage}");
106 $machine1->waitUntilSucceeds("kubectl create -f ${redisPod}");
107 $machine1->waitUntilSucceeds("kubectl create -f ${redisService}");
108 $machine2->execute("docker load < ${probeImage}");
109 $machine1->waitUntilSucceeds("kubectl create -f ${probePod}");
110
111 # check if pods are running
112 $machine1->waitUntilSucceeds("kubectl get pod redis | grep Running");
113 $machine1->waitUntilSucceeds("kubectl get pod probe | grep Running");
114 $machine1->waitUntilSucceeds("kubectl get pods -n kube-system | grep 'kube-dns.*3/3'");
115
116 # check dns on hosts (dnsmasq)
117 $machine1->succeed("host redis.default.svc.cluster.local");
118 $machine2->succeed("host redis.default.svc.cluster.local");
119
120 # check dns inside the container
121 $machine1->succeed("kubectl exec -ti probe -- /bin/host redis.default.svc.cluster.local");
122 '';
123 };
124in {
125 singlenode = mkKubernetesSingleNodeTest (base // singleNodeTest);
126 multinode = mkKubernetesMultiNodeTest (base // multiNodeTest);
127}