pyrox.dev
1# This test runs two node kubernetes cluster and checks if simple redis pod works
2
3import ./make-test.nix ({ pkgs, ...} : rec {
4 name = "kubernetes";
5 meta = with pkgs.stdenv.lib.maintainers; {
6 maintainers = [ offline ];
7 };
8
9 redisMaster = builtins.toFile "redis-master-pod.yaml" ''
10 id: redis-master-pod
11 kind: Pod
12 apiVersion: v1beta1
13 desiredState:
14 manifest:
15 version: v1beta1
16 id: redis-master-pod
17 containers:
18 - name: master
19 image: master:5000/nix
20 cpu: 100
21 ports:
22 - name: redis-server
23 containerPort: 6379
24 hostPort: 6379
25 volumeMounts:
26 - name: nix-store
27 mountPath: /nix/store
28 readOnly: true
29 volumeMounts:
30 - name: system-profile
31 mountPath: /bin
32 readOnly: true
33 command:
34 - /bin/redis-server
35 volumes:
36 - name: nix-store
37 source:
38 hostDir:
39 path: /nix/store
40 - name: system-profile
41 source:
42 hostDir:
43 path: /run/current-system/sw/bin
44 labels:
45 name: redis
46 role: master
47 '';
48
49 nodes = {
50 master =
51 { config, pkgs, lib, nodes, ... }:
52 {
53 virtualisation.memorySize = 768;
54 services.kubernetes = {
55 roles = ["master" "node"];
56 dockerCfg = ''{"master:5000":{}}'';
57 controllerManager.machines = ["master" "node"];
58 apiserver.address = "0.0.0.0";
59 verbose = true;
60 };
61 virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false -b cbr0 --insecure-registry master:5000";
62
63 services.etcd = {
64 listenPeerUrls = ["http://0.0.0.0:7001"];
65 initialAdvertisePeerUrls = ["http://master:7001"];
66 initialCluster = ["master=http://master:7001" "node=http://node:7001"];
67 };
68 services.dockerRegistry.enable = true;
69 services.dockerRegistry.host = "0.0.0.0";
70 services.dockerRegistry.port = 5000;
71
72 virtualisation.vlans = [ 1 2 ];
73 networking.bridges = {
74 cbr0.interfaces = [ "eth2" ];
75 };
76 networking.interfaces = {
77 cbr0 = {
78 ipAddress = "10.10.0.1";
79 prefixLength = 24;
80 };
81 eth2.ip4 = lib.mkOverride 0 [ ];
82 };
83 networking.localCommands = ''
84 ip route add 10.10.0.0/16 dev cbr0
85 ip route flush cache
86 '';
87 networking.extraHosts = "127.0.0.1 master";
88
89 networking.firewall.enable = false;
90 #networking.firewall.allowedTCPPorts = [ 4001 7001 ];
91
92 environment.systemPackages = [ pkgs.redis ];
93 };
94
95 node =
96 { config, pkgs, lib, nodes, ... }:
97 {
98 services.kubernetes = {
99 roles = ["node"];
100 dockerCfg = ''{"master:5000":{}}'';
101 kubelet.apiServers = ["master:8080"];
102 verbose = true;
103 };
104 virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false -b cbr0 --insecure-registry master:5000";
105 services.etcd = {
106 listenPeerUrls = ["http://0.0.0.0:7001"];
107 initialAdvertisePeerUrls = ["http://node:7001"];
108 initialCluster = ["master=http://master:7001" "node=http://node:7001"];
109 };
110
111 virtualisation.vlans = [ 1 2 ];
112 networking.bridges = {
113 cbr0.interfaces = [ "eth2" ];
114 };
115 networking.interfaces = {
116 cbr0 = {
117 ipAddress = "10.10.1.1";
118 prefixLength = 24;
119 };
120 eth2.ip4 = lib.mkOverride 0 [ ];
121 };
122 networking.localCommands = ''
123 ip route add 10.10.0.0/16 dev cbr0
124 ip route flush cache
125 '';
126 networking.extraHosts = "127.0.0.1 node";
127
128 networking.firewall.enable = false;
129 #networking.firewall.allowedTCPPorts = [ 4001 7001 ];
130
131 environment.systemPackages = [ pkgs.redis ];
132 };
133
134 client =
135 { config, pkgs, nodes, ... }:
136 {
137 virtualisation.docker.enable = true;
138 virtualisation.docker.extraOptions = "--insecure-registry master:5000";
139 environment.systemPackages = [ pkgs.kubernetes ];
140 environment.etc."test/redis-master-pod.yaml".source = redisMaster;
141 environment.etc."test/pause".source = "${pkgs.kubernetes}/bin/kube-pause";
142 environment.etc."test/Dockerfile".source = pkgs.writeText "Dockerfile" ''
143 FROM scratch
144 ADD pause /
145 ENTRYPOINT ["/pause"]
146 '';
147 };
148 };
149
150 testScript = ''
151 startAll;
152
153 $master->waitForUnit("kubernetes-apiserver.service");
154 $master->waitForUnit("kubernetes-scheduler.service");
155 $master->waitForUnit("kubernetes-controller-manager.service");
156 $master->waitForUnit("kubernetes-kubelet.service");
157 $master->waitForUnit("kubernetes-proxy.service");
158
159 $node->waitForUnit("kubernetes-kubelet.service");
160 $node->waitForUnit("kubernetes-proxy.service");
161
162 $master->waitUntilSucceeds("kubectl get minions | grep master");
163 $master->waitUntilSucceeds("kubectl get minions | grep node");
164
165 $client->waitForUnit("docker.service");
166 $client->succeed("tar cv --files-from /dev/null | docker import - nix");
167 $client->succeed("docker tag nix master:5000/nix");
168 $master->waitForUnit("docker-registry.service");
169 $client->succeed("docker push master:5000/nix");
170 $client->succeed("mkdir -p /root/pause");
171 $client->succeed("cp /etc/test/pause /root/pause/");
172 $client->succeed("cp /etc/test/Dockerfile /root/pause/");
173 $client->succeed("cd /root/pause && docker build -t master:5000/pause .");
174 $client->succeed("docker push master:5000/pause");
175
176 subtest "simple pod", sub {
177 $client->succeed("kubectl create -f ${redisMaster} -s http://master:8080");
178 $client->waitUntilSucceeds("kubectl get pods -s http://master:8080 | grep redis-master | grep -i running");
179 }
180
181 '';
182})