pyrox.dev
1{
2 system ? builtins.currentSystem,
3 config ? { },
4 pkgs ? import ../../.. { inherit system config; },
5}:
6
7with import ../../lib/testing-python.nix { inherit system pkgs; };
8with pkgs.lib;
9
10let
11 mkKubernetesBaseTest =
12 {
13 name,
14 domain ? "my.zyx",
15 test,
16 machines,
17 extraConfiguration ? null,
18 }:
19 let
20 masterName = head (
21 filter (machineName: any (role: role == "master") machines.${machineName}.roles) (
22 attrNames machines
23 )
24 );
25 master = machines.${masterName};
26 extraHosts = ''
27 ${master.ip} etcd.${domain}
28 ${master.ip} api.${domain}
29 ${concatMapStringsSep "\n" (
30 machineName: "${machines.${machineName}.ip} ${machineName}.${domain}"
31 ) (attrNames machines)}
32 '';
33 wrapKubectl =
34 with pkgs;
35 runCommand "wrap-kubectl" { nativeBuildInputs = [ makeWrapper ]; } ''
36 mkdir -p $out/bin
37 makeWrapper ${pkgs.kubernetes}/bin/kubectl $out/bin/kubectl --set KUBECONFIG "/etc/kubernetes/cluster-admin.kubeconfig"
38 '';
39 in
40 makeTest {
41 inherit name;
42
43 nodes = mapAttrs (
44 machineName: machine:
45 {
46 config,
47 pkgs,
48 lib,
49 nodes,
50 ...
51 }:
52 mkMerge [
53 {
54 boot.postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*";
55 virtualisation.memorySize = mkDefault 1536;
56 virtualisation.diskSize = mkDefault 4096;
57 networking = {
58 inherit domain extraHosts;
59 primaryIPAddress = mkForce machine.ip;
60
61 firewall = {
62 allowedTCPPorts = [
63 10250 # kubelet
64 ];
65 trustedInterfaces = [ "mynet" ];
66
67 extraCommands = concatMapStrings (node: ''
68 iptables -A INPUT -s ${node.networking.primaryIPAddress} -j ACCEPT
69 '') (attrValues nodes);
70 };
71 };
72 programs.bash.completion.enable = true;
73 environment.systemPackages = [ wrapKubectl ];
74 services.flannel.iface = "eth1";
75 services.kubernetes = {
76 proxy.hostname = "${masterName}.${domain}";
77
78 easyCerts = true;
79 inherit (machine) roles;
80 apiserver = {
81 securePort = 443;
82 advertiseAddress = master.ip;
83 };
84 # NOTE: what featureGates are useful for testing might change in
85 # the future, see link below to find new ones
86 # https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/
87 featureGates = {
88 AnonymousAuthConfigurableEndpoints = true;
89 ConsistentListFromCache = false;
90 };
91 masterAddress = "${masterName}.${config.networking.domain}";
92 };
93 }
94 (optionalAttrs (any (role: role == "master") machine.roles) {
95 networking.firewall.allowedTCPPorts = [
96 443 # kubernetes apiserver
97 ];
98 })
99 (optionalAttrs (machine ? extraConfiguration) (
100 machine.extraConfiguration {
101 inherit
102 config
103 pkgs
104 lib
105 nodes
106 ;
107 }
108 ))
109 (optionalAttrs (extraConfiguration != null) (extraConfiguration {
110 inherit
111 config
112 pkgs
113 lib
114 nodes
115 ;
116 }))
117 ]
118 ) machines;
119
120 testScript =
121 ''
122 start_all()
123 ''
124 + test;
125 };
126
127 mkKubernetesMultiNodeTest =
128 attrs:
129 mkKubernetesBaseTest (
130 {
131 machines = {
132 machine1 = {
133 roles = [ "master" ];
134 ip = "192.168.1.1";
135 };
136 machine2 = {
137 roles = [ "node" ];
138 ip = "192.168.1.2";
139 };
140 };
141 }
142 // attrs
143 // {
144 name = "kubernetes-${attrs.name}-multinode";
145 }
146 );
147
148 mkKubernetesSingleNodeTest =
149 attrs:
150 mkKubernetesBaseTest (
151 {
152 machines = {
153 machine1 = {
154 roles = [
155 "master"
156 "node"
157 ];
158 ip = "192.168.1.1";
159 };
160 };
161 }
162 // attrs
163 // {
164 name = "kubernetes-${attrs.name}-singlenode";
165 }
166 );
167in
168{
169 inherit mkKubernetesBaseTest mkKubernetesSingleNodeTest mkKubernetesMultiNodeTest;
170}