pyrox.dev
1{
2 system ? builtins.currentSystem,
3 config ? { },
4 pkgs ? import ../../.. { inherit system config; },
5}:
6
7with import ../../lib/testing-python.nix { inherit system pkgs; };
8with pkgs.lib;
9
10let
11 mkKubernetesBaseTest =
12 {
13 name,
14 domain ? "my.zyx",
15 test,
16 machines,
17 extraConfiguration ? null,
18 }:
19 let
20 masterName = head (
21 filter (machineName: any (role: role == "master") machines.${machineName}.roles) (
22 attrNames machines
23 )
24 );
25 master = machines.${masterName};
26 extraHosts = ''
27 ${master.ip} etcd.${domain}
28 ${master.ip} api.${domain}
29 ${concatMapStringsSep "\n" (
30 machineName: "${machines.${machineName}.ip} ${machineName}.${domain}"
31 ) (attrNames machines)}
32 '';
33 wrapKubectl =
34 with pkgs;
35 runCommand "wrap-kubectl" { nativeBuildInputs = [ makeWrapper ]; } ''
36 mkdir -p $out/bin
37 makeWrapper ${pkgs.kubernetes}/bin/kubectl $out/bin/kubectl --set KUBECONFIG "/etc/kubernetes/cluster-admin.kubeconfig"
38 '';
39 in
40 makeTest {
41 inherit name;
42
43 nodes = mapAttrs (
44 machineName: machine:
45 {
46 config,
47 pkgs,
48 lib,
49 nodes,
50 ...
51 }:
52 mkMerge [
53 {
54 boot.postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*";
55 virtualisation.memorySize = mkDefault 1536;
56 virtualisation.diskSize = mkDefault 4096;
57 networking = {
58 inherit domain extraHosts;
59 primaryIPAddress = mkForce machine.ip;
60
61 firewall = {
62 allowedTCPPorts = [
63 10250 # kubelet
64 ];
65 trustedInterfaces = [ "mynet" ];
66
67 extraCommands = concatMapStrings (node: ''
68 iptables -A INPUT -s ${node.networking.primaryIPAddress} -j ACCEPT
69 '') (attrValues nodes);
70 };
71 };
72 programs.bash.completion.enable = true;
73 environment.systemPackages = [ wrapKubectl ];
74 services.flannel.iface = "eth1";
75 services.kubernetes = {
76 proxy.hostname = "${masterName}.${domain}";
77
78 easyCerts = true;
79 inherit (machine) roles;
80 apiserver = {
81 securePort = 443;
82 advertiseAddress = master.ip;
83 };
84 # NOTE: what featureGates are useful for testing might change in
85 # the future, see link below to find new ones
86 # https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/
87 featureGates = {
88 AllowParsingUserUIDFromCertAuth = true;
89 ClusterTrustBundle = false;
90 };
91 masterAddress = "${masterName}.${config.networking.domain}";
92 };
93 }
94 (optionalAttrs (any (role: role == "master") machine.roles) {
95 networking.firewall.allowedTCPPorts = [
96 443 # kubernetes apiserver
97 ];
98 })
99 (optionalAttrs (machine ? extraConfiguration) (
100 machine.extraConfiguration {
101 inherit
102 config
103 pkgs
104 lib
105 nodes
106 ;
107 }
108 ))
109 (optionalAttrs (extraConfiguration != null) (extraConfiguration {
110 inherit
111 config
112 pkgs
113 lib
114 nodes
115 ;
116 }))
117 ]
118 ) machines;
119
120 testScript = ''
121 start_all()
122 ''
123 + test;
124 };
125
126 mkKubernetesMultiNodeTest =
127 attrs:
128 mkKubernetesBaseTest (
129 {
130 machines = {
131 machine1 = {
132 roles = [ "master" ];
133 ip = "192.168.1.1";
134 };
135 machine2 = {
136 roles = [ "node" ];
137 ip = "192.168.1.2";
138 };
139 };
140 }
141 // attrs
142 // {
143 name = "kubernetes-${attrs.name}-multinode";
144 }
145 );
146
147 mkKubernetesSingleNodeTest =
148 attrs:
149 mkKubernetesBaseTest (
150 {
151 machines = {
152 machine1 = {
153 roles = [
154 "master"
155 "node"
156 ];
157 ip = "192.168.1.1";
158 };
159 };
160 }
161 // attrs
162 // {
163 name = "kubernetes-${attrs.name}-singlenode";
164 }
165 );
166in
167{
168 inherit mkKubernetesBaseTest mkKubernetesSingleNodeTest mkKubernetesMultiNodeTest;
169}