pyrox.dev
1{ system ? builtins.currentSystem,
2 config ? {},
3 pkgs ? import ../../.. { inherit system config; }
4}:
5
6with import ../../lib/testing-python.nix { inherit system pkgs; };
7with pkgs.lib;
8
9let
10 mkKubernetesBaseTest =
11 { name, domain ? "my.zyx", test, machines
12 , extraConfiguration ? null }:
13 let
14 masterName = head (filter (machineName: any (role: role == "master") machines.${machineName}.roles) (attrNames machines));
15 master = machines.${masterName};
16 extraHosts = ''
17 ${master.ip} etcd.${domain}
18 ${master.ip} api.${domain}
19 ${concatMapStringsSep "\n" (machineName: "${machines.${machineName}.ip} ${machineName}.${domain}") (attrNames machines)}
20 '';
21 wrapKubectl = with pkgs; runCommand "wrap-kubectl" { nativeBuildInputs = [ makeWrapper ]; } ''
22 mkdir -p $out/bin
23 makeWrapper ${pkgs.kubernetes}/bin/kubectl $out/bin/kubectl --set KUBECONFIG "/etc/kubernetes/cluster-admin.kubeconfig"
24 '';
25 in makeTest {
26 inherit name;
27
28 nodes = mapAttrs (machineName: machine:
29 { config, pkgs, lib, nodes, ... }:
30 mkMerge [
31 {
32 boot.postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*";
33 virtualisation.memorySize = mkDefault 1536;
34 virtualisation.diskSize = mkDefault 4096;
35 networking = {
36 inherit domain extraHosts;
37 primaryIPAddress = mkForce machine.ip;
38
39 firewall = {
40 allowedTCPPorts = [
41 10250 # kubelet
42 ];
43 trustedInterfaces = ["mynet"];
44
45 extraCommands = concatMapStrings (node: ''
46 iptables -A INPUT -s ${node.networking.primaryIPAddress} -j ACCEPT
47 '') (attrValues nodes);
48 };
49 };
50 programs.bash.enableCompletion = true;
51 environment.systemPackages = [ wrapKubectl ];
52 services.flannel.iface = "eth1";
53 services.kubernetes = {
54 proxy.hostname = "${masterName}.${domain}";
55
56 easyCerts = true;
57 inherit (machine) roles;
58 apiserver = {
59 securePort = 443;
60 advertiseAddress = master.ip;
61 };
62 masterAddress = "${masterName}.${config.networking.domain}";
63 };
64 }
65 (optionalAttrs (any (role: role == "master") machine.roles) {
66 networking.firewall.allowedTCPPorts = [
67 443 # kubernetes apiserver
68 ];
69 })
70 (optionalAttrs (machine ? extraConfiguration) (machine.extraConfiguration { inherit config pkgs lib nodes; }))
71 (optionalAttrs (extraConfiguration != null) (extraConfiguration { inherit config pkgs lib nodes; }))
72 ]
73 ) machines;
74
75 testScript = ''
76 start_all()
77 '' + test;
78 };
79
80 mkKubernetesMultiNodeTest = attrs: mkKubernetesBaseTest ({
81 machines = {
82 machine1 = {
83 roles = ["master"];
84 ip = "192.168.1.1";
85 };
86 machine2 = {
87 roles = ["node"];
88 ip = "192.168.1.2";
89 };
90 };
91 } // attrs // {
92 name = "kubernetes-${attrs.name}-multinode";
93 });
94
95 mkKubernetesSingleNodeTest = attrs: mkKubernetesBaseTest ({
96 machines = {
97 machine1 = {
98 roles = ["master" "node"];
99 ip = "192.168.1.1";
100 };
101 };
102 } // attrs // {
103 name = "kubernetes-${attrs.name}-singlenode";
104 });
105in {
106 inherit mkKubernetesBaseTest mkKubernetesSingleNodeTest mkKubernetesMultiNodeTest;
107}