pyrox.dev
1{ system ? builtins.currentSystem,
2 config ? {},
3 pkgs ? import ../../.. { inherit system config; }
4}:
5
6with import ../../lib/testing-python.nix { inherit system pkgs; };
7with pkgs.lib;
8
9let
10 mkKubernetesBaseTest =
11 { name, domain ? "my.zyx", test, machines
12 , extraConfiguration ? null }:
13 let
14 masterName = head (filter (machineName: any (role: role == "master") machines.${machineName}.roles) (attrNames machines));
15 master = machines.${masterName};
16 extraHosts = ''
17 ${master.ip} etcd.${domain}
18 ${master.ip} api.${domain}
19 ${concatMapStringsSep "\n" (machineName: "${machines.${machineName}.ip} ${machineName}.${domain}") (attrNames machines)}
20 '';
21 kubectl = with pkgs; runCommand "wrap-kubectl" { buildInputs = [ makeWrapper ]; } ''
22 mkdir -p $out/bin
23 makeWrapper ${pkgs.kubernetes}/bin/kubectl $out/bin/kubectl --set KUBECONFIG "/etc/kubernetes/cluster-admin.kubeconfig"
24 '';
25 in makeTest {
26 inherit name;
27
28 nodes = mapAttrs (machineName: machine:
29 { config, pkgs, lib, nodes, ... }:
30 mkMerge [
31 {
32 boot.postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*";
33 virtualisation.memorySize = mkDefault 1536;
34 virtualisation.diskSize = mkDefault 4096;
35 networking = {
36 inherit domain extraHosts;
37 primaryIPAddress = mkForce machine.ip;
38
39 firewall = {
40 allowedTCPPorts = [
41 10250 # kubelet
42 ];
43 trustedInterfaces = ["docker0"];
44
45 extraCommands = concatMapStrings (node: ''
46 iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT
47 '') (attrValues nodes);
48 };
49 };
50 programs.bash.enableCompletion = true;
51 environment.systemPackages = [ kubectl ];
52 services.flannel.iface = "eth1";
53 services.kubernetes = {
54 addons.dashboard.enable = true;
55 proxy.hostname = "${masterName}.${domain}";
56
57 easyCerts = true;
58 inherit (machine) roles;
59 apiserver = {
60 securePort = 443;
61 advertiseAddress = master.ip;
62 };
63 masterAddress = "${masterName}.${config.networking.domain}";
64 };
65 }
66 (optionalAttrs (any (role: role == "master") machine.roles) {
67 networking.firewall.allowedTCPPorts = [
68 443 # kubernetes apiserver
69 ];
70 })
71 (optionalAttrs (machine ? extraConfiguration) (machine.extraConfiguration { inherit config pkgs lib nodes; }))
72 (optionalAttrs (extraConfiguration != null) (extraConfiguration { inherit config pkgs lib nodes; }))
73 ]
74 ) machines;
75
76 testScript = ''
77 start_all()
78 '' + test;
79 };
80
81 mkKubernetesMultiNodeTest = attrs: mkKubernetesBaseTest ({
82 machines = {
83 machine1 = {
84 roles = ["master"];
85 ip = "192.168.1.1";
86 };
87 machine2 = {
88 roles = ["node"];
89 ip = "192.168.1.2";
90 };
91 };
92 } // attrs // {
93 name = "kubernetes-${attrs.name}-multinode";
94 });
95
96 mkKubernetesSingleNodeTest = attrs: mkKubernetesBaseTest ({
97 machines = {
98 machine1 = {
99 roles = ["master" "node"];
100 ip = "192.168.1.1";
101 };
102 };
103 } // attrs // {
104 name = "kubernetes-${attrs.name}-singlenode";
105 });
106in {
107 inherit mkKubernetesBaseTest mkKubernetesSingleNodeTest mkKubernetesMultiNodeTest;
108}