pyrox.dev
1{ system ? builtins.currentSystem }:
2
3with import ../../lib/testing.nix { inherit system; };
4with pkgs.lib;
5
6let
7 mkKubernetesBaseTest =
8 { name, domain ? "my.zyx", test, machines
9 , pkgs ? import <nixpkgs> { inherit system; }
10 , certs ? import ./certs.nix { inherit pkgs; externalDomain = domain; kubelets = attrNames machines; }
11 , extraConfiguration ? null }:
12 let
13 masterName = head (filter (machineName: any (role: role == "master") machines.${machineName}.roles) (attrNames machines));
14 master = machines.${masterName};
15 extraHosts = ''
16 ${master.ip} etcd.${domain}
17 ${master.ip} api.${domain}
18 ${concatMapStringsSep "\n" (machineName: "${machines.${machineName}.ip} ${machineName}.${domain}") (attrNames machines)}
19 '';
20 in makeTest {
21 inherit name;
22
23 nodes = mapAttrs (machineName: machine:
24 { config, pkgs, lib, nodes, ... }:
25 mkMerge [
26 {
27 virtualisation.memorySize = mkDefault 1536;
28 virtualisation.diskSize = mkDefault 4096;
29 networking = {
30 inherit domain extraHosts;
31 primaryIPAddress = mkForce machine.ip;
32
33 firewall = {
34 allowedTCPPorts = [
35 10250 # kubelet
36 ];
37 trustedInterfaces = ["docker0"];
38
39 extraCommands = concatMapStrings (node: ''
40 iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT
41 '') (attrValues nodes);
42 };
43 };
44 programs.bash.enableCompletion = true;
45 environment.variables = {
46 ETCDCTL_CERT_FILE = "${certs.worker}/etcd-client.pem";
47 ETCDCTL_KEY_FILE = "${certs.worker}/etcd-client-key.pem";
48 ETCDCTL_CA_FILE = "${certs.worker}/ca.pem";
49 ETCDCTL_PEERS = "https://etcd.${domain}:2379";
50 };
51 services.flannel.iface = "eth1";
52 services.kubernetes.apiserver.advertiseAddress = master.ip;
53 }
54 (optionalAttrs (any (role: role == "master") machine.roles) {
55 networking.firewall.allowedTCPPorts = [
56 2379 2380 # etcd
57 443 # kubernetes apiserver
58 ];
59 services.etcd = {
60 enable = true;
61 certFile = "${certs.master}/etcd.pem";
62 keyFile = "${certs.master}/etcd-key.pem";
63 trustedCaFile = "${certs.master}/ca.pem";
64 peerClientCertAuth = true;
65 listenClientUrls = ["https://0.0.0.0:2379"];
66 listenPeerUrls = ["https://0.0.0.0:2380"];
67 advertiseClientUrls = ["https://etcd.${config.networking.domain}:2379"];
68 initialCluster = ["${masterName}=https://etcd.${config.networking.domain}:2380"];
69 initialAdvertisePeerUrls = ["https://etcd.${config.networking.domain}:2380"];
70 };
71 })
72 (import ./kubernetes-common.nix { inherit (machine) roles; inherit pkgs config certs; })
73 (optionalAttrs (machine ? "extraConfiguration") (machine.extraConfiguration { inherit config pkgs lib nodes; }))
74 (optionalAttrs (extraConfiguration != null) (extraConfiguration { inherit config pkgs lib nodes; }))
75 ]
76 ) machines;
77
78 testScript = ''
79 startAll;
80
81 ${test}
82 '';
83 };
84
85 mkKubernetesMultiNodeTest = attrs: mkKubernetesBaseTest ({
86 machines = {
87 machine1 = {
88 roles = ["master"];
89 ip = "192.168.1.1";
90 };
91 machine2 = {
92 roles = ["node"];
93 ip = "192.168.1.2";
94 };
95 };
96 } // attrs // {
97 name = "kubernetes-${attrs.name}-multinode";
98 });
99
100 mkKubernetesSingleNodeTest = attrs: mkKubernetesBaseTest ({
101 machines = {
102 machine1 = {
103 roles = ["master" "node"];
104 ip = "192.168.1.1";
105 };
106 };
107 } // attrs // {
108 name = "kubernetes-${attrs.name}-singlenode";
109 });
110in {
111 inherit mkKubernetesBaseTest mkKubernetesSingleNodeTest mkKubernetesMultiNodeTest;
112}