commit 6ef8cad2a7683e933411a3ff1c1cb70c4e45032b · pyrox.dev/nixpkgs

+113

nixos/tests/kubernetes/base.nix

···

       1
       1
       +
       { system ? builtins.currentSystem }:

     

       2
       2
       +
       

     

       3
       3
       +
       with import ../../lib/testing.nix { inherit system; };

     

       4
       4
       +
       with import ../../lib/qemu-flags.nix;

     

       5
       5
       +
       with pkgs.lib;

     

       6
       6
       +
       

     

       7
       7
       +
       let

     

       8
       8
       +
         mkKubernetesBaseTest =

     

       9
       9
       +
           { name, domain ? "my.zyx", test, machines

     

       10
       10
       +
           , pkgs ? import <nixpkgs> { inherit system; }

     

       11
       11
       +
           , certs ? import ./certs.nix { inherit pkgs; externalDomain = domain; }

     

       12
       12
       +
           , extraConfiguration ? null }:

     

       13
       13
       +
           let

     

       14
       14
       +
             masterName = head (filter (machineName: any (role: role == "master") machines.${machineName}.roles) (attrNames machines));

     

       15
       15
       +
             master = machines.${masterName};

     

       16
       16
       +
             extraHosts = ''

     

       17
       17
       +
               ${master.ip}  etcd.${domain}

     

       18
       18
       +
               ${master.ip}  api.${domain}

     

       19
       19
       +
               ${concatMapStringsSep "\n" (machineName: "${machines.${machineName}.ip}  ${machineName}.${domain}") (attrNames machines)}

     

       20
       20
       +
             '';

     

       21
       21
       +
           in makeTest {

     

       22
       22
       +
             inherit name;

     

       23
       23
       +
       

     

       24
       24
       +
             nodes = mapAttrs (machineName: machine:

     

       25
       25
       +
               { config, pkgs, lib, nodes, ... }:

     

       26
       26
       +
                 mkMerge [

     

       27
       27
       +
                   {

     

       28
       28
       +
                     virtualisation.memorySize = mkDefault 768;

     

       29
       29
       +
                     virtualisation.diskSize = mkDefault 4096;

     

       30
       30
       +
                     networking = {

     

       31
       31
       +
                       inherit domain extraHosts;

     

       32
       32
       +
                       primaryIPAddress = mkForce machine.ip;

     

       33
       33
       +
       

     

       34
       34
       +
                       firewall = {

     

       35
       35
       +
                         allowedTCPPorts = [

     

       36
       36
       +
                           10250 # kubelet

     

       37
       37
       +
                         ];

     

       38
       38
       +
                         trustedInterfaces = ["docker0"];

     

       39
       39
       +
       

     

       40
       40
       +
                         extraCommands = concatMapStrings  (node: ''

     

       41
       41
       +
                           iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT

     

       42
       42
       +
                         '') (attrValues nodes);

     

       43
       43
       +
                       };

     

       44
       44
       +
                     };

     

       45
       45
       +
                     programs.bash.enableCompletion = true;

     

       46
       46
       +
                     environment.variables = {

     

       47
       47
       +
                       ETCDCTL_CERT_FILE = "${certs.worker}/etcd-client.pem";

     

       48
       48
       +
                       ETCDCTL_KEY_FILE = "${certs.worker}/etcd-client-key.pem";

     

       49
       49
       +
                       ETCDCTL_CA_FILE = "${certs.worker}/ca.pem";

     

       50
       50
       +
                       ETCDCTL_PEERS = "https://etcd.${domain}:2379";

     

       51
       51
       +
                     };

     

       52
       52
       +
                     services.flannel.iface = "eth1";

     

       53
       53
       +
                     services.kubernetes.apiserver.advertiseAddress = master.ip;

     

       54
       54
       +
                   }

     

       55
       55
       +
                   (optionalAttrs (any (role: role == "master") machine.roles) {

     

       56
       56
       +
                     networking.firewall.allowedTCPPorts = [

     

       57
       57
       +
                       2379 2380  # etcd

     

       58
       58
       +
                       443 # kubernetes apiserver

     

       59
       59
       +
                     ];

     

       60
       60
       +
                     services.etcd = {

     

       61
       61
       +
                       enable = true;

     

       62
       62
       +
                       certFile = "${certs.master}/etcd.pem";

     

       63
       63
       +
                       keyFile = "${certs.master}/etcd-key.pem";

     

       64
       64
       +
                       trustedCaFile = "${certs.master}/ca.pem";

     

       65
       65
       +
                       peerClientCertAuth = true;

     

       66
       66
       +
                       listenClientUrls = ["https://0.0.0.0:2379"];

     

       67
       67
       +
                       listenPeerUrls = ["https://0.0.0.0:2380"];

     

       68
       68
       +
                       advertiseClientUrls = ["https://etcd.${config.networking.domain}:2379"];

     

       69
       69
       +
                       initialCluster = ["${masterName}=https://etcd.${config.networking.domain}:2380"];

     

       70
       70
       +
                       initialAdvertisePeerUrls = ["https://etcd.${config.networking.domain}:2380"];

     

       71
       71
       +
                     };

     

       72
       72
       +
                   })

     

       73
       73
       +
                   (import ./kubernetes-common.nix { inherit (machine) roles; inherit pkgs config certs; })

     

       74
       74
       +
                   (optionalAttrs (machine ? "extraConfiguration") (machine.extraConfiguration { inherit config pkgs lib nodes; }))

     

       75
       75
       +
                   (optionalAttrs (extraConfiguration != null) (extraConfiguration { inherit config pkgs lib nodes; }))

     

       76
       76
       +
                 ]

     

       77
       77
       +
             ) machines;

     

       78
       78
       +
       

     

       79
       79
       +
             testScript = ''

     

       80
       80
       +
               startAll;

     

       81
       81
       +
       

     

       82
       82
       +
               ${test}

     

       83
       83
       +
             '';

     

       84
       84
       +
           };

     

       85
       85
       +
       

     

       86
       86
       +
         mkKubernetesMultiNodeTest = attrs: mkKubernetesBaseTest ({

     

       87
       87
       +
           machines = {

     

       88
       88
       +
             machine1 = {

     

       89
       89
       +
               roles = ["master"];

     

       90
       90
       +
               ip = "192.168.1.1";

     

       91
       91
       +
             };

     

       92
       92
       +
             machine2 = {

     

       93
       93
       +
               roles = ["node"];

     

       94
       94
       +
               ip = "192.168.1.2";

     

       95
       95
       +
             };

     

       96
       96
       +
           };

     

       97
       97
       +
         } // attrs // {

     

       98
       98
       +
           name = "kubernetes-${attrs.name}-multinode";

     

       99
       99
       +
         });

     

       100
       100
       +
       

     

       101
       101
       +
         mkKubernetesSingleNodeTest = attrs: mkKubernetesBaseTest ({

     

       102
       102
       +
           machines = {

     

       103
       103
       +
             machine1 = {

     

       104
       104
       +
               roles = ["master" "node"];

     

       105
       105
       +
               ip = "192.168.1.1";

     

       106
       106
       +
             };

     

       107
       107
       +
           };

     

       108
       108
       +
         } // attrs // {

     

       109
       109
       +
           name = "kubernetes-${attrs.name}-singlenode";

     

       110
       110
       +
         });

     

       111
       111
       +
       in {

     

       112
       112
       +
         inherit mkKubernetesBaseTest mkKubernetesSingleNodeTest mkKubernetesMultiNodeTest;

     

       113
       113
       +
       }

+161 -205

nixos/tests/kubernetes/certs.nix

···

       1
       1
        
       {

     

       2
       2
        
         pkgs ? import <nixpkgs> {},

     

       3
       3
       -
         servers ? {test = "1.2.3.4";},

     

       4
       4
       -
         internalDomain ? "cluster.local",

     

       5
       5
       -
         externalDomain ? "nixos.xyz"

     

       3
       3
       +
         internalDomain ? "cloud.yourdomain.net",

     

       4
       4
       +
         externalDomain ? "myawesomecluster.cluster.yourdomain.net",

     

       5
       5
       +
         serviceClusterIp ? "10.0.0.1"

     

       6
       6
        
       }:

     

       7
       7
        
       let

     

       8
       8
       -
         mkAltNames = ipFrom: dnsFrom:

     

       9
       9
       -
           pkgs.lib.concatImapStringsSep "\n" (i: v: "IP.${toString (i+ipFrom)} = ${v.ip}\nDNS.${toString (i+dnsFrom)} = ${v.name}.${externalDomain}") (pkgs.lib.mapAttrsToList (n: v: {name = n; ip = v;}) servers);

     

       8
       8
       +
         runWithCFSSL = name: cmd:

     

       9
       9
       +
           builtins.fromJSON (builtins.readFile (

     

       10
       10
       +
             pkgs.runCommand "${name}-cfss.json" {

     

       11
       11
       +
               buildInputs = [ pkgs.cfssl ];

     

       12
       12
       +
             } "cfssl ${cmd} > $out"

     

       13
       13
       +
           ));

     

       10
       14
        
       

     

       11
       11
       -
         runWithOpenSSL = file: cmd: pkgs.runCommand file {

     

       12
       12
       -
           buildInputs = [ pkgs.openssl ];

     

       13
       13
       -
           passthru = { inherit file; };

     

       14
       14
       -
         } cmd;

     

       15
       15
       +
         writeCFSSL = content:

     

       16
       16
       +
           pkgs.runCommand content.name {

     

       17
       17
       +
             buildInputs = [ pkgs.cfssl ];

     

       18
       18
       +
           } ''

     

       19
       19
       +
             mkdir -p $out

     

       20
       20
       +
             cd $out

     

       21
       21
       +
             cat ${writeFile content} | cfssljson -bare ${content.name}

     

       22
       22
       +
           '';

     

       15
       23
        
       

     

       16
       16
       -
         ca_key = runWithOpenSSL "ca-key.pem" "openssl genrsa -out $out 2048";

     

       17
       17
       -
         ca_pem = runWithOpenSSL "ca.pem" ''

     

       18
       18
       -
           openssl req \

     

       19
       19
       -
             -x509 -new -nodes -key ${ca_key} \

     

       20
       20
       -
             -days 10000 -out $out -subj "/CN=etcd-ca"

     

       21
       21
       -
         '';

     

       24
       24
       +
         noCSR = content: pkgs.lib.filterAttrs (n: v: n != "csr") content;

     

       25
       25
       +
         noKey = content: pkgs.lib.filterAttrs (n: v: n != "key") content;

     

       22
       26
        
       

     

       23
       23
       -
         etcd_cnf = pkgs.writeText "openssl.cnf" ''

     

       24
       24
       -
           [req]

     

       25
       25
       -
           req_extensions = v3_req

     

       26
       26
       -
           distinguished_name = req_distinguished_name

     

       27
       27
       -
           [req_distinguished_name]

     

       28
       28
       -
           [ v3_req ]

     

       29
       29
       -
           basicConstraints = CA:FALSE

     

       30
       30
       -
           keyUsage = digitalSignature, keyEncipherment

     

       31
       31
       -
           extendedKeyUsage = serverAuth

     

       32
       32
       -
           subjectAltName = @alt_names

     

       33
       33
       -
           [alt_names]

     

       34
       34
       -
           DNS.1 = etcd.kubernetes.${externalDomain}

     

       35
       35
       -
           IP.1 = 127.0.0.1

     

       36
       36
       -
         '';

     

       37
       37
       -
         etcd_key = runWithOpenSSL "etcd-key.pem" "openssl genrsa -out $out 2048";

     

       38
       38
       -
         etcd_csr = runWithOpenSSL "etcd.csr" ''

     

       39
       39
       -
           openssl req \

     

       40
       40
       -
             -new -key ${etcd_key} \

     

       41
       41
       -
             -out $out -subj "/CN=etcd" \

     

       42
       42
       -
             -config ${etcd_cnf}

     

       43
       43
       -
         '';

     

       44
       44
       -
         etcd_cert = runWithOpenSSL "etcd.pem" ''

     

       45
       45
       -
           openssl x509 \

     

       46
       46
       -
             -req -in ${etcd_csr} \

     

       47
       47
       -
             -CA ${ca_pem} -CAkey ${ca_key} \

     

       48
       48
       -
             -CAcreateserial -out $out \

     

       49
       49
       -
             -days 3650 -extensions v3_req \

     

       50
       50
       -
             -extfile ${etcd_cnf}

     

       51
       51
       -
         '';

     

       27
       27
       +
         writeFile = content: pkgs.writeText "content" (

     

       28
       28
       +
           if pkgs.lib.isAttrs content then builtins.toJSON content

     

       29
       29
       +
           else toString content

     

       30
       30
       +
         );

     

       52
       31
        
       

     

       53
       53
       -
         etcd_client_key = runWithOpenSSL "etcd-client-key.pem"

     

       54
       54
       -
           "openssl genrsa -out $out 2048";

     

       32
       32
       +
         createServingCertKey = { ca, cn, hosts? [], size ? 2048, name ? cn }:

     

       33
       33
       +
           noCSR (

     

       34
       34
       +
             (runWithCFSSL name "gencert -ca=${writeFile ca.cert} -ca-key=${writeFile ca.key} -profile=server -config=${writeFile ca.config} ${writeFile {

     

       35
       35
       +
               CN = cn;

     

       36
       36
       +
               hosts = hosts;

     

       37
       37
       +
               key = { algo = "rsa"; inherit size; };

     

       38
       38
       +
             }}") // { inherit name; }

     

       39
       39
       +
           );

     

       55
       40
        
       

     

       56
       56
       -
         etcd_client_csr = runWithOpenSSL "etcd-client.csr" ''

     

       57
       57
       -
           openssl req \

     

       58
       58
       -
             -new -key ${etcd_client_key} \

     

       59
       59
       -
             -out $out -subj "/CN=etcd-client" \

     

       60
       60
       -
             -config ${client_openssl_cnf}

     

       61
       61
       -
         '';

     

       41
       41
       +
         createClientCertKey = { ca, cn, groups ? [], size ? 2048, name ? cn }:

     

       42
       42
       +
           noCSR (

     

       43
       43
       +
             (runWithCFSSL name "gencert -ca=${writeFile ca.cert} -ca-key=${writeFile ca.key} -profile=client -config=${writeFile ca.config} ${writeFile {

     

       44
       44
       +
               CN = cn;

     

       45
       45
       +
               names = map (group: {O = group;}) groups;

     

       46
       46
       +
               hosts = [""];

     

       47
       47
       +
               key = { algo = "rsa"; inherit size; };

     

       48
       48
       +
             }}") // { inherit name; }

     

       49
       49
       +
           );

     

       62
       50
        
       

     

       63
       63
       -
         etcd_client_cert = runWithOpenSSL "etcd-client.crt" ''

     

       64
       64
       -
           openssl x509 \

     

       65
       65
       -
             -req -in ${etcd_client_csr} \

     

       66
       66
       -
             -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \

     

       67
       67
       -
             -out $out -days 3650 -extensions v3_req \

     

       68
       68
       -
             -extfile ${client_openssl_cnf}

     

       69
       69
       -
         '';

     

       51
       51
       +
         createSigningCertKey = { C ? "xx", ST ? "x", L ? "x", O ? "x", OU ? "x", CN ? "ca", emailAddress ? "x", expiry ? "43800h", size ? 2048, name ? CN }:

     

       52
       52
       +
           (noCSR (runWithCFSSL CN "genkey -initca ${writeFile {

     

       53
       53
       +
             key = { algo = "rsa"; inherit size; };

     

       54
       54
       +
             names = [{ inherit C ST L O OU CN emailAddress; }];

     

       55
       55
       +
           }}")) // {

     

       56
       56
       +
             inherit name;

     

       57
       57
       +
             config.signing = {

     

       58
       58
       +
               default.expiry = expiry;

     

       59
       59
       +
               profiles = {

     

       60
       60
       +
                 server = {

     

       61
       61
       +
                   inherit expiry;

     

       62
       62
       +
                   usages = [

     

       63
       63
       +
                     "signing"

     

       64
       64
       +
                     "key encipherment"

     

       65
       65
       +
                     "server auth"

     

       66
       66
       +
                   ];

     

       67
       67
       +
                 };

     

       68
       68
       +
                 client = {

     

       69
       69
       +
                   inherit expiry;

     

       70
       70
       +
                   usages = [

     

       71
       71
       +
                     "signing"

     

       72
       72
       +
                     "key encipherment"

     

       73
       73
       +
                     "client auth"

     

       74
       74
       +
                   ];

     

       75
       75
       +
                 };

     

       76
       76
       +
                 peer = {

     

       77
       77
       +
                   inherit expiry;

     

       78
       78
       +
                   usages = [

     

       79
       79
       +
                     "signing"

     

       80
       80
       +
                     "key encipherment"

     

       81
       81
       +
                     "server auth"

     

       82
       82
       +
                     "client auth"

     

       83
       83
       +
                   ];

     

       84
       84
       +
                 };

     

       85
       85
       +
               };

     

       86
       86
       +
             };

     

       87
       87
       +
           };

     

       70
       88
        
       

     

       71
       71
       -
         admin_key = runWithOpenSSL "admin-key.pem"

     

       72
       72
       -
           "openssl genrsa -out $out 2048";

     

       89
       89
       +
         ca = createSigningCertKey {};

     

       73
       90
        
       

     

       74
       74
       -
         admin_csr = runWithOpenSSL "admin.csr" ''

     

       75
       75
       -
           openssl req \

     

       76
       76
       -
             -new -key ${admin_key} \

     

       77
       77
       -
             -out $out -subj "/CN=admin/O=system:masters" \

     

       78
       78
       -
             -config ${client_openssl_cnf}

     

       79
       79
       -
         '';

     

       91
       91
       +
         kube-apiserver = createServingCertKey {

     

       92
       92
       +
           inherit ca;

     

       93
       93
       +
           cn = "kube-apiserver";

     

       94
       94
       +
           hosts = ["kubernetes.default" "kubernetes.default.svc" "localhost" "api.${externalDomain}" serviceClusterIp];

     

       95
       95
       +
         };

     

       80
       96
        
       

     

       81
       81
       -
         admin_cert = runWithOpenSSL "admin.crt" ''

     

       82
       82
       -
           openssl x509 \

     

       83
       83
       -
             -req -in ${admin_csr} \

     

       84
       84
       -
             -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \

     

       85
       85
       -
             -out $out -days 3650 -extensions v3_req \

     

       86
       86
       -
             -extfile ${client_openssl_cnf}

     

       87
       87
       -
         '';

     

       88
       88
       -
       

     

       89
       89
       -
         apiserver_key = runWithOpenSSL "apiserver-key.pem" "openssl genrsa -out $out 2048";

     

       90
       90
       -
       

     

       91
       91
       -
         apiserver_csr = runWithOpenSSL "apiserver.csr" ''

     

       92
       92
       -
           openssl req \

     

       93
       93
       -
             -new -key ${apiserver_key} \

     

       94
       94
       -
             -out $out -subj "/CN=kube-apiserver" \

     

       95
       95
       -
             -config ${apiserver_cnf}

     

       96
       96
       -
         '';

     

       97
       97
       -
       

     

       98
       98
       -
         apiserver_cert = runWithOpenSSL "apiserver.pem" ''

     

       99
       99
       -
           openssl x509 \

     

       100
       100
       -
             -req -in ${apiserver_csr} \

     

       101
       101
       -
             -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \

     

       102
       102
       -
             -out $out -days 3650 -extensions v3_req \

     

       103
       103
       -
             -extfile ${apiserver_cnf}

     

       104
       104
       -
         '';

     

       105
       105
       -
       

     

       106
       106
       -
         worker_key = runWithOpenSSL "worker-key.pem" "openssl genrsa -out $out 2048";

     

       107
       107
       -
       

     

       108
       108
       -
         worker_csr = runWithOpenSSL "worker.csr" ''

     

       109
       109
       -
           openssl req \

     

       110
       110
       -
             -new -key ${worker_key} \

     

       111
       111
       -
             -out $out -subj "/CN=kube-worker/O=system:authenticated" \

     

       112
       112
       -
             -config ${worker_cnf}

     

       113
       113
       -
         '';

     

       97
       97
       +
         kubelet = createServingCertKey {

     

       98
       98
       +
           inherit ca;

     

       99
       99
       +
           cn = "kubelet";

     

       100
       100
       +
           hosts = ["*.${externalDomain}"];

     

       101
       101
       +
         };

     

       114
       102
        
       

     

       115
       115
       -
         worker_cert = runWithOpenSSL "worker.pem" ''

     

       116
       116
       -
           openssl x509 \

     

       117
       117
       -
             -req -in ${worker_csr} \

     

       118
       118
       -
             -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \

     

       119
       119
       -
             -out $out -days 3650 -extensions v3_req \

     

       120
       120
       -
             -extfile ${worker_cnf}

     

       121
       121
       -
         '';

     

       103
       103
       +
         service-accounts = createServingCertKey {

     

       104
       104
       +
           inherit ca;

     

       105
       105
       +
           cn = "kube-service-accounts";

     

       106
       106
       +
         };

     

       122
       107
        
       

     

       123
       123
       -
         openssl_cnf = pkgs.writeText "openssl.cnf" ''

     

       124
       124
       -
           [req]

     

       125
       125
       -
           req_extensions = v3_req

     

       126
       126
       -
           distinguished_name = req_distinguished_name

     

       127
       127
       -
           [req_distinguished_name]

     

       128
       128
       -
           [ v3_req ]

     

       129
       129
       -
           basicConstraints = CA:FALSE

     

       130
       130
       -
           keyUsage = digitalSignature, keyEncipherment

     

       131
       131
       -
           extendedKeyUsage = serverAuth

     

       132
       132
       -
           subjectAltName = @alt_names

     

       133
       133
       -
           [alt_names]

     

       134
       134
       -
           DNS.1 = *.cluster.${externalDomain}

     

       135
       135
       -
           IP.1 = 127.0.0.1

     

       136
       136
       -
           ${mkAltNames 1 1}

     

       137
       137
       -
         '';

     

       108
       108
       +
         etcd = createServingCertKey {

     

       109
       109
       +
           inherit ca;

     

       110
       110
       +
           cn = "etcd";

     

       111
       111
       +
           hosts = ["etcd.${externalDomain}"];

     

       112
       112
       +
         };

     

       138
       113
        
       

     

       139
       139
       -
         client_openssl_cnf = pkgs.writeText "client-openssl.cnf" ''

     

       140
       140
       -
           [req]

     

       141
       141
       -
           req_extensions = v3_req

     

       142
       142
       -
           distinguished_name = req_distinguished_name

     

       143
       143
       -
           [req_distinguished_name]

     

       144
       144
       -
           [ v3_req ]

     

       145
       145
       -
           basicConstraints = CA:FALSE

     

       146
       146
       -
           keyUsage = digitalSignature, keyEncipherment

     

       147
       147
       -
           extendedKeyUsage = clientAuth

     

       148
       148
       -
           subjectAltName = @alt_names

     

       149
       149
       -
           [alt_names]

     

       150
       150
       -
           DNS.1 = kubernetes

     

       151
       151
       -
           DNS.2 = kubernetes.default

     

       152
       152
       -
           DNS.3 = kubernetes.default.svc

     

       153
       153
       -
           DNS.4 = kubernetes.default.svc.${internalDomain}

     

       154
       154
       -
           DNS.5 = kubernetes.${externalDomain}

     

       155
       155
       -
           DNS.6 = *.cluster.${externalDomain}

     

       156
       156
       -
           IP.1 = 10.1.10.1

     

       157
       157
       -
           ${mkAltNames 1 6}

     

       158
       158
       -
         '';

     

       114
       114
       +
         etcd-client = createClientCertKey {

     

       115
       115
       +
           inherit ca;

     

       116
       116
       +
           cn = "etcd-client";

     

       117
       117
       +
         };

     

       159
       118
        
       

     

       160
       160
       -
         apiserver_cnf = pkgs.writeText "apiserver-openssl.cnf" ''

     

       161
       161
       -
           [req]

     

       162
       162
       -
           req_extensions = v3_req

     

       163
       163
       -
           distinguished_name = req_distinguished_name

     

       164
       164
       -
           [req_distinguished_name]

     

       165
       165
       -
           [ v3_req ]

     

       166
       166
       -
           basicConstraints = CA:FALSE

     

       167
       167
       -
           keyUsage = nonRepudiation, digitalSignature, keyEncipherment

     

       168
       168
       -
           extendedKeyUsage = serverAuth

     

       169
       169
       -
           subjectAltName = @alt_names

     

       170
       170
       -
           [alt_names]

     

       171
       171
       -
           DNS.1 = kubernetes

     

       172
       172
       -
           DNS.2 = kubernetes.default

     

       173
       173
       -
           DNS.3 = kubernetes.default.svc

     

       174
       174
       -
           DNS.4 = kubernetes.default.svc.${internalDomain}

     

       175
       175
       -
           DNS.5 = kubernetes.${externalDomain}

     

       176
       176
       -
           DNS.6 = *.cluster.${externalDomain}

     

       177
       177
       -
           IP.1 = 10.1.10.1

     

       178
       178
       -
           ${mkAltNames 1 6}

     

       179
       179
       -
         '';

     

       119
       119
       +
         kubelet-client = createClientCertKey {

     

       120
       120
       +
           inherit ca;

     

       121
       121
       +
           cn = "kubelet-client";

     

       122
       122
       +
           groups = ["system:masters"];

     

       123
       123
       +
         };

     

       180
       124
        
       

     

       181
       181
       -
         worker_cnf = pkgs.writeText "worker-openssl.cnf" ''

     

       182
       182
       -
           [req]

     

       183
       183
       -
           req_extensions = v3_req

     

       184
       184
       -
           distinguished_name = req_distinguished_name

     

       185
       185
       -
           [req_distinguished_name]

     

       186
       186
       -
           [ v3_req ]

     

       187
       187
       -
           basicConstraints = CA:FALSE

     

       188
       188
       -
           keyUsage = nonRepudiation, digitalSignature, keyEncipherment

     

       189
       189
       -
           subjectAltName = @alt_names

     

       190
       190
       -
           [alt_names]

     

       191
       191
       -
           DNS.1 = *.cluster.${externalDomain}

     

       192
       192
       -
           IP.1 = 10.1.10.1

     

       193
       193
       -
           ${mkAltNames 1 1}

     

       194
       194
       -
         '';

     

       125
       125
       +
         apiserver-client = {

     

       126
       126
       +
           kubelet = createClientCertKey {

     

       127
       127
       +
             inherit ca;

     

       128
       128
       +
             cn = "apiserver-client-kubelet";

     

       129
       129
       +
             groups = ["system:nodes"];

     

       130
       130
       +
           };

     

       195
       131
        
       

     

       196
       196
       -
         ln = cert: target: ''

     

       197
       197
       -
           cp -v ${cert} ${target}/${cert.file}

     

       198
       198
       -
         '';

     

       199
       199
       -
       in

     

       200
       200
       -
         pkgs.stdenv.mkDerivation rec {

     

       201
       201
       -
           name = "kubernetes-certs";

     

       202
       202
       -
           unpackPhase = "true";

     

       203
       203
       -
           installPhase = ''

     

       204
       204
       -
             set -xe

     

       205
       205
       -
             mkdir -p $out

     

       206
       206
       -
             ${ln ca_key "$out"}

     

       207
       207
       -
             ${ln ca_pem "$out"}

     

       132
       132
       +
           kube-proxy = createClientCertKey {

     

       133
       133
       +
             inherit ca;

     

       134
       134
       +
             name = "apiserver-client-kube-proxy";

     

       135
       135
       +
             cn = "system:kube-proxy";

     

       136
       136
       +
             groups = ["system:kube-proxy" "system:nodes"];

     

       137
       137
       +
           };

     

       208
       138
        
       

     

       209
       209
       -
             ${ln etcd_key "$out"}

     

       210
       210
       -
             ${ln etcd_csr "$out"}

     

       211
       211
       -
             ${ln etcd_cert "$out"}

     

       139
       139
       +
           kube-controller-manager = createClientCertKey {

     

       140
       140
       +
             inherit ca;

     

       141
       141
       +
             name = "apiserver-client-kube-controller-manager";

     

       142
       142
       +
             cn = "system:kube-controller-manager";

     

       143
       143
       +
             groups = ["system:masters"];

     

       144
       144
       +
           };

     

       212
       145
        
       

     

       213
       213
       -
             ${ln etcd_client_key "$out"}

     

       214
       214
       -
             ${ln etcd_client_csr "$out"}

     

       215
       215
       -
             ${ln etcd_client_cert "$out"}

     

       146
       146
       +
           kube-scheduler = createClientCertKey {

     

       147
       147
       +
             inherit ca;

     

       148
       148
       +
             name = "apiserver-client-kube-scheduler";

     

       149
       149
       +
             cn = "system:kube-scheduler";

     

       150
       150
       +
             groups = ["system:kube-scheduler"];

     

       151
       151
       +
           };

     

       216
       152
        
       

     

       217
       217
       -
             ${ln apiserver_key "$out"}

     

       218
       218
       -
             ${ln apiserver_csr "$out"}

     

       219
       219
       -
             ${ln apiserver_cert "$out"}

     

       153
       153
       +
           admin = createClientCertKey {

     

       154
       154
       +
             inherit ca;

     

       155
       155
       +
             cn = "admin";

     

       156
       156
       +
             groups = ["system:masters"];

     

       157
       157
       +
           };

     

       158
       158
       +
         };

     

       159
       159
       +
       in {

     

       160
       160
       +
         master = pkgs.buildEnv {

     

       161
       161
       +
           name = "master-keys";

     

       162
       162
       +
           paths = [

     

       163
       163
       +
             (writeCFSSL (noKey ca))

     

       164
       164
       +
             (writeCFSSL kube-apiserver)

     

       165
       165
       +
             (writeCFSSL kubelet-client)

     

       166
       166
       +
             (writeCFSSL apiserver-client.kube-controller-manager)

     

       167
       167
       +
             (writeCFSSL apiserver-client.kube-scheduler)

     

       168
       168
       +
             (writeCFSSL service-accounts)

     

       169
       169
       +
             (writeCFSSL etcd)

     

       170
       170
       +
           ];

     

       171
       171
       +
         };

     

       220
       172
        
       

     

       221
       221
       -
             ${ln worker_key "$out"}

     

       222
       222
       -
             ${ln worker_csr "$out"}

     

       223
       223
       -
             ${ln worker_cert "$out"}

     

       173
       173
       +
         worker = pkgs.buildEnv {

     

       174
       174
       +
           name = "worker-keys";

     

       175
       175
       +
           paths = [

     

       176
       176
       +
             (writeCFSSL (noKey ca))

     

       177
       177
       +
             (writeCFSSL kubelet)

     

       178
       178
       +
             (writeCFSSL apiserver-client.kubelet)

     

       179
       179
       +
             (writeCFSSL apiserver-client.kube-proxy)

     

       180
       180
       +
             (writeCFSSL etcd-client)

     

       181
       181
       +
           ];

     

       182
       182
       +
         };

     

       224
       183
        
       

     

       225
       225
       -
             ${ln admin_key "$out"}

     

       226
       226
       -
             ${ln admin_csr "$out"}

     

       227
       227
       -
             ${ln admin_cert "$out"}

     

       228
       228
       -
           '';

     

       229
       229
       -
         }

     

       184
       184
       +
         admin = writeCFSSL apiserver-client.admin;

     

       185
       185
       +
       }

+4 -4

nixos/tests/kubernetes/default.nix

···

       1
       1
        
       { system ? builtins.currentSystem }:

     

       2
       2
        
       {

     

       3
       3
       -
           kubernetes-singlenode = import ./singlenode.nix { inherit system; };

     

       4
       4
       -
           kubernetes-multinode-kubectl = import ./multinode-kubectl.nix { inherit system; };

     

       5
       5
       -
           kubernetes-rbac = import ./rbac.nix { inherit system; };

     

       6
       6
       -
           kubernetes-dns = import ./dns.nix { inherit system; };

     

       3
       3
       +
         dns = import ./dns.nix { inherit system; };

     

       4
       4
       +
         # e2e = import ./e2e.nix { inherit system; };  # TODO: make it pass

     

       5
       5
       +
         # the following test(s) can be removed when e2e is working:

     

       6
       6
       +
         rbac = import ./rbac.nix { inherit system; };

     

       7
       7
        
       }

+80 -56

nixos/tests/kubernetes/dns.nix

···

       1
       1
       -
       { system ? builtins.currentSystem }:

     

       2
       2
       -
       

     

       3
       3
       -
       with import ../../lib/testing.nix { inherit system; };

     

       4
       4
       -
       with import ../../lib/qemu-flags.nix;

     

       5
       5
       -
       with pkgs.lib;

     

       6
       6
       -
       

     

       1
       1
       +
       { system ? builtins.currentSystem, pkgs ? import <nixpkgs> { inherit system; } }:

     

       2
       2
       +
       with import ./base.nix { inherit system; };

     

       7
       3
        
       let

     

       8
       8
       -
         servers.master = "192.168.1.1";

     

       9
       9
       -
         servers.one = "192.168.1.10";

     

       4
       4
       +
         domain = "my.zyx";

     

       10
       5
        
       

     

       11
       11
       -
         certs = import ./certs.nix { inherit servers; };

     

       6
       6
       +
         certs = import ./certs.nix { externalDomain = domain; };

     

       12
       7
        
       

     

       13
       13
       -
         redisPod = pkgs.writeText "redis-master-pod.json" (builtins.toJSON {

     

       8
       8
       +
         redisPod = pkgs.writeText "redis-pod.json" (builtins.toJSON {

     

       14
       9
        
           kind = "Pod";

     

       15
       10
        
           apiVersion = "v1";

     

       16
       11
        
           metadata.name = "redis";

     
···

       40
       35
        
         redisImage = pkgs.dockerTools.buildImage {

     

       41
       36
        
           name = "redis";

     

       42
       37
        
           tag = "latest";

     

       43
       43
       -
           contents = [ pkgs.redis pkgs.bind.dnsutils pkgs.coreutils pkgs.inetutils pkgs.nmap ];

     

       38
       38
       +
           contents = [ pkgs.redis pkgs.bind.host ];

     

       44
       39
        
           config.Entrypoint = "/bin/redis-server";

     

       45
       40
        
         };

     

       46
       41
        
       

     

       47
       47
       -
         test = ''

     

       48
       48
       -
           $master->waitUntilSucceeds("kubectl get node master.nixos.xyz | grep Ready");

     

       49
       49
       -
           $master->waitUntilSucceeds("kubectl get node one.nixos.xyz | grep Ready");

     

       50
       50
       -
       

     

       51
       51
       -
           $one->execute("docker load < ${redisImage}");

     

       42
       42
       +
         probePod = pkgs.writeText "probe-pod.json" (builtins.toJSON {

     

       43
       43
       +
           kind = "Pod";

     

       44
       44
       +
           apiVersion = "v1";

     

       45
       45
       +
           metadata.name = "probe";

     

       46
       46
       +
           metadata.labels.name = "probe";

     

       47
       47
       +
           spec.containers = [{

     

       48
       48
       +
             name = "probe";

     

       49
       49
       +
             image = "probe";

     

       50
       50
       +
             args = [ "-f" ];

     

       51
       51
       +
             tty = true;

     

       52
       52
       +
             imagePullPolicy = "Never";

     

       53
       53
       +
           }];

     

       54
       54
       +
         });

     

       52
       55
        
       

     

       53
       53
       -
           $master->waitUntilSucceeds("kubectl create -f ${redisPod} || kubectl apply -f ${redisPod}");

     

       54
       54
       -
           $master->waitUntilSucceeds("kubectl create -f ${redisService} || kubectl apply -f ${redisService}");

     

       56
       56
       +
         probeImage = pkgs.dockerTools.buildImage {

     

       57
       57
       +
           name = "probe";

     

       58
       58
       +
           tag = "latest";

     

       59
       59
       +
           contents = [ pkgs.bind.host pkgs.busybox ];

     

       60
       60
       +
           config.Entrypoint = "/bin/tail";

     

       61
       61
       +
         };

     

       55
       62
        
       

     

       56
       56
       -
           $master->waitUntilSucceeds("kubectl get pod redis | grep Running");

     

       63
       63
       +
         extraConfiguration = { config, pkgs, lib, nodes, ... }: {

     

       64
       64
       +
           environment.systemPackages = [ pkgs.bind.host ];

     

       65
       65
       +
           # virtualisation.docker.extraOptions = "--dns=${config.services.kubernetes.addons.dns.clusterIp}";

     

       66
       66
       +
           services.dnsmasq.enable = true;

     

       67
       67
       +
           services.dnsmasq.servers = [

     

       68
       68
       +
             "/cluster.local/${config.services.kubernetes.addons.dns.clusterIp}#53"

     

       69
       69
       +
           ];

     

       70
       70
       +
         };

     

       57
       71
        
       

     

       58
       58
       -
           $master->succeed("dig \@192.168.1.1 redis.default.svc.cluster.local");

     

       59
       59
       -
           $one->succeed("dig \@192.168.1.10 redis.default.svc.cluster.local");

     

       72
       72
       +
         base = {

     

       73
       73
       +
           name = "dns";

     

       74
       74
       +
           inherit domain certs extraConfiguration;

     

       75
       75
       +
         };

     

       60
       76
        
       

     

       77
       77
       +
         singleNodeTest = {

     

       78
       78
       +
           test = ''

     

       79
       79
       +
             # prepare machine1 for test

     

       80
       80
       +
             $machine1->waitUntilSucceeds("kubectl get node machine1.${domain} | grep -w Ready");

     

       81
       81
       +
             $machine1->execute("docker load < ${redisImage}");

     

       82
       82
       +
             $machine1->waitUntilSucceeds("kubectl create -f ${redisPod}");

     

       83
       83
       +
             $machine1->waitUntilSucceeds("kubectl create -f ${redisService}");

     

       84
       84
       +
             $machine1->execute("docker load < ${probeImage}");

     

       85
       85
       +
             $machine1->waitUntilSucceeds("kubectl create -f ${probePod}");

     

       61
       86
        
       

     

       62
       62
       -
           $master->succeed("kubectl exec -ti redis -- cat /etc/resolv.conf | grep 'nameserver 192.168.1.10'");

     

       87
       87
       +
             # check if pods are running

     

       88
       88
       +
             $machine1->waitUntilSucceeds("kubectl get pod redis | grep Running");

     

       89
       89
       +
             $machine1->waitUntilSucceeds("kubectl get pod probe | grep Running");

     

       90
       90
       +
             $machine1->waitUntilSucceeds("kubectl get pods -n kube-system | grep 'kube-dns.*3/3'");

     

       63
       91
        
       

     

       64
       64
       -
           $master->succeed("kubectl exec -ti redis -- dig \@192.168.1.10 redis.default.svc.cluster.local");

     

       65
       65
       -
         '';

     

       92
       92
       +
             # check dns on host (dnsmasq)

     

       93
       93
       +
             $machine1->succeed("host redis.default.svc.cluster.local");

     

       66
       94
        
       

     

       67
       67
       -
       in makeTest {

     

       68
       68
       -
         name = "kubernetes-dns";

     

       95
       95
       +
             # check dns inside the container

     

       96
       96
       +
             $machine1->succeed("kubectl exec -ti probe -- /bin/host redis.default.svc.cluster.local");

     

       97
       97
       +
           '';

     

       98
       98
       +
         };

     

       69
       99
        
       

     

       70
       70
       -
         nodes = {

     

       71
       71
       -
           master =

     

       72
       72
       -
             { config, pkgs, lib, nodes, ... }:

     

       73
       73
       -
               mkMerge [

     

       74
       74
       -
                 {

     

       75
       75
       -
                   virtualisation.memorySize = 768;

     

       76
       76
       -
                   virtualisation.diskSize = 4096;

     

       77
       77
       -
                   networking.interfaces.eth1.ip4 = mkForce [{address = servers.master; prefixLength = 24;}];

     

       78
       78
       -
                   networking.primaryIPAddress = mkForce servers.master;

     

       79
       79
       -
                 }

     

       80
       80
       -
                 (import ./kubernetes-common.nix { inherit pkgs config certs servers; })

     

       81
       81
       -
                 (import ./kubernetes-master.nix { inherit pkgs config certs; })

     

       82
       82
       -
               ];

     

       100
       100
       +
         multiNodeTest = {

     

       101
       101
       +
           test = ''

     

       102
       102
       +
             # prepare machines for test

     

       103
       103
       +
             $machine1->waitUntilSucceeds("kubectl get node machine1.${domain} | grep -w Ready");

     

       104
       104
       +
             $machine1->waitUntilSucceeds("kubectl get node machine2.${domain} | grep -w Ready");

     

       105
       105
       +
             $machine2->execute("docker load < ${redisImage}");

     

       106
       106
       +
             $machine1->waitUntilSucceeds("kubectl create -f ${redisPod}");

     

       107
       107
       +
             $machine1->waitUntilSucceeds("kubectl create -f ${redisService}");

     

       108
       108
       +
             $machine2->execute("docker load < ${probeImage}");

     

       109
       109
       +
             $machine1->waitUntilSucceeds("kubectl create -f ${probePod}");

     

       83
       110
        
       

     

       84
       84
       -
           one =

     

       85
       85
       -
             { config, pkgs, lib, nodes, ... }:

     

       86
       86
       -
               mkMerge [

     

       87
       87
       -
                 {

     

       88
       88
       -
                   virtualisation.memorySize = 768;

     

       89
       89
       -
                   virtualisation.diskSize = 4096;

     

       90
       90
       -
                   networking.interfaces.eth1.ip4 = mkForce [{address = servers.one; prefixLength = 24;}];

     

       91
       91
       -
                   networking.primaryIPAddress = mkForce servers.one;

     

       92
       92
       -
                   services.kubernetes.roles = ["node"];

     

       93
       93
       -
                 }

     

       94
       94
       -
                 (import ./kubernetes-common.nix { inherit pkgs config certs servers; })

     

       95
       95
       -
               ];

     

       96
       96
       -
         };

     

       111
       111
       +
             # check if pods are running

     

       112
       112
       +
             $machine1->waitUntilSucceeds("kubectl get pod redis | grep Running");

     

       113
       113
       +
             $machine1->waitUntilSucceeds("kubectl get pod probe | grep Running");

     

       114
       114
       +
             $machine1->waitUntilSucceeds("kubectl get pods -n kube-system | grep 'kube-dns.*3/3'");

     

       97
       115
        
       

     

       98
       98
       -
         testScript = ''

     

       99
       99
       -
           startAll;

     

       116
       116
       +
             # check dns on hosts (dnsmasq)

     

       117
       117
       +
             $machine1->succeed("host redis.default.svc.cluster.local");

     

       118
       118
       +
             $machine2->succeed("host redis.default.svc.cluster.local");

     

       100
       119
        
       

     

       101
       101
       -
           ${test}

     

       102
       102
       -
         '';

     

       120
       120
       +
             # check dns inside the container

     

       121
       121
       +
             $machine1->succeed("kubectl exec -ti probe -- /bin/host redis.default.svc.cluster.local");

     

       122
       122
       +
           '';

     

       123
       123
       +
         };

     

       124
       124
       +
       in {

     

       125
       125
       +
         singlenode = mkKubernetesSingleNodeTest (base // singleNodeTest);

     

       126
       126
       +
         multinode = mkKubernetesMultiNodeTest (base // multiNodeTest);

     

       103
       127
        
       }

+40

nixos/tests/kubernetes/e2e.nix

···

       1
       1
       +
       { system ? builtins.currentSystem, pkgs ? import <nixpkgs> { inherit system; } }:

     

       2
       2
       +
       with import ./base.nix { inherit system; };

     

       3
       3
       +
       let

     

       4
       4
       +
         domain = "my.zyx";

     

       5
       5
       +
         certs = import ./certs.nix { externalDomain = domain; };

     

       6
       6
       +
         kubeconfig = pkgs.writeText "kubeconfig.json" (builtins.toJSON {

     

       7
       7
       +
           apiVersion = "v1";

     

       8
       8
       +
           kind = "Config";

     

       9
       9
       +
           clusters = [{

     

       10
       10
       +
             name = "local";

     

       11
       11
       +
             cluster.certificate-authority = "${certs.master}/ca.pem";

     

       12
       12
       +
             cluster.server = "https://api.${domain}";

     

       13
       13
       +
           }];

     

       14
       14
       +
           users = [{

     

       15
       15
       +
             name = "kubelet";

     

       16
       16
       +
             user = {

     

       17
       17
       +
               client-certificate = "${certs.admin}/admin.pem";

     

       18
       18
       +
               client-key = "${certs.admin}/admin-key.pem";

     

       19
       19
       +
             };

     

       20
       20
       +
           }];

     

       21
       21
       +
           contexts = [{

     

       22
       22
       +
             context = {

     

       23
       23
       +
               cluster = "local";

     

       24
       24
       +
               user = "kubelet";

     

       25
       25
       +
             };

     

       26
       26
       +
             current-context = "kubelet-context";

     

       27
       27
       +
           }];

     

       28
       28
       +
         });

     

       29
       29
       +
       

     

       30
       30
       +
         base = {

     

       31
       31
       +
           name = "e2e";

     

       32
       32
       +
           inherit domain certs;

     

       33
       33
       +
           test = ''

     

       34
       34
       +
             $machine1->succeed("e2e.test -kubeconfig ${kubeconfig} -provider local -ginkgo.focus '\\[Conformance\\]' -ginkgo.skip '\\[Flaky\\]|\\[Serial\\]'");

     

       35
       35
       +
           '';

     

       36
       36
       +
         };

     

       37
       37
       +
       in {

     

       38
       38
       +
         singlenode = mkKubernetesSingleNodeTest base;

     

       39
       39
       +
         multinode = mkKubernetesMultiNodeTest base;

     

       40
       40
       +
       }

+49 -62

nixos/tests/kubernetes/kubernetes-common.nix

···

       1
       1
       -
       { config, pkgs, certs, servers }:

     

       2
       2
       -
       

     

       1
       1
       +
       { roles, config, pkgs, certs }:

     

       2
       2
       +
       with pkgs.lib;

     

       3
       3
        
       let

     

       4
       4
       -
         etcd_key = "${certs}/etcd-key.pem";

     

       5
       5
       -
         etcd_cert = "${certs}/etcd.pem";

     

       6
       6
       -
         ca_pem = "${certs}/ca.pem";

     

       7
       7
       -
         etcd_client_cert = "${certs}/etcd-client.crt";

     

       8
       8
       -
         etcd_client_key = "${certs}/etcd-client-key.pem";

     

       9
       9
       -
       

     

       10
       10
       -
         worker_key = "${certs}/worker-key.pem";

     

       11
       11
       -
         worker_cert = "${certs}/worker.pem";

     

       12
       12
       -
       

     

       13
       13
       -
         rootCaFile = pkgs.writeScript "rootCaFile.pem" ''

     

       14
       14
       -
           ${pkgs.lib.readFile "${certs}/ca.pem"}

     

       15
       15
       -
       

     

       16
       16
       -
           ${pkgs.lib.readFile ("${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt")}

     

       17
       17
       -
         '';

     

       18
       18
       -
       

     

       19
       19
       -
         mkHosts =

     

       20
       20
       -
           pkgs.lib.concatMapStringsSep "\n" (v: "${v.ip} ${v.name}.nixos.xyz") (pkgs.lib.mapAttrsToList (n: v: {name = n; ip = v;}) servers);

     

       21
       21
       -
       

     

       22
       22
       -
       in

     

       23
       23
       -
       {

     

       24
       24
       -
         programs.bash.enableCompletion = true;

     

       25
       25
       -
         environment.systemPackages = with pkgs; [ netcat bind etcd.bin ];

     

       4
       4
       +
         base = {

     

       5
       5
       +
           inherit roles;

     

       6
       6
       +
           featureGates = ["AllAlpha"];

     

       7
       7
       +
           flannel.enable = true;

     

       8
       8
       +
           addons.dashboard.enable = true;

     

       9
       9
       +
           verbose = true;

     

       26
       10
        
       

     

       27
       27
       -
         networking = {

     

       28
       28
       -
           firewall.allowedTCPPorts = [

     

       29
       29
       -
             10250 # kubelet

     

       30
       30
       -
           ];

     

       31
       31
       -
           extraHosts = ''

     

       32
       32
       -
             # register "external" domains

     

       33
       33
       -
             ${servers.master} etcd.kubernetes.nixos.xyz

     

       34
       34
       -
             ${servers.master} kubernetes.nixos.xyz

     

       35
       35
       -
             ${mkHosts}

     

       36
       36
       -
           '';

     

       37
       37
       -
         };

     

       38
       38
       -
         services.flannel.iface = "eth1";

     

       39
       39
       -
         environment.variables = {

     

       40
       40
       -
           ETCDCTL_CERT_FILE = "${etcd_client_cert}";

     

       41
       41
       -
           ETCDCTL_KEY_FILE = "${etcd_client_key}";

     

       42
       42
       -
           ETCDCTL_CA_FILE = "${rootCaFile}";

     

       43
       43
       -
           ETCDCTL_PEERS = "https://etcd.kubernetes.nixos.xyz:2379";

     

       44
       44
       -
         };

     

       45
       45
       -
       

     

       46
       46
       -
         services.kubernetes = {

     

       47
       47
       -
           kubelet = {

     

       48
       48
       -
             tlsKeyFile = worker_key;

     

       49
       49
       -
             tlsCertFile = worker_cert;

     

       50
       50
       -
             hostname = "${config.networking.hostName}.nixos.xyz";

     

       51
       51
       -
             nodeIp = config.networking.primaryIPAddress;

     

       11
       11
       +
           caFile = "${certs.master}/ca.pem";

     

       12
       12
       +
           apiserver = {

     

       13
       13
       +
             tlsCertFile = "${certs.master}/kube-apiserver.pem";

     

       14
       14
       +
             tlsKeyFile = "${certs.master}/kube-apiserver-key.pem";

     

       15
       15
       +
             kubeletClientCertFile = "${certs.master}/kubelet-client.pem";

     

       16
       16
       +
             kubeletClientKeyFile = "${certs.master}/kubelet-client-key.pem";

     

       17
       17
       +
             serviceAccountKeyFile = "${certs.master}/kube-service-accounts.pem";

     

       52
       18
        
           };

     

       53
       19
        
           etcd = {

     

       54
       54
       -
             servers = ["https://etcd.kubernetes.nixos.xyz:2379"];

     

       55
       55
       -
             keyFile = etcd_client_key;

     

       56
       56
       -
             certFile = etcd_client_cert;

     

       57
       57
       -
             caFile = ca_pem;

     

       20
       20
       +
             servers = ["https://etcd.${config.networking.domain}:2379"];

     

       21
       21
       +
             certFile = "${certs.worker}/etcd-client.pem";

     

       22
       22
       +
             keyFile = "${certs.worker}/etcd-client-key.pem";

     

       58
       23
        
           };

     

       59
       24
        
           kubeconfig = {

     

       60
       60
       -
             server = "https://kubernetes.nixos.xyz";

     

       61
       61
       -
             caFile = rootCaFile;

     

       62
       62
       -
             certFile = worker_cert;

     

       63
       63
       -
             keyFile = worker_key;

     

       25
       25
       +
             server = "https://api.${config.networking.domain}";

     

       26
       26
       +
           };

     

       27
       27
       +
           kubelet = {

     

       28
       28
       +
             tlsCertFile = "${certs.worker}/kubelet.pem";

     

       29
       29
       +
             tlsKeyFile = "${certs.worker}/kubelet-key.pem";

     

       30
       30
       +
             hostname = "${config.networking.hostName}.${config.networking.domain}";

     

       31
       31
       +
             kubeconfig = {

     

       32
       32
       +
               certFile = "${certs.worker}/apiserver-client-kubelet.pem";

     

       33
       33
       +
               keyFile = "${certs.worker}/apiserver-client-kubelet-key.pem";

     

       34
       34
       +
             };

     

       35
       35
       +
           };

     

       36
       36
       +
           controllerManager = {

     

       37
       37
       +
             serviceAccountKeyFile = "${certs.master}/kube-service-accounts-key.pem";

     

       38
       38
       +
             kubeconfig = {

     

       39
       39
       +
               certFile = "${certs.master}/apiserver-client-kube-controller-manager.pem";

     

       40
       40
       +
               keyFile = "${certs.master}/apiserver-client-kube-controller-manager-key.pem";

     

       41
       41
       +
             };

     

       42
       42
       +
           };

     

       43
       43
       +
           scheduler = {

     

       44
       44
       +
             kubeconfig = {

     

       45
       45
       +
               certFile = "${certs.master}/apiserver-client-kube-scheduler.pem";

     

       46
       46
       +
               keyFile = "${certs.master}/apiserver-client-kube-scheduler-key.pem";

     

       47
       47
       +
             };

     

       48
       48
       +
           };

     

       49
       49
       +
           proxy = {

     

       50
       50
       +
             kubeconfig = {

     

       51
       51
       +
               certFile = "${certs.worker}/apiserver-client-kube-proxy.pem";

     

       52
       52
       +
               keyFile = "${certs.worker}//apiserver-client-kube-proxy-key.pem";

     

       53
       53
       +
             };

     

       64
       54
        
           };

     

       65
       65
       -
           flannel.enable = true;

     

       66
       66
       -
       

     

       67
       67
       -
           dns.port = 4453;

     

       68
       55
        
         };

     

       69
       56
        
       

     

       70
       70
       -
         services.dnsmasq.enable = true;

     

       71
       71
       -
         services.dnsmasq.servers = ["/${config.services.kubernetes.dns.domain}/127.0.0.1#4453"];

     

       57
       57
       +
       in {

     

       58
       58
       +
         services.kubernetes = base;

     

       72
       59
        
       }

-63

nixos/tests/kubernetes/kubernetes-master.nix

···

       1
       1
       -
       { config, pkgs, certs }:

     

       2
       2
       -
       let

     

       3
       3
       -
         etcd_key = "${certs}/etcd-key.pem";

     

       4
       4
       -
         etcd_cert = "${certs}/etcd.pem";

     

       5
       5
       -
         ca_pem = "${certs}/ca.pem";

     

       6
       6
       -
         etcd_client_cert = "${certs}/etcd-client.crt";

     

       7
       7
       -
         etcd_client_key = "${certs}/etcd-client-key.pem";

     

       8
       8
       -
       

     

       9
       9
       -
         apiserver_key = "${certs}/apiserver-key.pem";

     

       10
       10
       -
         apiserver_cert = "${certs}/apiserver.pem";

     

       11
       11
       -
         worker_key = "${certs}/worker-key.pem";

     

       12
       12
       -
         worker_cert = "${certs}/worker.pem";

     

       13
       13
       -
       

     

       14
       14
       -
       

     

       15
       15
       -
         rootCaFile = pkgs.writeScript "rootCaFile.pem" ''

     

       16
       16
       -
           ${pkgs.lib.readFile "${certs}/ca.pem"}

     

       17
       17
       -
       

     

       18
       18
       -
           ${pkgs.lib.readFile ("${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt")}

     

       19
       19
       -
         '';

     

       20
       20
       -
       in

     

       21
       21
       -
       {

     

       22
       22
       -
         networking = {

     

       23
       23
       -
           firewall = {

     

       24
       24
       -
             enable = true;

     

       25
       25
       -
             allowPing = true;

     

       26
       26
       -
             allowedTCPPorts = [

     

       27
       27
       -
               2379 2380  # etcd

     

       28
       28
       -
               443 # kubernetes apiserver

     

       29
       29
       -
             ];

     

       30
       30
       -
           };

     

       31
       31
       -
         };

     

       32
       32
       -
       

     

       33
       33
       -
         services.etcd = {

     

       34
       34
       -
           enable = pkgs.lib.mkForce true;

     

       35
       35
       -
           keyFile = etcd_key;

     

       36
       36
       -
           certFile = etcd_cert;

     

       37
       37
       -
           trustedCaFile = rootCaFile;

     

       38
       38
       -
           peerClientCertAuth = true;

     

       39
       39
       -
           listenClientUrls = ["https://0.0.0.0:2379"];

     

       40
       40
       -
           listenPeerUrls = ["https://0.0.0.0:2380"];

     

       41
       41
       -
       

     

       42
       42
       -
           advertiseClientUrls = ["https://etcd.kubernetes.nixos.xyz:2379"];

     

       43
       43
       -
           initialCluster = ["master=https://etcd.kubernetes.nixos.xyz:2380"];

     

       44
       44
       -
           initialAdvertisePeerUrls = ["https://etcd.kubernetes.nixos.xyz:2380"];

     

       45
       45
       -
         };

     

       46
       46
       -
       

     

       47
       47
       -
         services.kubernetes = {

     

       48
       48
       -
           roles = ["master"];

     

       49
       49
       -
           scheduler.leaderElect = true;

     

       50
       50
       -
           controllerManager.rootCaFile = rootCaFile;

     

       51
       51
       -
           controllerManager.serviceAccountKeyFile = apiserver_key;

     

       52
       52
       -
           apiserver = {

     

       53
       53
       -
             publicAddress = "192.168.1.1";

     

       54
       54
       -
             advertiseAddress = "192.168.1.1";

     

       55
       55
       -
             tlsKeyFile = apiserver_key;

     

       56
       56
       -
             tlsCertFile = apiserver_cert;

     

       57
       57
       -
             clientCaFile = rootCaFile;

     

       58
       58
       -
             kubeletClientCaFile = rootCaFile;

     

       59
       59
       -
             kubeletClientKeyFile = worker_key;

     

       60
       60
       -
             kubeletClientCertFile = worker_cert;

     

       61
       61
       -
           };

     

       62
       62
       -
         };

     

       63
       63
       -
       }

-123

nixos/tests/kubernetes/multinode-kubectl.nix

···

       1
       1
       -
       { system ? builtins.currentSystem }:

     

       2
       2
       -
       

     

       3
       3
       -
       with import ../../lib/testing.nix { inherit system; };

     

       4
       4
       -
       with import ../../lib/qemu-flags.nix;

     

       5
       5
       -
       with pkgs.lib;

     

       6
       6
       -
       

     

       7
       7
       -
       let

     

       8
       8
       -
         servers.master = "192.168.1.1";

     

       9
       9
       -
         servers.one = "192.168.1.10";

     

       10
       10
       -
         servers.two = "192.168.1.20";

     

       11
       11
       -
       

     

       12
       12
       -
         certs = import ./certs.nix { inherit servers; };

     

       13
       13
       -
       

     

       14
       14
       -
         kubectlPod = pkgs.writeText "kubectl-pod.json" (builtins.toJSON {

     

       15
       15
       -
           kind = "Pod";

     

       16
       16
       -
           apiVersion = "v1";

     

       17
       17
       -
           metadata.name = "kubectl";

     

       18
       18
       -
           metadata.labels.name = "kubectl";

     

       19
       19
       -
           spec.containers = [{

     

       20
       20
       -
             name = "kubectl";

     

       21
       21
       -
             image = "kubectl:latest";

     

       22
       22
       -
             command = ["${pkgs.busybox}/bin/tail" "-f"];

     

       23
       23
       -
             imagePullPolicy = "Never";

     

       24
       24
       -
             tty = true;

     

       25
       25
       -
           }];

     

       26
       26
       -
         });

     

       27
       27
       -
       

     

       28
       28
       -
         kubectlImage = pkgs.dockerTools.buildImage {

     

       29
       29
       -
           name = "kubectl";

     

       30
       30
       -
           tag = "latest";

     

       31
       31
       -
           contents = [ pkgs.kubernetes pkgs.busybox certs kubeconfig ];

     

       32
       32
       -
           config.Entrypoint = "${pkgs.busybox}/bin/sh";

     

       33
       33
       -
         };

     

       34
       34
       -
       

     

       35
       35
       -
         kubeconfig = pkgs.writeTextDir "kubeconfig.json" (builtins.toJSON {

     

       36
       36
       -
           apiVersion = "v1";

     

       37
       37
       -
           kind = "Config";

     

       38
       38
       -
           clusters = [{

     

       39
       39
       -
             name = "local";

     

       40
       40
       -
             cluster.certificate-authority = "/ca.pem";

     

       41
       41
       -
             cluster.server = "https://${servers.master}";

     

       42
       42
       -
           }];

     

       43
       43
       -
           users = [{

     

       44
       44
       -
             name = "kubelet";

     

       45
       45
       -
             user = {

     

       46
       46
       -
               client-certificate = "/admin.crt";

     

       47
       47
       -
               client-key = "/admin-key.pem";

     

       48
       48
       -
             };

     

       49
       49
       -
           }];

     

       50
       50
       -
           contexts = [{

     

       51
       51
       -
             context = {

     

       52
       52
       -
               cluster = "local";

     

       53
       53
       -
               user = "kubelet";

     

       54
       54
       -
             };

     

       55
       55
       -
             current-context = "kubelet-context";

     

       56
       56
       -
           }];

     

       57
       57
       -
         });

     

       58
       58
       -
       

     

       59
       59
       -
         test = ''

     

       60
       60
       -
           $master->waitUntilSucceeds("kubectl get node master.nixos.xyz | grep Ready");

     

       61
       61
       -
           $master->waitUntilSucceeds("kubectl get node one.nixos.xyz | grep Ready");

     

       62
       62
       -
           $master->waitUntilSucceeds("kubectl get node two.nixos.xyz | grep Ready");

     

       63
       63
       -
       

     

       64
       64
       -
           $one->execute("docker load < ${kubectlImage}");

     

       65
       65
       -
           $two->execute("docker load < ${kubectlImage}");

     

       66
       66
       -
       

     

       67
       67
       -
           $master->waitUntilSucceeds("kubectl create -f ${kubectlPod} || kubectl apply -f ${kubectlPod}");

     

       68
       68
       -
       

     

       69
       69
       -
           $master->waitUntilSucceeds("kubectl get pod kubectl | grep Running");

     

       70
       70
       -
       

     

       71
       71
       -
           $master->succeed("kubectl exec -ti kubectl -- kubectl --kubeconfig=/kubeconfig.json version");

     

       72
       72
       -
         '';

     

       73
       73
       -
       

     

       74
       74
       -
       in makeTest {

     

       75
       75
       -
         name = "kubernetes-multinode-kubectl";

     

       76
       76
       -
       

     

       77
       77
       -
         nodes = {

     

       78
       78
       -
           master =

     

       79
       79
       -
             { config, pkgs, lib, nodes, ... }:

     

       80
       80
       -
               mkMerge [

     

       81
       81
       -
                 {

     

       82
       82
       -
                   virtualisation.memorySize = 768;

     

       83
       83
       -
                   virtualisation.diskSize = 4096;

     

       84
       84
       -
                   networking.interfaces.eth1.ip4 = mkForce [{address = servers.master; prefixLength = 24;}];

     

       85
       85
       -
                   networking.primaryIPAddress = mkForce servers.master;

     

       86
       86
       -
                 }

     

       87
       87
       -
                 (import ./kubernetes-common.nix { inherit pkgs config certs servers; })

     

       88
       88
       -
                 (import ./kubernetes-master.nix { inherit pkgs config certs; })

     

       89
       89
       -
               ];

     

       90
       90
       -
       

     

       91
       91
       -
           one =

     

       92
       92
       -
             { config, pkgs, lib, nodes, ... }:

     

       93
       93
       -
               mkMerge [

     

       94
       94
       -
                 {

     

       95
       95
       -
                   virtualisation.memorySize = 768;

     

       96
       96
       -
                   virtualisation.diskSize = 4096;

     

       97
       97
       -
                   networking.interfaces.eth1.ip4 = mkForce [{address = servers.one; prefixLength = 24;}];

     

       98
       98
       -
                   networking.primaryIPAddress = mkForce servers.one;

     

       99
       99
       -
                   services.kubernetes.roles = ["node"];

     

       100
       100
       -
                 }

     

       101
       101
       -
                 (import ./kubernetes-common.nix { inherit pkgs config certs servers; })

     

       102
       102
       -
               ];

     

       103
       103
       -
       

     

       104
       104
       -
           two =

     

       105
       105
       -
             { config, pkgs, lib, nodes, ... }:

     

       106
       106
       -
               mkMerge [

     

       107
       107
       -
                 {

     

       108
       108
       -
                   virtualisation.memorySize = 768;

     

       109
       109
       -
                   virtualisation.diskSize = 4096;

     

       110
       110
       -
                   networking.interfaces.eth1.ip4 = mkForce [{address = servers.two; prefixLength = 24;}];

     

       111
       111
       -
                   networking.primaryIPAddress = mkForce servers.two;

     

       112
       112
       -
                   services.kubernetes.roles = ["node"];

     

       113
       113
       -
                 }

     

       114
       114
       -
                 (import ./kubernetes-common.nix { inherit pkgs config certs servers; })

     

       115
       115
       -
               ];

     

       116
       116
       -
         };

     

       117
       117
       -
       

     

       118
       118
       -
         testScript = ''

     

       119
       119
       -
           startAll;

     

       120
       120
       -
       

     

       121
       121
       -
           ${test}

     

       122
       122
       -
         '';

     

       123
       123
       -
       }

+61 -72

nixos/tests/kubernetes/rbac.nix

···

       1
       1
       -
       { system ? builtins.currentSystem }:

     

       2
       2
       -
       

     

       3
       3
       -
       with import ../../lib/testing.nix { inherit system; };

     

       4
       4
       -
       with import ../../lib/qemu-flags.nix;

     

       5
       5
       -
       with pkgs.lib;

     

       6
       6
       -
       

     

       1
       1
       +
       { system ? builtins.currentSystem, pkgs ? import <nixpkgs> { inherit system; } }:

     

       2
       2
       +
       with import ./base.nix { inherit system; };

     

       7
       3
        
       let

     

       8
       8
       -
         servers.master = "192.168.1.1";

     

       9
       9
       -
         servers.one = "192.168.1.10";

     

       10
       10
       -
       

     

       11
       11
       -
         certs = import ./certs.nix { inherit servers; };

     

       12
       4
        
       

     

       13
       5
        
         roServiceAccount = pkgs.writeText "ro-service-account.json" (builtins.toJSON {

     

       14
       6
        
           kind = "ServiceAccount";

     
···

       20
       12
        
         });

     

       21
       13
        
       

     

       22
       14
        
         roRoleBinding = pkgs.writeText "ro-role-binding.json" (builtins.toJSON {

     

       23
       23
       -
           "apiVersion" = "rbac.authorization.k8s.io/v1beta1";

     

       24
       24
       -
           "kind" = "RoleBinding";

     

       25
       25
       -
           "metadata" = {

     

       26
       26
       -
             "name" = "read-pods";

     

       27
       27
       -
             "namespace" = "default";

     

       15
       15
       +
           apiVersion = "rbac.authorization.k8s.io/v1beta1";

     

       16
       16
       +
           kind = "RoleBinding";

     

       17
       17
       +
           metadata = {

     

       18
       18
       +
             name = "read-pods";

     

       19
       19
       +
             namespace = "default";

     

       28
       20
        
           };

     

       29
       29
       -
           "roleRef" = {

     

       30
       30
       -
             "apiGroup" = "rbac.authorization.k8s.io";

     

       31
       31
       -
             "kind" = "Role";

     

       32
       32
       -
             "name" = "pod-reader";

     

       21
       21
       +
           roleRef = {

     

       22
       22
       +
             apiGroup = "rbac.authorization.k8s.io";

     

       23
       23
       +
             kind = "Role";

     

       24
       24
       +
             name = "pod-reader";

     

       33
       25
        
           };

     

       34
       34
       -
           "subjects" = [{

     

       35
       35
       -
             "kind" = "ServiceAccount";

     

       36
       36
       -
             "name" = "read-only";

     

       37
       37
       -
             "namespace" = "default";

     

       26
       26
       +
           subjects = [{

     

       27
       27
       +
             kind = "ServiceAccount";

     

       28
       28
       +
             name = "read-only";

     

       29
       29
       +
             namespace = "default";

     

       38
       30
        
           }];

     

       39
       31
        
         });

     

       40
       32
        
       

     
···

       62
       54
        
           spec.containers = [{

     

       63
       55
        
             name = "kubectl";

     

       64
       56
        
             image = "kubectl:latest";

     

       65
       65
       -
             command = ["${pkgs.busybox}/bin/tail" "-f"];

     

       57
       57
       +
             command = ["/bin/tail" "-f"];

     

       66
       58
        
             imagePullPolicy = "Never";

     

       67
       59
        
             tty = true;

     

       68
       60
        
           }];

     
···

       78
       70
        
           spec.containers = [{

     

       79
       71
        
             name = "kubectl-2";

     

       80
       72
        
             image = "kubectl:latest";

     

       81
       81
       -
             command = ["${pkgs.busybox}/bin/tail" "-f"];

     

       73
       73
       +
             command = ["/bin/tail" "-f"];

     

       82
       74
        
             imagePullPolicy = "Never";

     

       83
       75
        
             tty = true;

     

       84
       76
        
           }];

     

       85
       77
        
         });

     

       86
       78
        
       

     

       79
       79
       +
         kubectl = pkgs.runCommand "copy-kubectl" { buildInputs = [ pkgs.kubernetes ]; } ''

     

       80
       80
       +
           mkdir -p $out/bin

     

       81
       81
       +
           cp ${pkgs.kubernetes}/bin/kubectl $out/bin/kubectl

     

       82
       82
       +
         '';

     

       83
       83
       +
       

     

       87
       84
        
         kubectlImage = pkgs.dockerTools.buildImage {

     

       88
       85
        
           name = "kubectl";

     

       89
       86
        
           tag = "latest";

     

       90
       90
       -
           contents = [ pkgs.kubernetes pkgs.busybox kubectlPod2 ];  # certs kubeconfig

     

       91
       91
       -
           config.Entrypoint = "${pkgs.busybox}/bin/sh";

     

       87
       87
       +
           contents = [ kubectl pkgs.busybox kubectlPod2 ];

     

       88
       88
       +
           config.Entrypoint = "/bin/sh";

     

       89
       89
       +
         };

     

       90
       90
       +
       

     

       91
       91
       +
         base = {

     

       92
       92
       +
           name = "rbac";

     

       92
       93
        
         };

     

       93
       94
        
       

     

       94
       94
       -
         test = ''

     

       95
       95
       -
           $master->waitUntilSucceeds("kubectl get node master.nixos.xyz | grep Ready");

     

       96
       96
       -
           $master->waitUntilSucceeds("kubectl get node one.nixos.xyz | grep Ready");

     

       95
       95
       +
         singlenode = base // {

     

       96
       96
       +
           test = ''

     

       97
       97
       +
             $machine1->waitUntilSucceeds("kubectl get node machine1.my.zyx | grep -w Ready");

     

       98
       98
       +
       

     

       99
       99
       +
             $machine1->execute("docker load < ${kubectlImage}");

     

       100
       100
       +
       

     

       101
       101
       +
             $machine1->waitUntilSucceeds("kubectl apply -f ${roServiceAccount}");

     

       102
       102
       +
             $machine1->waitUntilSucceeds("kubectl apply -f ${roRole}");

     

       103
       103
       +
             $machine1->waitUntilSucceeds("kubectl apply -f ${roRoleBinding}");

     

       104
       104
       +
             $machine1->waitUntilSucceeds("kubectl create -f ${kubectlPod}");

     

       97
       105
        
       

     

       98
       98
       -
           $one->execute("docker load < ${kubectlImage}");

     

       106
       106
       +
             $machine1->waitUntilSucceeds("kubectl get pod kubectl | grep Running");

     

       99
       107
        
       

     

       100
       100
       -
           $master->waitUntilSucceeds("kubectl apply -f ${roServiceAccount}");

     

       101
       101
       -
           $master->waitUntilSucceeds("kubectl apply -f ${roRole}");

     

       102
       102
       -
           $master->waitUntilSucceeds("kubectl apply -f ${roRoleBinding}");

     

       103
       103
       -
           $master->waitUntilSucceeds("kubectl create -f ${kubectlPod} || kubectl apply -f ${kubectlPod}");

     

       108
       108
       +
             $machine1->succeed("kubectl exec -ti kubectl -- kubectl get pods");

     

       109
       109
       +
             $machine1->fail("kubectl exec -ti kubectl -- kubectl create -f /kubectl-pod-2.json");

     

       110
       110
       +
             $machine1->fail("kubectl exec -ti kubectl -- kubectl delete pods -l name=kubectl");

     

       111
       111
       +
           '';

     

       112
       112
       +
         };

     

       104
       113
        
       

     

       105
       105
       -
           $master->waitUntilSucceeds("kubectl get pod kubectl | grep Running");

     

       114
       114
       +
         multinode = base // {

     

       115
       115
       +
           test = ''

     

       116
       116
       +
             $machine1->waitUntilSucceeds("kubectl get node machine1.my.zyx | grep -w Ready");

     

       117
       117
       +
             $machine1->waitUntilSucceeds("kubectl get node machine2.my.zyx | grep -w Ready");

     

       106
       118
        
       

     

       107
       107
       -
           $master->succeed("kubectl exec -ti kubectl -- kubectl get pods");

     

       108
       108
       -
           $master->fail("kubectl exec -ti kubectl -- kubectl create -f /kubectl-pod-2.json");

     

       109
       109
       -
           $master->fail("kubectl exec -ti kubectl -- kubectl delete pods -l name=kubectl");

     

       110
       110
       -
         '';

     

       119
       119
       +
             $machine2->execute("docker load < ${kubectlImage}");

     

       111
       120
        
       

     

       112
       112
       -
       in makeTest {

     

       113
       113
       -
         name = "kubernetes-rbac";

     

       121
       121
       +
             $machine1->waitUntilSucceeds("kubectl apply -f ${roServiceAccount}");

     

       122
       122
       +
             $machine1->waitUntilSucceeds("kubectl apply -f ${roRole}");

     

       123
       123
       +
             $machine1->waitUntilSucceeds("kubectl apply -f ${roRoleBinding}");

     

       124
       124
       +
             $machine1->waitUntilSucceeds("kubectl create -f ${kubectlPod}");

     

       114
       125
        
       

     

       115
       115
       -
         nodes = {

     

       116
       116
       -
           master =

     

       117
       117
       -
             { config, pkgs, lib, nodes, ... }:

     

       118
       118
       -
               mkMerge [

     

       119
       119
       -
                 {

     

       120
       120
       -
                   virtualisation.memorySize = 768;

     

       121
       121
       -
                   virtualisation.diskSize = 4096;

     

       122
       122
       -
                   networking.interfaces.eth1.ip4 = mkForce [{address = servers.master; prefixLength = 24;}];

     

       123
       123
       -
                   networking.primaryIPAddress = mkForce servers.master;

     

       124
       124
       -
                 }

     

       125
       125
       -
                 (import ./kubernetes-common.nix { inherit pkgs config certs servers; })

     

       126
       126
       -
                 (import ./kubernetes-master.nix { inherit pkgs config certs; })

     

       127
       127
       -
               ];

     

       126
       126
       +
             $machine1->waitUntilSucceeds("kubectl get pod kubectl | grep Running");

     

       128
       127
        
       

     

       129
       129
       -
           one =

     

       130
       130
       -
             { config, pkgs, lib, nodes, ... }:

     

       131
       131
       -
               mkMerge [

     

       132
       132
       -
                 {

     

       133
       133
       -
                   virtualisation.memorySize = 768;

     

       134
       134
       -
                   virtualisation.diskSize = 4096;

     

       135
       135
       -
                   networking.interfaces.eth1.ip4 = mkForce [{address = servers.one; prefixLength = 24;}];

     

       136
       136
       -
                   networking.primaryIPAddress = mkForce servers.one;

     

       137
       137
       -
                   services.kubernetes.roles = ["node"];

     

       138
       138
       -
                 }

     

       139
       139
       -
                 (import ./kubernetes-common.nix { inherit pkgs config certs servers; })

     

       140
       140
       -
               ];

     

       128
       128
       +
             $machine1->succeed("kubectl exec -ti kubectl -- kubectl get pods");

     

       129
       129
       +
             $machine1->fail("kubectl exec -ti kubectl -- kubectl create -f /kubectl-pod-2.json");

     

       130
       130
       +
             $machine1->fail("kubectl exec -ti kubectl -- kubectl delete pods -l name=kubectl");

     

       131
       131
       +
           '';

     

       141
       132
        
         };

     

       142
       133
        
       

     

       143
       143
       -
         testScript = ''

     

       144
       144
       -
           startAll;

     

       145
       145
       -
       

     

       146
       146
       -
           ${test}

     

       147
       147
       -
         '';

     

       134
       134
       +
       in {

     

       135
       135
       +
         singlenode = mkKubernetesSingleNodeTest singlenode;

     

       136
       136
       +
         multinode = mkKubernetesMultiNodeTest multinode;

     

       148
       137
        
       }

-75

nixos/tests/kubernetes/singlenode.nix

···

       1
       1
       -
       { system ? builtins.currentSystem }:

     

       2
       2
       -
       

     

       3
       3
       -
       with import ../../lib/testing.nix { inherit system; };

     

       4
       4
       -
       with import ../../lib/qemu-flags.nix;

     

       5
       5
       -
       with pkgs.lib;

     

       6
       6
       -
       

     

       7
       7
       -
       let

     

       8
       8
       -
         redisPod = pkgs.writeText "redis-master-pod.json" (builtins.toJSON {

     

       9
       9
       -
           kind = "Pod";

     

       10
       10
       -
           apiVersion = "v1";

     

       11
       11
       -
           metadata.name = "redis";

     

       12
       12
       -
           metadata.labels.name = "redis";

     

       13
       13
       -
           spec.containers = [{

     

       14
       14
       -
             name = "redis";

     

       15
       15
       -
             image = "redis";

     

       16
       16
       -
             args = ["--bind" "0.0.0.0"];

     

       17
       17
       -
             imagePullPolicy = "Never";

     

       18
       18
       -
             ports = [{

     

       19
       19
       -
               name = "redis-server";

     

       20
       20
       -
               containerPort = 6379;

     

       21
       21
       -
             }];

     

       22
       22
       -
           }];

     

       23
       23
       -
         });

     

       24
       24
       -
       

     

       25
       25
       -
         redisService = pkgs.writeText "redis-service.json" (builtins.toJSON {

     

       26
       26
       -
           kind = "Service";

     

       27
       27
       -
           apiVersion = "v1";

     

       28
       28
       -
           metadata.name = "redis";

     

       29
       29
       -
           spec = {

     

       30
       30
       -
             ports = [{port = 6379; targetPort = 6379;}];

     

       31
       31
       -
             selector = {name = "redis";};

     

       32
       32
       -
           };

     

       33
       33
       -
         });

     

       34
       34
       -
       

     

       35
       35
       -
         redisImage = pkgs.dockerTools.buildImage {

     

       36
       36
       -
           name = "redis";

     

       37
       37
       -
           tag = "latest";

     

       38
       38
       -
           contents = pkgs.redis;

     

       39
       39
       -
           config.Entrypoint = "/bin/redis-server";

     

       40
       40
       -
         };

     

       41
       41
       -
       

     

       42
       42
       -
         testSimplePod = ''

     

       43
       43
       -
           $kubernetes->execute("docker load < ${redisImage}");

     

       44
       44
       -
           $kubernetes->waitUntilSucceeds("kubectl create -f ${redisPod}");

     

       45
       45
       -
           $kubernetes->succeed("kubectl create -f ${redisService}");

     

       46
       46
       -
           $kubernetes->waitUntilSucceeds("kubectl get pod redis | grep Running");

     

       47
       47
       -
           $kubernetes->succeed("nc -z \$\(dig redis.default.svc.cluster.local +short\) 6379");

     

       48
       48
       -
         '';

     

       49
       49
       -
       in makeTest {

     

       50
       50
       -
         name = "kubernetes-singlenode";

     

       51
       51
       -
       

     

       52
       52
       -
         nodes = {

     

       53
       53
       -
           kubernetes =

     

       54
       54
       -
             { config, pkgs, lib, nodes, ... }:

     

       55
       55
       -
               {

     

       56
       56
       -
                 virtualisation.memorySize = 768;

     

       57
       57
       -
                 virtualisation.diskSize = 2048;

     

       58
       58
       -
       

     

       59
       59
       -
                 programs.bash.enableCompletion = true;

     

       60
       60
       -
                 environment.systemPackages = with pkgs; [ netcat bind ];

     

       61
       61
       -
       

     

       62
       62
       -
                 services.kubernetes.roles = ["master" "node"];

     

       63
       63
       -
                 services.kubernetes.dns.port = 4453;

     

       64
       64
       -
       

     

       65
       65
       -
                 services.dnsmasq.enable = true;

     

       66
       66
       -
                 services.dnsmasq.servers = ["/${config.services.kubernetes.dns.domain}/127.0.0.1#4453"];

     

       67
       67
       -
               };

     

       68
       68
       -
         };

     

       69
       69
       -
       

     

       70
       70
       -
         testScript = ''

     

       71
       71
       -
           startAll;

     

       72
       72
       -
       

     

       73
       73
       -
           ${testSimplePod}

     

       74
       74
       -
         '';

     

       75
       75
       -
       }