commit cb60a01188fe3579abb1b97a0e51b21bac90087d · pyrox.dev/nixpkgs

+128 -113

nixos/tests/rke2/multi-node.nix

···

       6
        
           ...

     

       7
        
         }:

     

       8
        
         let

     

       9
       -
           pauseImage = pkgs.dockerTools.streamLayeredImage {

     

       10
       -
             name = "test.local/pause";

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       11
        
             tag = "local";

     

       12
       -
             contents = pkgs.buildEnv {

     

       13
       -
               name = "rke2-pause-image-env";

     

       0
        
       
     

       14
        
               paths = with pkgs; [

     

       15
       -
                 tini

     

       16
       -
                 bashInteractive

     

       17
        
                 coreutils

     

       18
        
                 socat

     

       19
        
               ];

     

       20
        
             };

     

       21
       -
             config.Entrypoint = [

     

       22
       -
               "/bin/tini"

     

       23
       -
               "--"

     

       24
       -
               "/bin/sleep"

     

       25
       -
               "inf"

     

       26
       -
             ];

     

       27
        
           };

     

       28
       -
           # A daemonset that responds 'server' on port 8000

     

       29
        
           networkTestDaemonset = pkgs.writeText "test.yml" ''

     

       30
        
             apiVersion: apps/v1

     

       31
        
             kind: DaemonSet

     
···

       44
        
                 spec:

     

       45
        
                   containers:

     

       46
        
                   - name: test

     

       47
       -
                     image: test.local/pause:local

     

       48
        
                     imagePullPolicy: Never

     

       49
        
                     resources:

     

       50
        
                       limits:

     

       51
        
                         memory: 20Mi

     

       52
       -
                     command: ["socat", "TCP4-LISTEN:8000,fork", "EXEC:echo server"]

     

       53
        
           '';

     

       54
        
           tokenFile = pkgs.writeText "token" "p@s$w0rd";

     

       55
       -
           agentTokenFile = pkgs.writeText "agent-token" "p@s$w0rd";

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       56
        
         in

     

       57
        
         {

     

       58
        
           name = "${rke2.name}-multi-node";

     

       59
        
           meta.maintainers = rke2.meta.maintainers;

     

       60
        
       

     

       61
        
           nodes = {

     

       62
       -
             server1 =

     

       63
       -
               { pkgs, ... }:

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       64
        
               {

     

       65
       -
                 networking.firewall.enable = false;

     

       66
       -
                 networking.useDHCP = false;

     

       67
       -
                 networking.defaultGateway = "192.168.1.1";

     

       68
       -
                 networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [

     

       69
       -
                   {

     

       70
       -
                     address = "192.168.1.1";

     

       71
       -
                     prefixLength = 24;

     

       72
       -
                   }

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       73
        
                 ];

     

       74
        
       

     

       75
       -
                 virtualisation.memorySize = 1536;

     

       76
       -
                 virtualisation.diskSize = 4096;

     

       0
        
       
     

       0
        
       
     

       77
        
       

     

       78
        
                 services.rke2 = {

     

       79
        
                   enable = true;

     

       80
        
                   role = "server";

     

       0
        
       
     

       81
        
                   inherit tokenFile;

     

       82
        
                   inherit agentTokenFile;

     

       83
       -
                   nodeName = "${rke2.name}-server1";

     

       84
       -
                   package = rke2;

     

       85
       -
                   nodeIP = "192.168.1.1";

     

       86
        
                   disable = [

     

       87
        
                     "rke2-coredns"

     

       88
        
                     "rke2-metrics-server"

     

       89
        
                     "rke2-ingress-nginx"

     

       90
       -
                   ];

     

       91
       -
                   extraFlags = [

     

       92
       -
                     "--cluster-reset"

     

       93
        
                   ];

     

       94
        
                 };

     

       95
        
               };

     

       96
        
       

     

       97
       -
             server2 =

     

       98
       -
               { pkgs, ... }:

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       99
        
               {

     

       100
       -
                 networking.firewall.enable = false;

     

       101
       -
                 networking.useDHCP = false;

     

       102
       -
                 networking.defaultGateway = "192.168.1.2";

     

       103
       -
                 networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [

     

       104
       -
                   {

     

       105
       -
                     address = "192.168.1.2";

     

       106
       -
                     prefixLength = 24;

     

       107
       -
                   }

     

       108
       -
                 ];

     

       109
       -
       

     

       110
       -
                 virtualisation.memorySize = 1536;

     

       111
       -
                 virtualisation.diskSize = 4096;

     

       112
       -
       

     

       113
       -
                 services.rke2 = {

     

       114
       -
                   enable = true;

     

       115
       -
                   role = "server";

     

       116
       -
                   serverAddr = "https://192.168.1.1:6443";

     

       117
       -
                   inherit tokenFile;

     

       118
       -
                   inherit agentTokenFile;

     

       119
       -
                   nodeName = "${rke2.name}-server2";

     

       120
       -
                   package = rke2;

     

       121
       -
                   nodeIP = "192.168.1.2";

     

       122
       -
                   disable = [

     

       123
       -
                     "rke2-coredns"

     

       124
       -
                     "rke2-metrics-server"

     

       125
       -
                     "rke2-ingress-nginx"

     

       126
       -
                   ];

     

       127
        
                 };

     

       128
       -
               };

     

       129
        
       

     

       130
       -
             agent1 =

     

       131
       -
               { pkgs, ... }:

     

       132
       -
               {

     

       133
       -
                 networking.firewall.enable = false;

     

       134
       -
                 networking.useDHCP = false;

     

       135
       -
                 networking.defaultGateway = "192.168.1.3";

     

       136
       -
                 networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [

     

       137
       -
                   {

     

       138
       -
                     address = "192.168.1.3";

     

       139
       -
                     prefixLength = 24;

     

       140
       -
                   }

     

       141
       -
                 ];

     

       142
        
       

     

       143
       -
                 virtualisation.memorySize = 1536;

     

       144
       -
                 virtualisation.diskSize = 4096;

     

       0
        
       
     

       145
        
       

     

       146
        
                 services.rke2 = {

     

       147
        
                   enable = true;

     

       148
        
                   role = "agent";

     

       149
       -
                   tokenFile = agentTokenFile;

     

       150
       -
                   serverAddr = "https://192.168.1.2:6443";

     

       151
       -
                   nodeName = "${rke2.name}-agent1";

     

       152
        
                   package = rke2;

     

       153
       -
                   nodeIP = "192.168.1.3";

     

       0
        
       
     

       0
        
       
     

       154
        
                 };

     

       155
        
               };

     

       156
        
           };

     
···

       158
        
           testScript =

     

       159
        
             let

     

       160
        
               kubectl = "${pkgs.kubectl}/bin/kubectl --kubeconfig=/etc/rancher/rke2/rke2.yaml";

     

       161
       -
               ctr = "${pkgs.containerd}/bin/ctr -a /run/k3s/containerd/containerd.sock";

     

       162
        
               jq = "${pkgs.jq}/bin/jq";

     

       163
       -
               ping = "${pkgs.iputils}/bin/ping";

     

       164
        
             in

     

       0
        
       
     

       165
        
             ''

     

       166
       -
               machines = [server1, server2, agent1]

     

       167
       -
       

     

       168
       -
               for machine in machines:

     

       169
       -
                   machine.start()

     

       170
       -
                   machine.wait_for_unit("rke2")

     

       171
        
       

     

       172
       -
               # wait for the agent to show up

     

       173
       -
               server1.succeed("${kubectl} get node ${rke2.name}-agent1")

     

       174
        
       

     

       175
       -
               for machine in machines:

     

       176
       -
                   machine.succeed("${pauseImage} | ${ctr} image import -")

     

       177
        
       

     

       178
       -
               server1.succeed("${kubectl} cluster-info")

     

       179
       -
               server1.wait_until_succeeds("${kubectl} get serviceaccount default")

     

       180
        
       

     

       181
        
               # Now create a pod on each node via a daemonset and verify they can talk to each other.

     

       182
       -
               server1.succeed("${kubectl} apply -f ${networkTestDaemonset}")

     

       183
       -
               server1.wait_until_succeeds(

     

       184
        
                   f'[ "$(${kubectl} get ds test -o json | ${jq} .status.numberReady)" -eq {len(machines)} ]'

     

       185
        
               )

     

       186
        
       

     

       187
        
               # Get pod IPs

     

       188
       -
               pods = server1.succeed("${kubectl} get po -o json | ${jq} '.items[].metadata.name' -r").splitlines()

     

       189
        
               pod_ips = [

     

       190
       -
                   server1.succeed(f"${kubectl} get po {n} -o json | ${jq} '.status.podIP' -cr").strip() for n in pods

     

       191
        
               ]

     

       192
        
       

     

       193
       -
               # Verify each server can ping each pod ip

     

       194
        
               for pod_ip in pod_ips:

     

       195
       -
                   server1.succeed(f"${ping} -c 1 {pod_ip}")

     

       196
       -
                   agent1.succeed(f"${ping} -c 1 {pod_ip}")

     

       197
       -
       

     

       198
       -
               # Verify the pods can talk to each other

     

       199
       -
               resp = server1.wait_until_succeeds(f"${kubectl} exec {pods[0]} -- socat TCP:{pod_ips[1]}:8000 -")

     

       200
       -
               assert resp.strip() == "server"

     

       201
       -
               resp = server1.wait_until_succeeds(f"${kubectl} exec {pods[1]} -- socat TCP:{pod_ips[0]}:8000 -")

     

       202
       -
               assert resp.strip() == "server"

     

       203
       -
       

     

       204
       -
               # Cleanup

     

       205
       -
               server1.succeed("${kubectl} delete -f ${networkTestDaemonset}")

     

       206
       -
               for machine in machines:

     

       207
       -
                   machine.shutdown()

     

       208
        
             '';

     

       209
        
         }

     

       210
        
       )

···

       6
        
           ...

     

       7
        
         }:

     

       8
        
         let

     

       9
       +
           throwSystem = throw "RKE2: Unsupported system: ${pkgs.stdenv.hostPlatform.system}";

     

       10
       +
           coreImages =

     

       11
       +
             {

     

       12
       +
               aarch64-linux = rke2.images-core-linux-arm64-tar-zst;

     

       13
       +
               x86_64-linux = rke2.images-core-linux-amd64-tar-zst;

     

       14
       +
             }

     

       15
       +
             .${pkgs.stdenv.hostPlatform.system} or throwSystem;

     

       16
       +
           canalImages =

     

       17
       +
             {

     

       18
       +
               aarch64-linux = rke2.images-canal-linux-arm64-tar-zst;

     

       19
       +
               x86_64-linux = rke2.images-canal-linux-amd64-tar-zst;

     

       20
       +
             }

     

       21
       +
             .${pkgs.stdenv.hostPlatform.system} or throwSystem;

     

       22
       +
           helloImage = pkgs.dockerTools.buildImage {

     

       23
       +
             name = "test.local/hello";

     

       24
        
             tag = "local";

     

       25
       +
             compressor = "zstd";

     

       26
       +
             copyToRoot = pkgs.buildEnv {

     

       27
       +
               name = "rke2-hello-image-env";

     

       28
        
               paths = with pkgs; [

     

       0
        
       
     

       0
        
       
     

       29
        
                 coreutils

     

       30
        
                 socat

     

       31
        
               ];

     

       32
        
             };

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       33
        
           };

     

       34
       +
           # A daemonset that responds 'hello' on port 8000

     

       35
        
           networkTestDaemonset = pkgs.writeText "test.yml" ''

     

       36
        
             apiVersion: apps/v1

     

       37
        
             kind: DaemonSet

     
···

       50
        
                 spec:

     

       51
        
                   containers:

     

       52
        
                   - name: test

     

       53
       +
                     image: test.local/hello:local

     

       54
        
                     imagePullPolicy: Never

     

       55
        
                     resources:

     

       56
        
                       limits:

     

       57
        
                         memory: 20Mi

     

       58
       +
                     command: ["socat", "TCP4-LISTEN:8000,fork", "EXEC:echo hello"]

     

       59
        
           '';

     

       60
        
           tokenFile = pkgs.writeText "token" "p@s$w0rd";

     

       61
       +
           agentTokenFile = pkgs.writeText "agent-token" "agentP@s$w0rd";

     

       62
       +
           # Let flannel use eth1 to enable inter-node communication in tests

     

       63
       +
           canalConfig = pkgs.writeText "rke2-canal-config.yaml" ''

     

       64
       +
             apiVersion: helm.cattle.io/v1

     

       65
       +
             kind: HelmChartConfig

     

       66
       +
             metadata:

     

       67
       +
               name: rke2-canal

     

       68
       +
               namespace: kube-system

     

       69
       +
             spec:

     

       70
       +
               valuesContent: |-

     

       71
       +
                 flannel:

     

       72
       +
                   iface: "eth1"

     

       73
       +
           '';

     

       74
        
         in

     

       75
        
         {

     

       76
        
           name = "${rke2.name}-multi-node";

     

       77
        
           meta.maintainers = rke2.meta.maintainers;

     

       78
        
       

     

       79
        
           nodes = {

     

       80
       +
             server =

     

       81
       +
               {

     

       82
       +
                 config,

     

       83
       +
                 nodes,

     

       84
       +
                 pkgs,

     

       85
       +
                 ...

     

       86
       +
               }:

     

       87
        
               {

     

       88
       +
                 # Setup image archives to be imported by rke2

     

       89
       +
                 systemd.tmpfiles.settings."10-rke2" = {

     

       90
       +
                   "/var/lib/rancher/rke2/agent/images/rke2-images-core.tar.zst" = {

     

       91
       +
                     "L+".argument = "${coreImages}";

     

       92
       +
                   };

     

       93
       +
                   "/var/lib/rancher/rke2/agent/images/rke2-images-canal.tar.zst" = {

     

       94
       +
                     "L+".argument = "${canalImages}";

     

       95
       +
                   };

     

       96
       +
                   "/var/lib/rancher/rke2/agent/images/hello.tar.zst" = {

     

       97
       +
                     "L+".argument = "${helloImage}";

     

       98
       +
                   };

     

       99
       +
                   # Copy the canal config so that rke2 can write the remaining default values to it

     

       100
       +
                   "/var/lib/rancher/rke2/server/manifests/rke2-canal-config.yaml" = {

     

       101
       +
                     "C".argument = "${canalConfig}";

     

       102
       +
                   };

     

       103
       +
                 };

     

       104
       +
       

     

       105
       +
                 # Canal CNI with VXLAN

     

       106
       +
                 networking.firewall.allowedUDPPorts = [ 8472 ];

     

       107
       +
                 networking.firewall.allowedTCPPorts = [

     

       108
       +
                   # Kubernetes API

     

       109
       +
                   6443

     

       110
       +
                   # Canal CNI health checks

     

       111
       +
                   9099

     

       112
       +
                   # RKE2 supervisor API

     

       113
       +
                   9345

     

       114
        
                 ];

     

       115
        
       

     

       116
       +
                 # RKE2 needs more resources than the default

     

       117
       +
                 virtualisation.cores = 4;

     

       118
       +
                 virtualisation.memorySize = 4096;

     

       119
       +
                 virtualisation.diskSize = 8092;

     

       120
        
       

     

       121
        
                 services.rke2 = {

     

       122
        
                   enable = true;

     

       123
        
                   role = "server";

     

       124
       +
                   package = rke2;

     

       125
        
                   inherit tokenFile;

     

       126
        
                   inherit agentTokenFile;

     

       127
       +
                   # Without nodeIP the apiserver starts with the wrong service IP family

     

       128
       +
                   nodeIP = config.networking.primaryIPAddress;

     

       0
        
       
     

       129
        
                   disable = [

     

       130
        
                     "rke2-coredns"

     

       131
        
                     "rke2-metrics-server"

     

       132
        
                     "rke2-ingress-nginx"

     

       133
       +
                     "rke2-snapshot-controller"

     

       134
       +
                     "rke2-snapshot-controller-crd"

     

       135
       +
                     "rke2-snapshot-validation-webhook"

     

       136
        
                   ];

     

       137
        
                 };

     

       138
        
               };

     

       139
        
       

     

       140
       +
             agent =

     

       141
       +
               {

     

       142
       +
                 config,

     

       143
       +
                 nodes,

     

       144
       +
                 pkgs,

     

       145
       +
                 ...

     

       146
       +
               }:

     

       147
        
               {

     

       148
       +
                 # Setup image archives to be imported by rke2

     

       149
       +
                 systemd.tmpfiles.settings."10-rke2" = {

     

       150
       +
                   "/var/lib/rancher/rke2/agent/images/rke2-images-core.linux-amd64.tar.zst" = {

     

       151
       +
                     "L+".argument = "${coreImages}";

     

       152
       +
                   };

     

       153
       +
                   "/var/lib/rancher/rke2/agent/images/rke2-images-canal.linux-amd64.tar.zst" = {

     

       154
       +
                     "L+".argument = "${canalImages}";

     

       155
       +
                   };

     

       156
       +
                   "/var/lib/rancher/rke2/agent/images/hello.tar.zst" = {

     

       157
       +
                     "L+".argument = "${helloImage}";

     

       158
       +
                   };

     

       159
       +
                   "/var/lib/rancher/rke2/server/manifests/rke2-canal-config.yaml" = {

     

       160
       +
                     "C".argument = "${canalConfig}";

     

       161
       +
                   };

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       162
        
                 };

     

       0
        
       
     

       163
        
       

     

       164
       +
                 # Canal CNI health checks

     

       165
       +
                 networking.firewall.allowedTCPPorts = [ 9099 ];

     

       166
       +
                 # Canal CNI with VXLAN

     

       167
       +
                 networking.firewall.allowedUDPPorts = [ 8472 ];

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       168
        
       

     

       169
       +
                 # The agent node can work with less resources

     

       170
       +
                 virtualisation.memorySize = 2048;

     

       171
       +
                 virtualisation.diskSize = 8092;

     

       172
        
       

     

       173
        
                 services.rke2 = {

     

       174
        
                   enable = true;

     

       175
        
                   role = "agent";

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       176
        
                   package = rke2;

     

       177
       +
                   tokenFile = agentTokenFile;

     

       178
       +
                   serverAddr = "https://${nodes.server.networking.primaryIPAddress}:9345";

     

       179
       +
                   nodeIP = config.networking.primaryIPAddress;

     

       180
        
                 };

     

       181
        
               };

     

       182
        
           };

     
···

       184
        
           testScript =

     

       185
        
             let

     

       186
        
               kubectl = "${pkgs.kubectl}/bin/kubectl --kubeconfig=/etc/rancher/rke2/rke2.yaml";

     

       0
        
       
     

       187
        
               jq = "${pkgs.jq}/bin/jq";

     

       0
        
       
     

       188
        
             in

     

       189
       +
             # python

     

       190
        
             ''

     

       191
       +
               start_all()

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       192
        
       

     

       193
       +
               server.wait_for_unit("rke2-server")

     

       194
       +
               agent.wait_for_unit("rke2-agent")

     

       195
        
       

     

       196
       +
               # Wait for the agent to be ready

     

       197
       +
               server.wait_until_succeeds(r"""${kubectl} wait --for='jsonpath={.status.conditions[?(@.type=="Ready")].status}=True' nodes/agent""")

     

       198
        
       

     

       199
       +
               server.succeed("${kubectl} cluster-info")

     

       200
       +
               server.wait_until_succeeds("${kubectl} get serviceaccount default")

     

       201
        
       

     

       202
        
               # Now create a pod on each node via a daemonset and verify they can talk to each other.

     

       203
       +
               server.succeed("${kubectl} apply -f ${networkTestDaemonset}")

     

       204
       +
               server.wait_until_succeeds(

     

       205
        
                   f'[ "$(${kubectl} get ds test -o json | ${jq} .status.numberReady)" -eq {len(machines)} ]'

     

       206
        
               )

     

       207
        
       

     

       208
        
               # Get pod IPs

     

       209
       +
               pods = server.succeed("${kubectl} get po -o json | ${jq} '.items[].metadata.name' -r").splitlines()

     

       210
        
               pod_ips = [

     

       211
       +
                   server.succeed(f"${kubectl} get po {n} -o json | ${jq} '.status.podIP' -cr").strip() for n in pods

     

       212
        
               ]

     

       213
        
       

     

       214
       +
               # Verify each node can ping each pod ip

     

       215
        
               for pod_ip in pod_ips:

     

       216
       +
                   # The CNI sometimes needs a little time

     

       217
       +
                   server.wait_until_succeeds(f"ping -c 1 {pod_ip}", timeout=5)

     

       218
       +
                   agent.wait_until_succeeds(f"ping -c 1 {pod_ip}", timeout=5)

     

       219
       +
                   # Verify the server can exec into the pod

     

       220
       +
                   # for pod in pods:

     

       221
       +
                   #     resp = server.succeed(f"${kubectl} exec {pod} -- socat TCP:{pod_ip}:8000 -")

     

       222
       +
                   #     assert resp.strip() == "hello", f"Unexpected response from hello daemonset: {resp.strip()}"

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       223
        
             '';

     

       224
        
         }

     

       225
        
       )

+61 -51

nixos/tests/rke2/single-node.nix

···

       6
        
           ...

     

       7
        
         }:

     

       8
        
         let

     

       9
       -
           pauseImage = pkgs.dockerTools.streamLayeredImage {

     

       10
       -
             name = "test.local/pause";

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       11
        
             tag = "local";

     

       12
       -
             contents = pkgs.buildEnv {

     

       13
       -
               name = "rke2-pause-image-env";

     

       14
       -
               paths = with pkgs; [

     

       15
       -
                 tini

     

       16
       -
                 (hiPrio coreutils)

     

       17
       -
                 busybox

     

       18
       -
               ];

     

       19
       -
             };

     

       20
       -
             config.Entrypoint = [

     

       21
       -
               "/bin/tini"

     

       22
       -
               "--"

     

       23
       -
               "/bin/sleep"

     

       24
       -
               "inf"

     

       25
       -
             ];

     

       26
        
           };

     

       27
       -
           testPodYaml = pkgs.writeText "test.yaml" ''

     

       28
       -
             apiVersion: v1

     

       29
       -
             kind: Pod

     

       30
        
             metadata:

     

       31
        
               name: test

     

       32
        
             spec:

     

       33
       -
               containers:

     

       34
       -
               - name: test

     

       35
       -
                 image: test.local/pause:local

     

       36
       -
                 imagePullPolicy: Never

     

       37
       -
                 command: ["sh", "-c", "sleep inf"]

     

       0
        
       
     

       38
        
           '';

     

       39
        
         in

     

       40
        
         {

     

       41
        
           name = "${rke2.name}-single-node";

     

       42
        
           meta.maintainers = rke2.meta.maintainers;

     

       43
       -
       

     

       44
        
           nodes.machine =

     

       45
       -
             { pkgs, ... }:

     

       46
        
             {

     

       47
       -
               networking.firewall.enable = false;

     

       48
       -
               networking.useDHCP = false;

     

       49
       -
               networking.defaultGateway = "192.168.1.1";

     

       50
       -
               networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [

     

       51
       -
                 {

     

       52
       -
                   address = "192.168.1.1";

     

       53
       -
                   prefixLength = 24;

     

       54
       -
                 }

     

       55
       -
               ];

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       56
        
       

     

       57
       -
               virtualisation.memorySize = 1536;

     

       58
       -
               virtualisation.diskSize = 4096;

     

       0
        
       
     

       0
        
       
     

       59
        
       

     

       60
        
               services.rke2 = {

     

       61
        
                 enable = true;

     

       62
        
                 role = "server";

     

       63
        
                 package = rke2;

     

       64
       -
                 nodeIP = "192.168.1.1";

     

       0
        
       
     

       0
        
       
     

       65
        
                 disable = [

     

       66
        
                   "rke2-coredns"

     

       67
        
                   "rke2-metrics-server"

     

       68
        
                   "rke2-ingress-nginx"

     

       69
       -
                 ];

     

       70
       -
                 extraFlags = [

     

       71
       -
                   "--cluster-reset"

     

       72
        
                 ];

     

       73
        
               };

     

       74
        
             };

     
···

       76
        
           testScript =

     

       77
        
             let

     

       78
        
               kubectl = "${pkgs.kubectl}/bin/kubectl --kubeconfig=/etc/rancher/rke2/rke2.yaml";

     

       79
       -
               ctr = "${pkgs.containerd}/bin/ctr -a /run/k3s/containerd/containerd.sock";

     

       80
        
             in

     

       0
        
       
     

       81
        
             ''

     

       82
        
               start_all()

     

       83
        
       

     

       84
       -
               machine.wait_for_unit("rke2")

     

       85
        
               machine.succeed("${kubectl} cluster-info")

     

       86
       -
               machine.wait_until_succeeds(

     

       87
       -
                 "${pauseImage} | ${ctr} -n k8s.io image import -"

     

       88
       -
               )

     

       89
        
       

     

       90
        
               machine.wait_until_succeeds("${kubectl} get serviceaccount default")

     

       91
       -
               machine.succeed("${kubectl} apply -f ${testPodYaml}")

     

       92
       -
               machine.succeed("${kubectl} wait --for 'condition=Ready' pod/test")

     

       93
       -
               machine.succeed("${kubectl} delete -f ${testPodYaml}")

     

       94
       -
       

     

       95
       -
               machine.shutdown()

     

       96
        
             '';

     

       97
        
         }

     

       98
        
       )

···

       6
        
           ...

     

       7
        
         }:

     

       8
        
         let

     

       9
       +
           throwSystem = throw "RKE2: Unsupported system: ${pkgs.stdenv.hostPlatform.system}";

     

       10
       +
           coreImages =

     

       11
       +
             {

     

       12
       +
               aarch64-linux = rke2.images-core-linux-arm64-tar-zst;

     

       13
       +
               x86_64-linux = rke2.images-core-linux-amd64-tar-zst;

     

       14
       +
             }

     

       15
       +
             .${pkgs.stdenv.hostPlatform.system} or throwSystem;

     

       16
       +
           canalImages =

     

       17
       +
             {

     

       18
       +
               aarch64-linux = rke2.images-canal-linux-arm64-tar-zst;

     

       19
       +
               x86_64-linux = rke2.images-canal-linux-amd64-tar-zst;

     

       20
       +
             }

     

       21
       +
             .${pkgs.stdenv.hostPlatform.system} or throwSystem;

     

       22
       +
           helloImage = pkgs.dockerTools.buildImage {

     

       23
       +
             name = "test.local/hello";

     

       24
        
             tag = "local";

     

       25
       +
             compressor = "zstd";

     

       26
       +
             copyToRoot = pkgs.hello;

     

       27
       +
             config.Entrypoint = [ "${pkgs.hello}/bin/hello" ];

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       28
        
           };

     

       29
       +
           testJobYaml = pkgs.writeText "test.yaml" ''

     

       30
       +
             apiVersion: batch/v1

     

       31
       +
             kind: Job

     

       32
        
             metadata:

     

       33
        
               name: test

     

       34
        
             spec:

     

       35
       +
               template:

     

       36
       +
                 spec:

     

       37
       +
                   containers:

     

       38
       +
                   - name: test

     

       39
       +
                     image: "test.local/hello:local"

     

       40
       +
                   restartPolicy: Never

     

       41
        
           '';

     

       42
        
         in

     

       43
        
         {

     

       44
        
           name = "${rke2.name}-single-node";

     

       45
        
           meta.maintainers = rke2.meta.maintainers;

     

       0
        
       
     

       46
        
           nodes.machine =

     

       0
        
       
     

       47
        
             {

     

       48
       +
               config,

     

       49
       +
               nodes,

     

       50
       +
               pkgs,

     

       51
       +
               ...

     

       52
       +
             }:

     

       53
       +
             {

     

       54
       +
               # Setup image archives to be imported by rke2

     

       55
       +
               systemd.tmpfiles.settings."10-rke2" = {

     

       56
       +
                 "/var/lib/rancher/rke2/agent/images/rke2-images-core.tar.zst" = {

     

       57
       +
                   "L+".argument = "${coreImages}";

     

       58
       +
                 };

     

       59
       +
                 "/var/lib/rancher/rke2/agent/images/rke2-images-canal.tar.zst" = {

     

       60
       +
                   "L+".argument = "${canalImages}";

     

       61
       +
                 };

     

       62
       +
                 "/var/lib/rancher/rke2/agent/images/hello.tar.zst" = {

     

       63
       +
                   "L+".argument = "${helloImage}";

     

       64
       +
                 };

     

       65
       +
               };

     

       66
        
       

     

       67
       +
               # RKE2 needs more resources than the default

     

       68
       +
               virtualisation.cores = 4;

     

       69
       +
               virtualisation.memorySize = 4096;

     

       70
       +
               virtualisation.diskSize = 8092;

     

       71
        
       

     

       72
        
               services.rke2 = {

     

       73
        
                 enable = true;

     

       74
        
                 role = "server";

     

       75
        
                 package = rke2;

     

       76
       +
                 # Without nodeIP the apiserver starts with the wrong service IP family

     

       77
       +
                 nodeIP = config.networking.primaryIPAddress;

     

       78
       +
                 # Slightly reduce resource consumption

     

       79
        
                 disable = [

     

       80
        
                   "rke2-coredns"

     

       81
        
                   "rke2-metrics-server"

     

       82
        
                   "rke2-ingress-nginx"

     

       83
       +
                   "rke2-snapshot-controller"

     

       84
       +
                   "rke2-snapshot-controller-crd"

     

       85
       +
                   "rke2-snapshot-validation-webhook"

     

       86
        
                 ];

     

       87
        
               };

     

       88
        
             };

     
···

       90
        
           testScript =

     

       91
        
             let

     

       92
        
               kubectl = "${pkgs.kubectl}/bin/kubectl --kubeconfig=/etc/rancher/rke2/rke2.yaml";

     

       0
        
       
     

       93
        
             in

     

       94
       +
             # python

     

       95
        
             ''

     

       96
        
               start_all()

     

       97
        
       

     

       98
       +
               machine.wait_for_unit("rke2-server")

     

       99
        
               machine.succeed("${kubectl} cluster-info")

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       100
        
       

     

       101
        
               machine.wait_until_succeeds("${kubectl} get serviceaccount default")

     

       102
       +
               machine.succeed("${kubectl} apply -f ${testJobYaml}")

     

       103
       +
               machine.wait_until_succeeds("${kubectl} wait --for 'condition=complete' job/test")

     

       104
       +
               output = machine.succeed("${kubectl} logs -l batch.kubernetes.io/job-name=test")

     

       105
       +
               assert output.rstrip() == "Hello, world!", f"unexpected output of test job: {output}"

     

       0
        
       
     

       106
        
             '';

     

       107
        
         }

     

       108
        
       )

+9 -3

pkgs/applications/networking/cluster/rke2/builder.nix

···

       134
        
           passthru = {

     

       135
        
             inherit updateScript;

     

       136
        
             tests =

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       137
        
               {

     

       138
        
                 version = testers.testVersion {

     

       139
        
                   package = rke2;

     

       140
        
                   version = "v${version}";

     

       141
        
                 };

     

       142
        
               }

     

       143
       -
               // lib.optionalAttrs stdenv.hostPlatform.isLinux {

     

       144
       -
                 inherit (nixosTests) rke2;

     

       145
       -
               };

     

       146
        
           } // (lib.mapAttrs (_: value: fetchurl value) imagesVersions);

     

       147
        
       

     

       148
        
           meta = with lib; {

···

       134
        
           passthru = {

     

       135
        
             inherit updateScript;

     

       136
        
             tests =

     

       137
       +
               let

     

       138
       +
                 moduleTests =

     

       139
       +
                   let

     

       140
       +
                     package_version =

     

       141
       +
                       "rke2_" + lib.replaceStrings [ "." ] [ "_" ] (lib.versions.majorMinor rke2Version);

     

       142
       +
                   in

     

       143
       +
                   lib.mapAttrs (name: value: nixosTests.rke2.${name}.${package_version}) nixosTests.rke2;

     

       144
       +
               in

     

       145
        
               {

     

       146
        
                 version = testers.testVersion {

     

       147
        
                   package = rke2;

     

       148
        
                   version = "v${version}";

     

       149
        
                 };

     

       150
        
               }

     

       151
       +
               // moduleTests;

     

       0
        
       
     

       0
        
       
     

       152
        
           } // (lib.mapAttrs (_: value: fetchurl value) imagesVersions);

     

       153
        
       

     

       154
        
           meta = with lib; {