pyrox.dev
1# A test that sets extra kubelet configuration and enables graceful node shutdown
2import ../make-test-python.nix (
3 {
4 pkgs,
5 lib,
6 k3s,
7 ...
8 }:
9 let
10 nodeName = "test";
11 shutdownGracePeriod = "1m13s";
12 shutdownGracePeriodCriticalPods = "13s";
13 podsPerCore = 3;
14 memoryThrottlingFactor = 0.69;
15 containerLogMaxSize = "5Mi";
16 in
17 {
18 name = "${k3s.name}-kubelet-config";
19 nodes.machine =
20 { pkgs, ... }:
21 {
22 environment.systemPackages = [ pkgs.jq ];
23
24 # k3s uses enough resources the default vm fails.
25 virtualisation.memorySize = 1536;
26 virtualisation.diskSize = 4096;
27
28 services.k3s = {
29 enable = true;
30 package = k3s;
31 # Slightly reduce resource usage
32 extraFlags = [
33 "--disable coredns"
34 "--disable local-storage"
35 "--disable metrics-server"
36 "--disable servicelb"
37 "--disable traefik"
38 "--node-name ${nodeName}"
39 ];
40 gracefulNodeShutdown = {
41 enable = true;
42 inherit shutdownGracePeriod shutdownGracePeriodCriticalPods;
43 };
44 extraKubeletConfig = {
45 inherit podsPerCore memoryThrottlingFactor containerLogMaxSize;
46 };
47 };
48 };
49
50 testScript = # python
51 ''
52 import json
53
54 start_all()
55 machine.wait_for_unit("k3s")
56 # wait until the node is ready
57 machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""")
58 # test whether the kubelet registered an inhibitor lock
59 machine.succeed("systemd-inhibit --list --no-legend | grep \"kubelet.*k3s-server.*shutdown\"")
60 # run kubectl proxy in the background, close stdout through redirection to not wait for the command to finish
61 machine.execute("kubectl proxy --address 127.0.0.1 --port=8001 >&2 &")
62 machine.wait_until_succeeds("nc -z 127.0.0.1 8001")
63 # get the kubeletconfig
64 kubelet_config=json.loads(machine.succeed("curl http://127.0.0.1:8001/api/v1/nodes/${nodeName}/proxy/configz | jq '.kubeletconfig'"))
65
66 with subtest("Kubelet config values are set correctly"):
67 t.assertEqual(kubelet_config["shutdownGracePeriod"], "${shutdownGracePeriod}")
68 t.assertEqual(kubelet_config["shutdownGracePeriodCriticalPods"], "${shutdownGracePeriodCriticalPods}")
69 t.assertEqual(kubelet_config["podsPerCore"], ${toString podsPerCore})
70 t.assertEqual(kubelet_config["memoryThrottlingFactor"], ${toString memoryThrottlingFactor})
71 t.assertEqual(kubelet_config["containerLogMaxSize"],"${containerLogMaxSize}")
72 '';
73
74 meta.maintainers = lib.teams.k3s.members;
75 }
76)