pyrox.dev
1# A test that sets extra kubelet configuration and enables graceful node shutdown
2import ../make-test-python.nix (
3 {
4 pkgs,
5 lib,
6 k3s,
7 ...
8 }:
9 let
10 nodeName = "test";
11 shutdownGracePeriod = "1m13s";
12 shutdownGracePeriodCriticalPods = "13s";
13 podsPerCore = 3;
14 memoryThrottlingFactor = 0.69;
15 containerLogMaxSize = "5Mi";
16 in
17 {
18 name = "${k3s.name}-kubelet-config";
19 nodes.machine =
20 { pkgs, ... }:
21 {
22 environment.systemPackages = [ pkgs.jq ];
23
24 # k3s uses enough resources the default vm fails.
25 virtualisation.memorySize = 1536;
26 virtualisation.diskSize = 4096;
27
28 services.k3s = {
29 enable = true;
30 package = k3s;
31 # Slightly reduce resource usage
32 extraFlags = [
33 "--disable coredns"
34 "--disable local-storage"
35 "--disable metrics-server"
36 "--disable servicelb"
37 "--disable traefik"
38 "--node-name ${nodeName}"
39 ];
40 gracefulNodeShutdown = {
41 enable = true;
42 inherit shutdownGracePeriod shutdownGracePeriodCriticalPods;
43 };
44 extraKubeletConfig = {
45 inherit podsPerCore memoryThrottlingFactor containerLogMaxSize;
46 };
47 };
48 };
49
50 testScript = ''
51 import json
52
53 start_all()
54 machine.wait_for_unit("k3s")
55 # wait until the node is ready
56 machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""")
57 # test whether the kubelet registered an inhibitor lock
58 machine.succeed("systemd-inhibit --list --no-legend | grep \"kubelet.*k3s-server.*shutdown\"")
59 # run kubectl proxy in the background, close stdout through redirection to not wait for the command to finish
60 machine.execute("kubectl proxy --address 127.0.0.1 --port=8001 >&2 &")
61 machine.wait_until_succeeds("nc -z 127.0.0.1 8001")
62 # get the kubeletconfig
63 kubelet_config=json.loads(machine.succeed("curl http://127.0.0.1:8001/api/v1/nodes/${nodeName}/proxy/configz | jq '.kubeletconfig'"))
64
65 with subtest("Kubelet config values are set correctly"):
66 assert kubelet_config["shutdownGracePeriod"] == "${shutdownGracePeriod}", \
67 f"unexpected value for shutdownGracePeriod: {kubelet_config["shutdownGracePeriod"]}"
68 assert kubelet_config["shutdownGracePeriodCriticalPods"] == "${shutdownGracePeriodCriticalPods}", \
69 f"unexpected value for shutdownGracePeriodCriticalPods: {kubelet_config["shutdownGracePeriodCriticalPods"]}"
70 assert kubelet_config["podsPerCore"] == ${toString podsPerCore}, \
71 f"unexpected value for podsPerCore: {kubelet_config["podsPerCore"]}"
72 assert kubelet_config["memoryThrottlingFactor"] == ${toString memoryThrottlingFactor}, \
73 f"unexpected value for memoryThrottlingFactor: {kubelet_config["memoryThrottlingFactor"]}"
74 assert kubelet_config["containerLogMaxSize"] == "${containerLogMaxSize}", \
75 f"unexpected value for containerLogMaxSize: {kubelet_config["containerLogMaxSize"]}"
76 '';
77
78 meta.maintainers = lib.teams.k3s.members;
79 }
80)