pyrox.dev
1import ./make-test-python.nix ({ pkgs, lib, testPackage ? pkgs.cassandra, ... }:
2let
3 clusterName = "NixOS Automated-Test Cluster";
4
5 testRemoteAuth = lib.versionAtLeast testPackage.version "3.11";
6 jmxRoles = [{ username = "me"; password = "password"; }];
7 jmxRolesFile = ./cassandra-jmx-roles;
8 jmxAuthArgs = "-u ${(builtins.elemAt jmxRoles 0).username} -pw ${(builtins.elemAt jmxRoles 0).password}";
9 jmxPort = 7200; # Non-standard port so it doesn't accidentally work
10 jmxPortStr = toString jmxPort;
11
12 # Would usually be assigned to 512M.
13 # Set it to a different value, so that we can check whether our config
14 # actually changes it.
15 numMaxHeapSize = "400";
16 getHeapLimitCommand = ''
17 nodetool info -p ${jmxPortStr} | grep "^Heap Memory" | awk '{print $NF}'
18 '';
19 checkHeapLimitCommand = pkgs.writeShellScript "check-heap-limit.sh" ''
20 [ 1 -eq "$(echo "$(${getHeapLimitCommand}) < ${numMaxHeapSize}" | ${pkgs.bc}/bin/bc)" ]
21 '';
22
23 cassandraCfg = ipAddress:
24 { enable = true;
25 inherit clusterName;
26 listenAddress = ipAddress;
27 rpcAddress = ipAddress;
28 seedAddresses = [ "192.168.1.1" ];
29 package = testPackage;
30 maxHeapSize = "${numMaxHeapSize}M";
31 heapNewSize = "100M";
32 inherit jmxPort;
33 };
34 nodeCfg = ipAddress: extra: {pkgs, config, ...}: rec {
35 environment.systemPackages = [ testPackage ];
36 networking = {
37 firewall.allowedTCPPorts = [ 7000 9042 services.cassandra.jmxPort ];
38 useDHCP = false;
39 interfaces.eth1.ipv4.addresses = pkgs.lib.mkOverride 0 [
40 { address = ipAddress; prefixLength = 24; }
41 ];
42 };
43 services.cassandra = cassandraCfg ipAddress // extra;
44 };
45in
46{
47 name = "cassandra-${testPackage.version}";
48 meta = {
49 maintainers = with lib.maintainers; [ johnazoidberg ];
50 };
51
52 nodes = {
53 cass0 = nodeCfg "192.168.1.1" {};
54 cass1 = nodeCfg "192.168.1.2" (lib.optionalAttrs testRemoteAuth { inherit jmxRoles; remoteJmx = true; });
55 cass2 = nodeCfg "192.168.1.3" { jvmOpts = [ "-Dcassandra.replace_address=cass1" ]; };
56 };
57
58 testScript = ''
59 # Check configuration
60 with subtest("Timers exist"):
61 cass0.succeed("systemctl list-timers | grep cassandra-full-repair.timer")
62 cass0.succeed("systemctl list-timers | grep cassandra-incremental-repair.timer")
63
64 with subtest("Can connect via cqlsh"):
65 cass0.wait_for_unit("cassandra.service")
66 cass0.wait_until_succeeds("nc -z cass0 9042")
67 cass0.succeed("echo 'show version;' | cqlsh cass0")
68
69 with subtest("Nodetool is operational"):
70 cass0.wait_for_unit("cassandra.service")
71 cass0.wait_until_succeeds("nc -z localhost ${jmxPortStr}")
72 cass0.succeed("nodetool status -p ${jmxPortStr} --resolve-ip | egrep '^UN[[:space:]]+cass0'")
73
74 with subtest("Cluster name was set"):
75 cass0.wait_for_unit("cassandra.service")
76 cass0.wait_until_succeeds("nc -z localhost ${jmxPortStr}")
77 cass0.wait_until_succeeds(
78 "nodetool describecluster -p ${jmxPortStr} | grep 'Name: ${clusterName}'"
79 )
80
81 with subtest("Heap limit set correctly"):
82 # Nodetool takes a while until it can display info
83 cass0.wait_until_succeeds("nodetool info -p ${jmxPortStr}")
84 cass0.succeed("${checkHeapLimitCommand}")
85
86 # Check cluster interaction
87 with subtest("Bring up cluster"):
88 cass1.wait_for_unit("cassandra.service")
89 cass1.wait_until_succeeds(
90 "nodetool -p ${jmxPortStr} ${jmxAuthArgs} status | egrep -c '^UN' | grep 2"
91 )
92 cass0.succeed("nodetool status -p ${jmxPortStr} --resolve-ip | egrep '^UN[[:space:]]+cass1'")
93 '' + lib.optionalString testRemoteAuth ''
94 with subtest("Remote authenticated jmx"):
95 # Doesn't work if not enabled
96 cass0.wait_until_succeeds("nc -z localhost ${jmxPortStr}")
97 cass1.fail("nc -z 192.168.1.1 ${jmxPortStr}")
98 cass1.fail("nodetool -p ${jmxPortStr} -h 192.168.1.1 status")
99
100 # Works if enabled
101 cass1.wait_until_succeeds("nc -z localhost ${jmxPortStr}")
102 cass0.succeed("nodetool -p ${jmxPortStr} -h 192.168.1.2 ${jmxAuthArgs} status")
103 '' + ''
104 with subtest("Break and fix node"):
105 cass1.block()
106 cass0.wait_until_succeeds(
107 "nodetool status -p ${jmxPortStr} --resolve-ip | egrep -c '^DN[[:space:]]+cass1'"
108 )
109 cass0.succeed("nodetool status -p ${jmxPortStr} | egrep -c '^UN' | grep 1")
110 cass1.unblock()
111 cass1.wait_until_succeeds(
112 "nodetool -p ${jmxPortStr} ${jmxAuthArgs} status | egrep -c '^UN' | grep 2"
113 )
114 cass0.succeed("nodetool status -p ${jmxPortStr} | egrep -c '^UN' | grep 2")
115
116 with subtest("Replace crashed node"):
117 cass1.block() # .crash() waits until it's fully shutdown
118 cass2.start()
119 cass0.wait_until_fails(
120 "nodetool status -p ${jmxPortStr} --resolve-ip | egrep '^UN[[:space:]]+cass1'"
121 )
122
123 cass2.wait_for_unit("cassandra.service")
124 cass0.wait_until_succeeds(
125 "nodetool status -p ${jmxPortStr} --resolve-ip | egrep '^UN[[:space:]]+cass2'"
126 )
127 '';
128
129 passthru = {
130 inherit testPackage;
131 };
132})