pyrox.dev
1{
2 pkgs,
3 lib,
4 getPackage ? pkgs: pkgs.cassandra_4,
5 ...
6}:
7let
8 testPackage = getPackage pkgs;
9 clusterName = "NixOS Automated-Test Cluster";
10 testRemoteAuth = lib.versionAtLeast testPackage.version "3.11";
11 jmxRoles = [
12 {
13 username = "me";
14 password = "password";
15 }
16 ];
17 jmxRolesFile = ./cassandra-jmx-roles;
18 jmxAuthArgs = "-u ${(builtins.elemAt jmxRoles 0).username} -pw ${(builtins.elemAt jmxRoles 0).password}";
19 jmxPort = 7200; # Non-standard port so it doesn't accidentally work
20 jmxPortStr = toString jmxPort;
21
22 # Would usually be assigned to 512M.
23 # Set it to a different value, so that we can check whether our config
24 # actually changes it.
25 numMaxHeapSize = "400";
26 getHeapLimitCommand = ''
27 nodetool info -p ${jmxPortStr} | grep "^Heap Memory" | awk '{print $NF}'
28 '';
29 checkHeapLimitCommand = pkgs.writeShellScript "check-heap-limit.sh" ''
30 [ 1 -eq "$(echo "$(${getHeapLimitCommand}) < ${numMaxHeapSize}" | ${pkgs.bc}/bin/bc)" ]
31 '';
32
33 cassandraCfg = pkgs: ipAddress: {
34 enable = true;
35 inherit clusterName;
36 listenAddress = ipAddress;
37 rpcAddress = ipAddress;
38 seedAddresses = [ "192.168.1.1" ];
39 package = getPackage pkgs;
40 maxHeapSize = "${numMaxHeapSize}M";
41 heapNewSize = "100M";
42 inherit jmxPort;
43 };
44 nodeCfg =
45 ipAddress: extra:
46 { pkgs, config, ... }:
47 rec {
48 environment.systemPackages = [ (getPackage pkgs) ];
49 networking = {
50 firewall.allowedTCPPorts = [
51 7000
52 9042
53 services.cassandra.jmxPort
54 ];
55 useDHCP = false;
56 interfaces.eth1.ipv4.addresses = pkgs.lib.mkOverride 0 [
57 {
58 address = ipAddress;
59 prefixLength = 24;
60 }
61 ];
62 };
63 services.cassandra = cassandraCfg pkgs ipAddress // extra;
64 };
65in
66{
67 name = "cassandra-${testPackage.version}";
68 meta = {
69 maintainers = with lib.maintainers; [ johnazoidberg ];
70 };
71
72 nodes = {
73 cass0 = nodeCfg "192.168.1.1" { };
74 cass1 = nodeCfg "192.168.1.2" (
75 lib.optionalAttrs testRemoteAuth {
76 inherit jmxRoles;
77 remoteJmx = true;
78 }
79 );
80 cass2 = nodeCfg "192.168.1.3" { jvmOpts = [ "-Dcassandra.replace_address=cass1" ]; };
81 };
82
83 testScript = ''
84 # Check configuration
85 with subtest("Timers exist"):
86 cass0.succeed("systemctl list-timers | grep cassandra-full-repair.timer")
87 cass0.succeed("systemctl list-timers | grep cassandra-incremental-repair.timer")
88
89 with subtest("Can connect via cqlsh"):
90 cass0.wait_for_unit("cassandra.service")
91 cass0.wait_until_succeeds("nc -z cass0 9042")
92 cass0.succeed("echo 'show version;' | cqlsh cass0")
93
94 with subtest("Nodetool is operational"):
95 cass0.wait_for_unit("cassandra.service")
96 cass0.wait_until_succeeds("nc -z localhost ${jmxPortStr}")
97 cass0.succeed("nodetool status -p ${jmxPortStr} --resolve-ip | egrep '^UN[[:space:]]+cass0'")
98
99 with subtest("Cluster name was set"):
100 cass0.wait_for_unit("cassandra.service")
101 cass0.wait_until_succeeds("nc -z localhost ${jmxPortStr}")
102 cass0.wait_until_succeeds(
103 "nodetool describecluster -p ${jmxPortStr} | grep 'Name: ${clusterName}'"
104 )
105
106 with subtest("Heap limit set correctly"):
107 # Nodetool takes a while until it can display info
108 cass0.wait_until_succeeds("nodetool info -p ${jmxPortStr}")
109 cass0.succeed("${checkHeapLimitCommand}")
110
111 # Check cluster interaction
112 with subtest("Bring up cluster"):
113 cass1.wait_for_unit("cassandra.service")
114 cass1.wait_until_succeeds(
115 "nodetool -p ${jmxPortStr} ${jmxAuthArgs} status | egrep -c '^UN' | grep 2"
116 )
117 cass0.succeed("nodetool status -p ${jmxPortStr} --resolve-ip | egrep '^UN[[:space:]]+cass1'")
118 ''
119 + lib.optionalString testRemoteAuth ''
120 with subtest("Remote authenticated jmx"):
121 # Doesn't work if not enabled
122 cass0.wait_until_succeeds("nc -z localhost ${jmxPortStr}")
123 cass1.fail("nc -z 192.168.1.1 ${jmxPortStr}")
124 cass1.fail("nodetool -p ${jmxPortStr} -h 192.168.1.1 status")
125
126 # Works if enabled
127 cass1.wait_until_succeeds("nc -z localhost ${jmxPortStr}")
128 cass0.succeed("nodetool -p ${jmxPortStr} -h 192.168.1.2 ${jmxAuthArgs} status")
129 ''
130 + ''
131 with subtest("Break and fix node"):
132 cass1.block()
133 cass0.wait_until_succeeds(
134 "nodetool status -p ${jmxPortStr} --resolve-ip | egrep -c '^DN[[:space:]]+cass1'"
135 )
136 cass0.succeed("nodetool status -p ${jmxPortStr} | egrep -c '^UN' | grep 1")
137 cass1.unblock()
138 cass1.wait_until_succeeds(
139 "nodetool -p ${jmxPortStr} ${jmxAuthArgs} status | egrep -c '^UN' | grep 2"
140 )
141 cass0.succeed("nodetool status -p ${jmxPortStr} | egrep -c '^UN' | grep 2")
142
143 with subtest("Replace crashed node"):
144 cass1.block() # .crash() waits until it's fully shutdown
145 cass2.start()
146 cass0.wait_until_fails(
147 "nodetool status -p ${jmxPortStr} --resolve-ip | egrep '^UN[[:space:]]+cass1'"
148 )
149
150 cass2.wait_for_unit("cassandra.service")
151 cass0.wait_until_succeeds(
152 "nodetool status -p ${jmxPortStr} --resolve-ip | egrep '^UN[[:space:]]+cass2'"
153 )
154 '';
155
156 passthru = {
157 inherit testPackage;
158 };
159}