pyrox.dev
1import ./make-test-python.nix (
2 {
3 pkgs,
4 lib,
5 testPackage ? pkgs.cassandra,
6 ...
7 }:
8 let
9 clusterName = "NixOS Automated-Test Cluster";
10
11 testRemoteAuth = lib.versionAtLeast testPackage.version "3.11";
12 jmxRoles = [
13 {
14 username = "me";
15 password = "password";
16 }
17 ];
18 jmxRolesFile = ./cassandra-jmx-roles;
19 jmxAuthArgs = "-u ${(builtins.elemAt jmxRoles 0).username} -pw ${(builtins.elemAt jmxRoles 0).password}";
20 jmxPort = 7200; # Non-standard port so it doesn't accidentally work
21 jmxPortStr = toString jmxPort;
22
23 # Would usually be assigned to 512M.
24 # Set it to a different value, so that we can check whether our config
25 # actually changes it.
26 numMaxHeapSize = "400";
27 getHeapLimitCommand = ''
28 nodetool info -p ${jmxPortStr} | grep "^Heap Memory" | awk '{print $NF}'
29 '';
30 checkHeapLimitCommand = pkgs.writeShellScript "check-heap-limit.sh" ''
31 [ 1 -eq "$(echo "$(${getHeapLimitCommand}) < ${numMaxHeapSize}" | ${pkgs.bc}/bin/bc)" ]
32 '';
33
34 cassandraCfg = ipAddress: {
35 enable = true;
36 inherit clusterName;
37 listenAddress = ipAddress;
38 rpcAddress = ipAddress;
39 seedAddresses = [ "192.168.1.1" ];
40 package = testPackage;
41 maxHeapSize = "${numMaxHeapSize}M";
42 heapNewSize = "100M";
43 inherit jmxPort;
44 };
45 nodeCfg =
46 ipAddress: extra:
47 { pkgs, config, ... }:
48 rec {
49 environment.systemPackages = [ testPackage ];
50 networking = {
51 firewall.allowedTCPPorts = [
52 7000
53 9042
54 services.cassandra.jmxPort
55 ];
56 useDHCP = false;
57 interfaces.eth1.ipv4.addresses = pkgs.lib.mkOverride 0 [
58 {
59 address = ipAddress;
60 prefixLength = 24;
61 }
62 ];
63 };
64 services.cassandra = cassandraCfg ipAddress // extra;
65 };
66 in
67 {
68 name = "cassandra-${testPackage.version}";
69 meta = {
70 maintainers = with lib.maintainers; [ johnazoidberg ];
71 };
72
73 nodes = {
74 cass0 = nodeCfg "192.168.1.1" { };
75 cass1 = nodeCfg "192.168.1.2" (
76 lib.optionalAttrs testRemoteAuth {
77 inherit jmxRoles;
78 remoteJmx = true;
79 }
80 );
81 cass2 = nodeCfg "192.168.1.3" { jvmOpts = [ "-Dcassandra.replace_address=cass1" ]; };
82 };
83
84 testScript =
85 ''
86 # Check configuration
87 with subtest("Timers exist"):
88 cass0.succeed("systemctl list-timers | grep cassandra-full-repair.timer")
89 cass0.succeed("systemctl list-timers | grep cassandra-incremental-repair.timer")
90
91 with subtest("Can connect via cqlsh"):
92 cass0.wait_for_unit("cassandra.service")
93 cass0.wait_until_succeeds("nc -z cass0 9042")
94 cass0.succeed("echo 'show version;' | cqlsh cass0")
95
96 with subtest("Nodetool is operational"):
97 cass0.wait_for_unit("cassandra.service")
98 cass0.wait_until_succeeds("nc -z localhost ${jmxPortStr}")
99 cass0.succeed("nodetool status -p ${jmxPortStr} --resolve-ip | egrep '^UN[[:space:]]+cass0'")
100
101 with subtest("Cluster name was set"):
102 cass0.wait_for_unit("cassandra.service")
103 cass0.wait_until_succeeds("nc -z localhost ${jmxPortStr}")
104 cass0.wait_until_succeeds(
105 "nodetool describecluster -p ${jmxPortStr} | grep 'Name: ${clusterName}'"
106 )
107
108 with subtest("Heap limit set correctly"):
109 # Nodetool takes a while until it can display info
110 cass0.wait_until_succeeds("nodetool info -p ${jmxPortStr}")
111 cass0.succeed("${checkHeapLimitCommand}")
112
113 # Check cluster interaction
114 with subtest("Bring up cluster"):
115 cass1.wait_for_unit("cassandra.service")
116 cass1.wait_until_succeeds(
117 "nodetool -p ${jmxPortStr} ${jmxAuthArgs} status | egrep -c '^UN' | grep 2"
118 )
119 cass0.succeed("nodetool status -p ${jmxPortStr} --resolve-ip | egrep '^UN[[:space:]]+cass1'")
120 ''
121 + lib.optionalString testRemoteAuth ''
122 with subtest("Remote authenticated jmx"):
123 # Doesn't work if not enabled
124 cass0.wait_until_succeeds("nc -z localhost ${jmxPortStr}")
125 cass1.fail("nc -z 192.168.1.1 ${jmxPortStr}")
126 cass1.fail("nodetool -p ${jmxPortStr} -h 192.168.1.1 status")
127
128 # Works if enabled
129 cass1.wait_until_succeeds("nc -z localhost ${jmxPortStr}")
130 cass0.succeed("nodetool -p ${jmxPortStr} -h 192.168.1.2 ${jmxAuthArgs} status")
131 ''
132 + ''
133 with subtest("Break and fix node"):
134 cass1.block()
135 cass0.wait_until_succeeds(
136 "nodetool status -p ${jmxPortStr} --resolve-ip | egrep -c '^DN[[:space:]]+cass1'"
137 )
138 cass0.succeed("nodetool status -p ${jmxPortStr} | egrep -c '^UN' | grep 1")
139 cass1.unblock()
140 cass1.wait_until_succeeds(
141 "nodetool -p ${jmxPortStr} ${jmxAuthArgs} status | egrep -c '^UN' | grep 2"
142 )
143 cass0.succeed("nodetool status -p ${jmxPortStr} | egrep -c '^UN' | grep 2")
144
145 with subtest("Replace crashed node"):
146 cass1.block() # .crash() waits until it's fully shutdown
147 cass2.start()
148 cass0.wait_until_fails(
149 "nodetool status -p ${jmxPortStr} --resolve-ip | egrep '^UN[[:space:]]+cass1'"
150 )
151
152 cass2.wait_for_unit("cassandra.service")
153 cass0.wait_until_succeeds(
154 "nodetool status -p ${jmxPortStr} --resolve-ip | egrep '^UN[[:space:]]+cass2'"
155 )
156 '';
157
158 passthru = {
159 inherit testPackage;
160 };
161 }
162)