pyrox.dev
1import ./make-test-python.nix ({ pkgs, lib, testPackage ? pkgs.cassandra, ... }:
2let
3 clusterName = "NixOS Automated-Test Cluster";
4
5 testRemoteAuth = lib.versionAtLeast testPackage.version "3.11";
6 jmxRoles = [{ username = "me"; password = "password"; }];
7 jmxRolesFile = ./cassandra-jmx-roles;
8 jmxAuthArgs = "-u ${(builtins.elemAt jmxRoles 0).username} -pw ${(builtins.elemAt jmxRoles 0).password}";
9 jmxPort = 7200; # Non-standard port so it doesn't accidentally work
10 jmxPortStr = toString jmxPort;
11
12 # Would usually be assigned to 512M.
13 # Set it to a different value, so that we can check whether our config
14 # actually changes it.
15 numMaxHeapSize = "400";
16 getHeapLimitCommand = ''
17 nodetool info -p ${jmxPortStr} | grep "^Heap Memory" | awk '{print $NF}'
18 '';
19 checkHeapLimitCommand = pkgs.writeShellScript "check-heap-limit.sh" ''
20 [ 1 -eq "$(echo "$(${getHeapLimitCommand}) < ${numMaxHeapSize}" | ${pkgs.bc}/bin/bc)" ]
21 '';
22
23 cassandraCfg = ipAddress:
24 { enable = true;
25 inherit clusterName;
26 listenAddress = ipAddress;
27 rpcAddress = ipAddress;
28 seedAddresses = [ "192.168.1.1" ];
29 package = testPackage;
30 maxHeapSize = "${numMaxHeapSize}M";
31 heapNewSize = "100M";
32 inherit jmxPort;
33 };
34 nodeCfg = ipAddress: extra: {pkgs, config, ...}: rec {
35 environment.systemPackages = [ testPackage ];
36 networking = {
37 firewall.allowedTCPPorts = [ 7000 9042 services.cassandra.jmxPort ];
38 useDHCP = false;
39 interfaces.eth1.ipv4.addresses = pkgs.lib.mkOverride 0 [
40 { address = ipAddress; prefixLength = 24; }
41 ];
42 };
43 services.cassandra = cassandraCfg ipAddress // extra;
44 virtualisation.memorySize = 1024;
45 };
46in
47{
48 name = "cassandra-${testPackage.version}";
49 meta = {
50 maintainers = with lib.maintainers; [ johnazoidberg ];
51 };
52
53 nodes = {
54 cass0 = nodeCfg "192.168.1.1" {};
55 cass1 = nodeCfg "192.168.1.2" (lib.optionalAttrs testRemoteAuth { inherit jmxRoles; remoteJmx = true; });
56 cass2 = nodeCfg "192.168.1.3" { jvmOpts = [ "-Dcassandra.replace_address=cass1" ]; };
57 };
58
59 testScript = ''
60 # Check configuration
61 with subtest("Timers exist"):
62 cass0.succeed("systemctl list-timers | grep cassandra-full-repair.timer")
63 cass0.succeed("systemctl list-timers | grep cassandra-incremental-repair.timer")
64
65 with subtest("Can connect via cqlsh"):
66 cass0.wait_for_unit("cassandra.service")
67 cass0.wait_until_succeeds("nc -z cass0 9042")
68 cass0.succeed("echo 'show version;' | cqlsh cass0")
69
70 with subtest("Nodetool is operational"):
71 cass0.wait_for_unit("cassandra.service")
72 cass0.wait_until_succeeds("nc -z localhost ${jmxPortStr}")
73 cass0.succeed("nodetool status -p ${jmxPortStr} --resolve-ip | egrep '^UN[[:space:]]+cass0'")
74
75 with subtest("Cluster name was set"):
76 cass0.wait_for_unit("cassandra.service")
77 cass0.wait_until_succeeds("nc -z localhost ${jmxPortStr}")
78 cass0.wait_until_succeeds(
79 "nodetool describecluster -p ${jmxPortStr} | grep 'Name: ${clusterName}'"
80 )
81
82 with subtest("Heap limit set correctly"):
83 # Nodetool takes a while until it can display info
84 cass0.wait_until_succeeds("nodetool info -p ${jmxPortStr}")
85 cass0.succeed("${checkHeapLimitCommand}")
86
87 # Check cluster interaction
88 with subtest("Bring up cluster"):
89 cass1.wait_for_unit("cassandra.service")
90 cass1.wait_until_succeeds(
91 "nodetool -p ${jmxPortStr} ${jmxAuthArgs} status | egrep -c '^UN' | grep 2"
92 )
93 cass0.succeed("nodetool status -p ${jmxPortStr} --resolve-ip | egrep '^UN[[:space:]]+cass1'")
94 '' + lib.optionalString testRemoteAuth ''
95 with subtest("Remote authenticated jmx"):
96 # Doesn't work if not enabled
97 cass0.wait_until_succeeds("nc -z localhost ${jmxPortStr}")
98 cass1.fail("nc -z 192.168.1.1 ${jmxPortStr}")
99 cass1.fail("nodetool -p ${jmxPortStr} -h 192.168.1.1 status")
100
101 # Works if enabled
102 cass1.wait_until_succeeds("nc -z localhost ${jmxPortStr}")
103 cass0.succeed("nodetool -p ${jmxPortStr} -h 192.168.1.2 ${jmxAuthArgs} status")
104 '' + ''
105 with subtest("Break and fix node"):
106 cass1.block()
107 cass0.wait_until_succeeds(
108 "nodetool status -p ${jmxPortStr} --resolve-ip | egrep -c '^DN[[:space:]]+cass1'"
109 )
110 cass0.succeed("nodetool status -p ${jmxPortStr} | egrep -c '^UN' | grep 1")
111 cass1.unblock()
112 cass1.wait_until_succeeds(
113 "nodetool -p ${jmxPortStr} ${jmxAuthArgs} status | egrep -c '^UN' | grep 2"
114 )
115 cass0.succeed("nodetool status -p ${jmxPortStr} | egrep -c '^UN' | grep 2")
116
117 with subtest("Replace crashed node"):
118 cass1.block() # .crash() waits until it's fully shutdown
119 cass2.start()
120 cass0.wait_until_fails(
121 "nodetool status -p ${jmxPortStr} --resolve-ip | egrep '^UN[[:space:]]+cass1'"
122 )
123
124 cass2.wait_for_unit("cassandra.service")
125 cass0.wait_until_succeeds(
126 "nodetool status -p ${jmxPortStr} --resolve-ip | egrep '^UN[[:space:]]+cass2'"
127 )
128 '';
129
130 passthru = {
131 inherit testPackage;
132 };
133})