pyrox.dev
1{
2 lib,
3 pkgs,
4 package,
5 ...
6}:
7rec {
8 name = "clickhouse-keeper";
9 meta.maintainers = with pkgs.lib.maintainers; [
10 jpds
11 thevar1able
12 ];
13
14 nodes =
15 let
16 node = i: {
17
18 environment.etc = {
19 "clickhouse-server/config.d/cluster.xml".text = ''
20 <clickhouse>
21 <remote_servers>
22 <perftest_2shards_1replicas>
23 ${lib.concatStrings (
24 lib.imap0 (j: name: ''
25 <shard>
26 <replica>
27 <host>${name}</host>
28 <port>9000</port>
29 </replica>
30 </shard>
31 '') (builtins.attrNames nodes)
32 )}
33 </perftest_2shards_1replicas>
34 </remote_servers>
35 </clickhouse>
36 '';
37
38 "clickhouse-server/config.d/keeper.xml".text = ''
39 <clickhouse>
40 <keeper_server>
41 <server_id>${toString i}</server_id>
42 <tcp_port>9181</tcp_port>
43 <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
44 <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
45
46 <coordination_settings>
47 <operation_timeout_ms>10000</operation_timeout_ms>
48 <session_timeout_ms>30000</session_timeout_ms>
49 <raft_logs_level>trace</raft_logs_level>
50 <rotate_log_storage_interval>10000</rotate_log_storage_interval>
51 </coordination_settings>
52
53 <raft_configuration>
54 ${lib.concatStrings (
55 lib.imap1 (j: name: ''
56 <server>
57 <id>${toString j}</id>
58 <hostname>${name}</hostname>
59 <port>9444</port>
60 </server>
61 '') (builtins.attrNames nodes)
62 )}
63 </raft_configuration>
64 </keeper_server>
65
66 <zookeeper>
67 ${lib.concatStrings (
68 lib.imap0 (j: name: ''
69 <node>
70 <host>${name}</host>
71 <port>9181</port>
72 </node>
73 '') (builtins.attrNames nodes)
74 )}
75 </zookeeper>
76
77 <distributed_ddl>
78 <path>/clickhouse/testcluster/task_queue/ddl</path>
79 </distributed_ddl>
80 </clickhouse>
81 '';
82
83 "clickhouse-server/config.d/listen.xml".text = ''
84 <clickhouse>
85 <listen_host>::</listen_host>
86 </clickhouse>
87 '';
88
89 "clickhouse-server/config.d/macros.xml".text = ''
90 <clickhouse>
91 <macros>
92 <replica>${toString i}</replica>
93 <cluster>perftest_2shards_1replicas</cluster>
94 </macros>
95 </clickhouse>
96 '';
97 };
98
99 networking.firewall.allowedTCPPorts = [
100 9009
101 9181
102 9444
103 ];
104
105 services.clickhouse = {
106 enable = true;
107 inherit package;
108 };
109
110 systemd.services.clickhouse = {
111 after = [ "network-online.target" ];
112 requires = [ "network-online.target" ];
113 };
114
115 virtualisation.memorySize = 1024 * 4;
116 virtualisation.diskSize = 1024 * 10;
117 };
118 in
119 {
120 clickhouse1 = node 1;
121 clickhouse2 = node 2;
122 };
123
124 testScript =
125 let
126 # work around quote/substitution complexity by Nix, Perl, bash and SQL.
127 clustersQuery = pkgs.writeText "clusters.sql" "SHOW clusters";
128 keeperQuery = pkgs.writeText "keeper.sql" "SELECT * FROM system.zookeeper WHERE path IN ('/', '/clickhouse') FORMAT VERTICAL";
129 systemClustersQuery = pkgs.writeText "system-clusters.sql" "SELECT host_name, host_address, replica_num FROM system.clusters WHERE cluster = 'perftest_2shards_1replicas'";
130
131 tableDDL = pkgs.writeText "table.sql" ''
132 CREATE TABLE test ON cluster 'perftest_2shards_1replicas' ( A Int64, S String)
133 Engine = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{database}/{table}', '{replica}')
134 ORDER BY A;
135 '';
136
137 insertDDL = pkgs.writeText "insert.sql" "
138 INSERT INTO test SELECT number, '' FROM numbers(100000000);
139 ";
140
141 selectCountQuery = pkgs.writeText "select-count.sql" "
142 select count() from test;
143 ";
144 in
145 ''
146 clickhouse1.start()
147 clickhouse2.start()
148
149 for machine in clickhouse1, clickhouse2:
150 machine.wait_for_unit("clickhouse.service")
151 machine.wait_for_open_port(9000)
152 machine.wait_for_open_port(9009)
153 machine.wait_for_open_port(9181)
154 machine.wait_for_open_port(9444)
155
156 machine.wait_until_succeeds(
157 """
158 journalctl -o cat -u clickhouse.service | grep "Merging configuration file '/etc/clickhouse-server/config.d/keeper.xml'"
159 """
160 )
161
162 machine.log(machine.succeed(
163 "cat ${clustersQuery} | clickhouse-client | grep perftest_2shards_1replicas"
164 ))
165
166 machine.log(machine.succeed(
167 "cat ${keeperQuery} | clickhouse-client"
168 ))
169
170 machine.succeed(
171 "cat ${systemClustersQuery} | clickhouse-client | grep clickhouse1"
172 )
173 machine.succeed(
174 "cat ${systemClustersQuery} | clickhouse-client | grep clickhouse2"
175 )
176
177 machine.succeed(
178 "ls /var/lib/clickhouse/coordination/log | grep changelog"
179 )
180
181 clickhouse2.succeed(
182 "cat ${tableDDL} | clickhouse-client"
183 )
184
185 clickhouse2.succeed(
186 "cat ${insertDDL} | clickhouse-client"
187 )
188
189 for machine in clickhouse1, clickhouse2:
190 machine.wait_until_succeeds(
191 "cat ${selectCountQuery} | clickhouse-client | grep 100000000"
192 )
193 '';
194}