pyrox.dev
1{ pkgs, ... }:
2let
3 inherit (pkgs) lib;
4 commonConfig = {
5 "druid.zk.service.host" = "zk1:2181";
6 "druid.extensions.loadList" =
7 ''[ "druid-histogram", "druid-datasketches", "mysql-metadata-storage", "druid-avro-extensions", "druid-parquet-extensions", "druid-lookups-cached-global", "druid-hdfs-storage","druid-kafka-indexing-service","druid-basic-security","druid-kinesis-indexing-service"]'';
8 "druid.startup.logging.logProperties" = "true";
9 "druid.metadata.storage.connector.connectURI" = "jdbc:mysql://mysql:3306/druid";
10 "druid.metadata.storage.connector.user" = "druid";
11 "druid.metadata.storage.connector.password" = "druid";
12 "druid.request.logging.type" = "file";
13 "druid.request.logging.dir" = "/var/log/druid/requests";
14 "druid.javascript.enabled" = "true";
15 "druid.sql.enable" = "true";
16 "druid.metadata.storage.type" = "mysql";
17 "druid.storage.type" = "hdfs";
18 "druid.storage.storageDirectory" = "/druid-deepstore";
19 };
20 log4jConfig = ''
21 <?xml version="1.0" encoding="UTF-8" ?>
22 <Configuration status="WARN">
23 <Appenders>
24 <Console name="Console" target="SYSTEM_OUT">
25 <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/>
26 </Console>
27 </Appenders>
28 <Loggers>
29 <Root level="error">
30 <AppenderRef ref="Console"/>
31 </Root>
32 </Loggers>
33 </Configuration>
34 '';
35 log4j = pkgs.writeText "log4j2.xml" log4jConfig;
36 coreSite = {
37 "fs.defaultFS" = "hdfs://namenode:8020";
38 };
39 tests = {
40 default = testsForPackage {
41 druidPackage = pkgs.druid;
42 hadoopPackage = pkgs.hadoop_3_3;
43 };
44 };
45 testsForPackage =
46 args:
47 lib.recurseIntoAttrs {
48 druidCluster = testDruidCluster args;
49 passthru.override = args': testsForPackage (args // args');
50 };
51 testDruidCluster =
52 { druidPackage, hadoopPackage, ... }:
53 pkgs.testers.nixosTest {
54 name = "druid-hdfs";
55 nodes = {
56 zk1 =
57 { ... }:
58 {
59 services.zookeeper.enable = true;
60 networking.firewall.allowedTCPPorts = [ 2181 ];
61 };
62 namenode =
63 { ... }:
64 {
65 services.hadoop = {
66 package = hadoopPackage;
67 hdfs = {
68 namenode = {
69 enable = true;
70 openFirewall = true;
71 formatOnInit = true;
72 };
73 };
74 inherit coreSite;
75 };
76 };
77 datanode =
78 { ... }:
79 {
80 services.hadoop = {
81 package = hadoopPackage;
82 hdfs.datanode = {
83 enable = true;
84 openFirewall = true;
85 };
86 inherit coreSite;
87 };
88 };
89 mm =
90 { ... }:
91 {
92 virtualisation.memorySize = 1024;
93 services.druid = {
94 inherit commonConfig log4j;
95 package = druidPackage;
96 extraClassPaths = [ "/etc/hadoop-conf" ];
97 middleManager = {
98 config = {
99 "druid.indexer.task.baseTaskDir" = "/tmp/druid/persistent/task";
100 "druid.worker.capacity" = 1;
101 "druid.indexer.logs.type" = "file";
102 "druid.indexer.logs.directory" = "/var/log/druid/indexer";
103 "druid.indexer.runner.startPort" = 8100;
104 "druid.indexer.runner.endPort" = 8101;
105 };
106 enable = true;
107 openFirewall = true;
108 };
109 };
110 services.hadoop = {
111 gatewayRole.enable = true;
112 package = hadoopPackage;
113 inherit coreSite;
114 };
115 };
116 overlord =
117 { ... }:
118 {
119 services.druid = {
120 inherit commonConfig log4j;
121 package = druidPackage;
122 extraClassPaths = [ "/etc/hadoop-conf" ];
123 overlord = {
124 config = {
125 "druid.indexer.runner.type" = "remote";
126 "druid.indexer.storage.type" = "metadata";
127 };
128 enable = true;
129 openFirewall = true;
130 };
131 };
132 services.hadoop = {
133 gatewayRole.enable = true;
134 package = hadoopPackage;
135 inherit coreSite;
136 };
137 };
138 broker =
139 { ... }:
140 {
141 services.druid = {
142 package = druidPackage;
143 inherit commonConfig log4j;
144 extraClassPaths = [ "/etc/hadoop-conf" ];
145 broker = {
146 config = {
147 "druid.plaintextPort" = 8082;
148 "druid.broker.http.numConnections" = "2";
149 "druid.server.http.numThreads" = "2";
150 "druid.processing.buffer.sizeBytes" = "100";
151 "druid.processing.numThreads" = "1";
152 "druid.processing.numMergeBuffers" = "1";
153 "druid.broker.cache.unCacheable" = ''["groupBy"]'';
154 "druid.lookup.snapshotWorkingDir" = "/opt/broker/lookups";
155 };
156 enable = true;
157 openFirewall = true;
158 };
159 };
160 services.hadoop = {
161 gatewayRole.enable = true;
162 package = hadoopPackage;
163 inherit coreSite;
164 };
165
166 };
167 historical =
168 { ... }:
169 {
170 services.druid = {
171 package = druidPackage;
172 inherit commonConfig log4j;
173 extraClassPaths = [ "/etc/hadoop-conf" ];
174 historical = {
175 config = {
176 "maxSize" = 200000000;
177 "druid.lookup.snapshotWorkingDir" = "/opt/historical/lookups";
178 };
179 segmentLocations = [
180 {
181 "path" = "/tmp/1";
182 "maxSize" = "100000000";
183 }
184 {
185 "path" = "/tmp/2";
186 "maxSize" = "100000000";
187 }
188 ];
189 enable = true;
190 openFirewall = true;
191 };
192 };
193 services.hadoop = {
194 gatewayRole.enable = true;
195 package = hadoopPackage;
196 inherit coreSite;
197 };
198
199 };
200 coordinator =
201 { ... }:
202 {
203 services.druid = {
204 package = druidPackage;
205 inherit commonConfig log4j;
206 extraClassPaths = [ "/etc/hadoop-conf" ];
207 coordinator = {
208 config = {
209 "druid.plaintextPort" = 9091;
210 "druid.service" = "coordinator";
211 "druid.coordinator.startDelay" = "PT10S";
212 "druid.coordinator.period" = "PT10S";
213 "druid.manager.config.pollDuration" = "PT10S";
214 "druid.manager.segments.pollDuration" = "PT10S";
215 "druid.manager.rules.pollDuration" = "PT10S";
216 };
217 enable = true;
218 openFirewall = true;
219 };
220 };
221 services.hadoop = {
222 gatewayRole.enable = true;
223 package = hadoopPackage;
224 inherit coreSite;
225 };
226
227 };
228
229 mysql =
230 { ... }:
231 {
232 services.mysql = {
233 enable = true;
234 package = pkgs.mariadb;
235 initialDatabases = [ { name = "druid"; } ];
236 initialScript = pkgs.writeText "mysql-init.sql" ''
237 CREATE USER 'druid'@'%' IDENTIFIED BY 'druid';
238 GRANT ALL PRIVILEGES ON druid.* TO 'druid'@'%';
239 '';
240 };
241 networking.firewall.allowedTCPPorts = [ 3306 ];
242 };
243
244 };
245 testScript = ''
246 start_all()
247 namenode.wait_for_unit("hdfs-namenode")
248 namenode.wait_for_unit("network.target")
249 namenode.wait_for_open_port(8020)
250 namenode.succeed("ss -tulpne | systemd-cat")
251 namenode.succeed("cat /etc/hadoop*/hdfs-site.xml | systemd-cat")
252 namenode.wait_for_open_port(9870)
253 datanode.wait_for_unit("hdfs-datanode")
254 datanode.wait_for_unit("network.target")
255
256 mm.succeed("mkdir -p /quickstart/")
257 mm.succeed("cp -r ${pkgs.druid}/quickstart/* /quickstart/")
258 mm.succeed("touch /quickstart/tutorial/wikiticker-2015-09-12-sampled.json")
259 mm.succeed("zcat /quickstart/tutorial/wikiticker-2015-09-12-sampled.json.gz | head -n 10 > /quickstart/tutorial/wikiticker-2015-09-12-sampled.json || true")
260 mm.succeed("rm /quickstart/tutorial/wikiticker-2015-09-12-sampled.json.gz && gzip /quickstart/tutorial/wikiticker-2015-09-12-sampled.json")
261
262 namenode.succeed("sudo -u hdfs hdfs dfs -mkdir /druid-deepstore")
263 namenode.succeed("HADOOP_USER_NAME=druid sudo -u hdfs hdfs dfs -chown druid:hadoop /druid-deepstore")
264
265
266 ### Druid tests
267 coordinator.wait_for_unit("druid-coordinator")
268 overlord.wait_for_unit("druid-overlord")
269 historical.wait_for_unit("druid-historical")
270 mm.wait_for_unit("druid-middleManager")
271
272 coordinator.wait_for_open_port(9091)
273 overlord.wait_for_open_port(8090)
274 historical.wait_for_open_port(8083)
275 mm.wait_for_open_port(8091)
276
277 broker.wait_for_unit("network.target")
278 broker.wait_for_open_port(8082)
279
280 broker.succeed("curl -X 'POST' -H 'Content-Type:application/json' -d @${pkgs.druid}/quickstart/tutorial/wikipedia-index.json http://coordinator:9091/druid/indexer/v1/task")
281 broker.wait_until_succeeds("curl http://coordinator:9091/druid/coordinator/v1/metadata/datasources | grep 'wikipedia'")
282
283 broker.wait_until_succeeds("curl http://localhost:8082/druid/v2/datasources/ | grep wikipedia")
284 broker.succeed("curl -X 'POST' -H 'Content-Type:application/json' -d @${pkgs.druid}/quickstart/tutorial/wikipedia-top-pages.json http://localhost:8082/druid/v2/")
285
286 '';
287
288 };
289in
290tests