pyrox.dev
1# This test is very comprehensive. It tests whether all hadoop services work well with each other.
2# Run this when updating the Hadoop package or making significant changes to the hadoop module.
3# For a more basic test, see hdfs.nix and yarn.nix
4import ../make-test-python.nix ({pkgs, ...}: {
5
6 nodes = let
7 package = pkgs.hadoop;
8 coreSite = {
9 "fs.defaultFS" = "hdfs://ns1";
10 };
11 hdfsSite = {
12 "dfs.namenode.rpc-bind-host" = "0.0.0.0";
13 "dfs.namenode.http-bind-host" = "0.0.0.0";
14 "dfs.namenode.servicerpc-bind-host" = "0.0.0.0";
15
16 # HA Quorum Journal Manager configuration
17 "dfs.nameservices" = "ns1";
18 "dfs.ha.namenodes.ns1" = "nn1,nn2";
19 "dfs.namenode.shared.edits.dir.ns1.nn1" = "qjournal://jn1:8485;jn2:8485;jn3:8485/ns1";
20 "dfs.namenode.shared.edits.dir.ns1.nn2" = "qjournal://jn1:8485;jn2:8485;jn3:8485/ns1";
21 "dfs.namenode.rpc-address.ns1.nn1" = "nn1:8020";
22 "dfs.namenode.rpc-address.ns1.nn2" = "nn2:8020";
23 "dfs.namenode.servicerpc-address.ns1.nn1" = "nn1:8022";
24 "dfs.namenode.servicerpc-address.ns1.nn2" = "nn2:8022";
25 "dfs.namenode.http-address.ns1.nn1" = "nn1:9870";
26 "dfs.namenode.http-address.ns1.nn2" = "nn2:9870";
27
28 # Automatic failover configuration
29 "dfs.client.failover.proxy.provider.ns1" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider";
30 "dfs.ha.automatic-failover.enabled.ns1" = "true";
31 "dfs.ha.fencing.methods" = "shell(true)";
32 "ha.zookeeper.quorum" = "zk1:2181";
33 };
34 yarnSiteHA = {
35 "yarn.resourcemanager.zk-address" = "zk1:2181";
36 "yarn.resourcemanager.ha.enabled" = "true";
37 "yarn.resourcemanager.ha.rm-ids" = "rm1,rm2";
38 "yarn.resourcemanager.hostname.rm1" = "rm1";
39 "yarn.resourcemanager.hostname.rm2" = "rm2";
40 "yarn.resourcemanager.ha.automatic-failover.enabled" = "true";
41 "yarn.resourcemanager.cluster-id" = "cluster1";
42 # yarn.resourcemanager.webapp.address needs to be defined even though yarn.resourcemanager.hostname is set. This shouldn't be necessary, but there's a bug in
43 # hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java:70
44 # that causes AM containers to fail otherwise.
45 "yarn.resourcemanager.webapp.address.rm1" = "rm1:8088";
46 "yarn.resourcemanager.webapp.address.rm2" = "rm2:8088";
47 };
48 in {
49 zk1 = { ... }: {
50 services.zookeeper.enable = true;
51 networking.firewall.allowedTCPPorts = [ 2181 ];
52 };
53
54 # HDFS cluster
55 nn1 = {pkgs, options, ...}: {
56 services.hadoop = {
57 inherit package coreSite hdfsSite;
58 hdfs.namenode.enable = true;
59 hdfs.zkfc.enable = true;
60 };
61 };
62 nn2 = {pkgs, options, ...}: {
63 services.hadoop = {
64 inherit package coreSite hdfsSite;
65 hdfs.namenode.enable = true;
66 hdfs.zkfc.enable = true;
67 };
68 };
69
70 jn1 = {pkgs, options, ...}: {
71 services.hadoop = {
72 inherit package coreSite hdfsSite;
73 hdfs.journalnode.enable = true;
74 };
75 };
76 jn2 = {pkgs, options, ...}: {
77 services.hadoop = {
78 inherit package coreSite hdfsSite;
79 hdfs.journalnode.enable = true;
80 };
81 };
82 jn3 = {pkgs, options, ...}: {
83 services.hadoop = {
84 inherit package coreSite hdfsSite;
85 hdfs.journalnode.enable = true;
86 };
87 };
88
89 dn1 = {pkgs, options, ...}: {
90 services.hadoop = {
91 inherit package coreSite hdfsSite;
92 hdfs.datanode.enable = true;
93 };
94 };
95
96 # YARN cluster
97 rm1 = {pkgs, options, ...}: {
98 services.hadoop = {
99 inherit package coreSite hdfsSite;
100 yarnSite = options.services.hadoop.yarnSite.default // yarnSiteHA;
101 yarn.resourcemanager.enable = true;
102 };
103 };
104 rm2 = {pkgs, options, ...}: {
105 services.hadoop = {
106 inherit package coreSite hdfsSite;
107 yarnSite = options.services.hadoop.yarnSite.default // yarnSiteHA;
108 yarn.resourcemanager.enable = true;
109 };
110 };
111 nm1 = {pkgs, options, ...}: {
112 virtualisation.memorySize = 2048;
113 services.hadoop = {
114 inherit package coreSite hdfsSite;
115 yarnSite = options.services.hadoop.yarnSite.default // yarnSiteHA;
116 yarn.nodemanager.enable = true;
117 };
118 };
119 };
120
121 testScript = ''
122 start_all()
123
124 #### HDFS tests ####
125
126 zk1.wait_for_unit("network.target")
127 jn1.wait_for_unit("network.target")
128 jn2.wait_for_unit("network.target")
129 jn3.wait_for_unit("network.target")
130 nn1.wait_for_unit("network.target")
131 nn2.wait_for_unit("network.target")
132 dn1.wait_for_unit("network.target")
133
134 zk1.wait_for_unit("zookeeper")
135 jn1.wait_for_unit("hdfs-journalnode")
136 jn2.wait_for_unit("hdfs-journalnode")
137 jn3.wait_for_unit("hdfs-journalnode")
138
139 zk1.wait_for_open_port(2181)
140 jn1.wait_for_open_port(8480)
141 jn1.wait_for_open_port(8485)
142 jn2.wait_for_open_port(8480)
143 jn2.wait_for_open_port(8485)
144
145 # Namenodes must be stopped before initializing the cluster
146 nn1.succeed("systemctl stop hdfs-namenode")
147 nn2.succeed("systemctl stop hdfs-namenode")
148 nn1.succeed("systemctl stop hdfs-zkfc")
149 nn2.succeed("systemctl stop hdfs-zkfc")
150
151 # Initialize zookeeper for failover controller
152 nn1.succeed("sudo -u hdfs hdfs zkfc -formatZK 2>&1 | systemd-cat")
153
154 # Format NN1 and start it
155 nn1.succeed("sudo -u hdfs hadoop namenode -format 2>&1 | systemd-cat")
156 nn1.succeed("systemctl start hdfs-namenode")
157 nn1.wait_for_open_port(9870)
158 nn1.wait_for_open_port(8022)
159 nn1.wait_for_open_port(8020)
160
161 # Bootstrap NN2 from NN1 and start it
162 nn2.succeed("sudo -u hdfs hdfs namenode -bootstrapStandby 2>&1 | systemd-cat")
163 nn2.succeed("systemctl start hdfs-namenode")
164 nn2.wait_for_open_port(9870)
165 nn2.wait_for_open_port(8022)
166 nn2.wait_for_open_port(8020)
167 nn1.succeed("netstat -tulpne | systemd-cat")
168
169 # Start failover controllers
170 nn1.succeed("systemctl start hdfs-zkfc")
171 nn2.succeed("systemctl start hdfs-zkfc")
172
173 # DN should have started by now, but confirm anyway
174 dn1.wait_for_unit("hdfs-datanode")
175 # Print states of namenodes
176 dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")
177 # Wait for cluster to exit safemode
178 dn1.succeed("sudo -u hdfs hdfs dfsadmin -safemode wait")
179 dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")
180 # test R/W
181 dn1.succeed("echo testfilecontents | sudo -u hdfs hdfs dfs -put - /testfile")
182 assert "testfilecontents" in dn1.succeed("sudo -u hdfs hdfs dfs -cat /testfile")
183
184 # Test NN failover
185 nn1.succeed("systemctl stop hdfs-namenode")
186 assert "active" in dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState")
187 dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")
188 assert "testfilecontents" in dn1.succeed("sudo -u hdfs hdfs dfs -cat /testfile")
189
190 nn1.succeed("systemctl start hdfs-namenode")
191 nn1.wait_for_open_port(9870)
192 nn1.wait_for_open_port(8022)
193 nn1.wait_for_open_port(8020)
194 assert "standby" in dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState")
195 dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")
196
197 #### YARN tests ####
198
199 rm1.wait_for_unit("network.target")
200 rm2.wait_for_unit("network.target")
201 nm1.wait_for_unit("network.target")
202
203 rm1.wait_for_unit("yarn-resourcemanager")
204 rm1.wait_for_open_port(8088)
205 rm2.wait_for_unit("yarn-resourcemanager")
206 rm2.wait_for_open_port(8088)
207
208 nm1.wait_for_unit("yarn-nodemanager")
209 nm1.wait_for_open_port(8042)
210 nm1.wait_for_open_port(8040)
211 nm1.wait_until_succeeds("yarn node -list | grep Nodes:1")
212 nm1.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat")
213 nm1.succeed("sudo -u yarn yarn node -list | systemd-cat")
214
215 # Test RM failover
216 rm1.succeed("systemctl stop yarn-resourcemanager")
217 assert "standby" not in nm1.succeed("sudo -u yarn yarn rmadmin -getAllServiceState")
218 nm1.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat")
219 rm1.succeed("systemctl start yarn-resourcemanager")
220 rm1.wait_for_unit("yarn-resourcemanager")
221 rm1.wait_for_open_port(8088)
222 assert "standby" in nm1.succeed("sudo -u yarn yarn rmadmin -getAllServiceState")
223 nm1.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat")
224
225 assert "Estimated value of Pi is" in nm1.succeed("HADOOP_USER_NAME=hdfs yarn jar $(readlink $(which yarn) | sed -r 's~bin/yarn~lib/hadoop-*/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar~g') pi 2 10")
226 assert "SUCCEEDED" in nm1.succeed("yarn application -list -appStates FINISHED")
227 '';
228})