pyrox.dev
1# Test a minimal HDFS cluster with no HA
2import ../make-test-python.nix (
3 { package, lib, ... }:
4 {
5 name = "hadoop-hdfs";
6
7 nodes =
8 let
9 coreSite = {
10 "fs.defaultFS" = "hdfs://namenode:8020";
11 "hadoop.proxyuser.httpfs.groups" = "*";
12 "hadoop.proxyuser.httpfs.hosts" = "*";
13 };
14 in
15 {
16 namenode =
17 { pkgs, ... }:
18 {
19 services.hadoop = {
20 inherit package;
21 hdfs = {
22 namenode = {
23 enable = true;
24 openFirewall = true;
25 formatOnInit = true;
26 };
27 httpfs = {
28 # The NixOS hadoop module only support webHDFS on 3.3 and newer
29 enable = lib.mkIf (lib.versionAtLeast package.version "3.3") true;
30 openFirewall = true;
31 };
32 };
33 inherit coreSite;
34 };
35 };
36 datanode =
37 { pkgs, ... }:
38 {
39 virtualisation.diskSize = 4096;
40 services.hadoop = {
41 inherit package;
42 hdfs.datanode = {
43 enable = true;
44 openFirewall = true;
45 dataDirs = [
46 {
47 type = "DISK";
48 path = "/tmp/dn1";
49 }
50 ];
51 };
52 inherit coreSite;
53 };
54 };
55 };
56
57 testScript =
58 ''
59 start_all()
60
61 namenode.wait_for_unit("hdfs-namenode")
62 namenode.wait_for_unit("network.target")
63 namenode.wait_for_open_port(8020)
64 namenode.succeed("systemd-cat ss -tulpne")
65 namenode.succeed("systemd-cat cat /etc/hadoop*/hdfs-site.xml")
66 namenode.wait_for_open_port(9870)
67
68 datanode.wait_for_unit("hdfs-datanode")
69 datanode.wait_for_unit("network.target")
70 ''
71 + (
72 if lib.versionAtLeast package.version "3" then
73 ''
74 datanode.wait_for_open_port(9864)
75 datanode.wait_for_open_port(9866)
76 datanode.wait_for_open_port(9867)
77
78 datanode.succeed("curl -f http://datanode:9864")
79 ''
80 else
81 ''
82 datanode.wait_for_open_port(50075)
83 datanode.wait_for_open_port(50010)
84 datanode.wait_for_open_port(50020)
85
86 datanode.succeed("curl -f http://datanode:50075")
87 ''
88 )
89 + ''
90 namenode.succeed("curl -f http://namenode:9870")
91
92 datanode.succeed("sudo -u hdfs hdfs dfsadmin -safemode wait")
93 datanode.succeed("echo testfilecontents | sudo -u hdfs hdfs dfs -put - /testfile")
94 assert "testfilecontents" in datanode.succeed("sudo -u hdfs hdfs dfs -cat /testfile")
95
96 ''
97 + lib.optionalString (lib.versionAtLeast package.version "3.3") ''
98 namenode.wait_for_unit("hdfs-httpfs")
99 namenode.wait_for_open_port(14000)
100 assert "testfilecontents" in datanode.succeed("curl -f \"http://namenode:14000/webhdfs/v1/testfile?user.name=hdfs&op=OPEN\" 2>&1")
101 '';
102 }
103)