pyrox.dev
1# Test a minimal HDFS cluster with no HA
2import ../make-test-python.nix (
3 { package, lib, ... }:
4 {
5 name = "hadoop-hdfs";
6
7 nodes =
8 let
9 coreSite = {
10 "fs.defaultFS" = "hdfs://namenode:8020";
11 "hadoop.proxyuser.httpfs.groups" = "*";
12 "hadoop.proxyuser.httpfs.hosts" = "*";
13 };
14 in
15 {
16 namenode =
17 { pkgs, ... }:
18 {
19 services.hadoop = {
20 inherit package;
21 hdfs = {
22 namenode = {
23 enable = true;
24 openFirewall = true;
25 formatOnInit = true;
26 };
27 httpfs = {
28 # The NixOS hadoop module only support webHDFS on 3.3 and newer
29 enable = lib.mkIf (lib.versionAtLeast package.version "3.3") true;
30 openFirewall = true;
31 };
32 };
33 inherit coreSite;
34 };
35 };
36 datanode =
37 { pkgs, ... }:
38 {
39 virtualisation.diskSize = 4096;
40 services.hadoop = {
41 inherit package;
42 hdfs.datanode = {
43 enable = true;
44 openFirewall = true;
45 dataDirs = [
46 {
47 type = "DISK";
48 path = "/tmp/dn1";
49 }
50 ];
51 };
52 inherit coreSite;
53 };
54 };
55 };
56
57 testScript = ''
58 start_all()
59
60 namenode.wait_for_unit("hdfs-namenode")
61 namenode.wait_for_unit("network.target")
62 namenode.wait_for_open_port(8020)
63 namenode.succeed("systemd-cat ss -tulpne")
64 namenode.succeed("systemd-cat cat /etc/hadoop*/hdfs-site.xml")
65 namenode.wait_for_open_port(9870)
66
67 datanode.wait_for_unit("hdfs-datanode")
68 datanode.wait_for_unit("network.target")
69 ''
70 + (
71 if lib.versionAtLeast package.version "3" then
72 ''
73 datanode.wait_for_open_port(9864)
74 datanode.wait_for_open_port(9866)
75 datanode.wait_for_open_port(9867)
76
77 datanode.succeed("curl -f http://datanode:9864")
78 ''
79 else
80 ''
81 datanode.wait_for_open_port(50075)
82 datanode.wait_for_open_port(50010)
83 datanode.wait_for_open_port(50020)
84
85 datanode.succeed("curl -f http://datanode:50075")
86 ''
87 )
88 + ''
89 namenode.succeed("curl -f http://namenode:9870")
90
91 datanode.succeed("sudo -u hdfs hdfs dfsadmin -safemode wait")
92 datanode.succeed("echo testfilecontents | sudo -u hdfs hdfs dfs -put - /testfile")
93 assert "testfilecontents" in datanode.succeed("sudo -u hdfs hdfs dfs -cat /testfile")
94
95 ''
96 + lib.optionalString (lib.versionAtLeast package.version "3.3") ''
97 namenode.wait_for_unit("hdfs-httpfs")
98 namenode.wait_for_open_port(14000)
99 assert "testfilecontents" in datanode.succeed("curl -f \"http://namenode:14000/webhdfs/v1/testfile?user.name=hdfs&op=OPEN\" 2>&1")
100 '';
101 }
102)