pyrox.dev
1{ config, lib, pkgs, ...}:
2
3with lib;
4let
5 cfg = config.services.hadoop;
6 hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/";
7 mkIfNotNull = x: mkIf (x != null) x;
8in
9{
10 options.services.hadoop = {
11
12 gatewayRole.enableHbaseCli = mkEnableOption (lib.mdDoc "HBase CLI tools");
13
14 hbaseSiteDefault = mkOption {
15 default = {
16 "hbase.regionserver.ipc.address" = "0.0.0.0";
17 "hbase.master.ipc.address" = "0.0.0.0";
18 "hbase.master.info.bindAddress" = "0.0.0.0";
19 "hbase.regionserver.info.bindAddress" = "0.0.0.0";
20
21 "hbase.cluster.distributed" = "true";
22 };
23 type = types.attrsOf types.anything;
24 description = lib.mdDoc ''
25 Default options for hbase-site.xml
26 '';
27 };
28 hbaseSite = mkOption {
29 default = {};
30 type = with types; attrsOf anything;
31 example = literalExpression ''
32 '';
33 description = lib.mdDoc ''
34 Additional options and overrides for hbase-site.xml
35 <https://github.com/apache/hbase/blob/rel/2.4.11/hbase-common/src/main/resources/hbase-default.xml>
36 '';
37 };
38 hbaseSiteInternal = mkOption {
39 default = {};
40 type = with types; attrsOf anything;
41 internal = true;
42 description = lib.mdDoc ''
43 Internal option to add configs to hbase-site.xml based on module options
44 '';
45 };
46
47 hbase = {
48
49 package = mkOption {
50 type = types.package;
51 default = pkgs.hbase;
52 defaultText = literalExpression "pkgs.hbase";
53 description = lib.mdDoc "HBase package";
54 };
55
56 rootdir = mkOption {
57 description = lib.mdDoc ''
58 This option will set "hbase.rootdir" in hbase-site.xml and determine
59 the directory shared by region servers and into which HBase persists.
60 The URL should be 'fully-qualified' to include the filesystem scheme.
61 If a core-site.xml is provided, the FS scheme defaults to the value
62 of "fs.defaultFS".
63
64 Filesystems other than HDFS (like S3, QFS, Swift) are also supported.
65 '';
66 type = types.str;
67 example = "hdfs://nameservice1/hbase";
68 default = "/hbase";
69 };
70 zookeeperQuorum = mkOption {
71 description = lib.mdDoc ''
72 This option will set "hbase.zookeeper.quorum" in hbase-site.xml.
73 Comma separated list of servers in the ZooKeeper ensemble.
74 '';
75 type = with types; nullOr commas;
76 example = "zk1.internal,zk2.internal,zk3.internal";
77 default = null;
78 };
79 master = {
80 enable = mkEnableOption (lib.mdDoc "HBase Master");
81 initHDFS = mkEnableOption (lib.mdDoc "initialization of the hbase directory on HDFS");
82
83 openFirewall = mkOption {
84 type = types.bool;
85 default = false;
86 description = lib.mdDoc ''
87 Open firewall ports for HBase master.
88 '';
89 };
90 };
91 regionServer = {
92 enable = mkEnableOption (lib.mdDoc "HBase RegionServer");
93
94 overrideHosts = mkOption {
95 type = types.bool;
96 default = true;
97 description = lib.mdDoc ''
98 Remove /etc/hosts entries for "127.0.0.2" and "::1" defined in nixos/modules/config/networking.nix
99 Regionservers must be able to resolve their hostnames to their IP addresses, through PTR records
100 or /etc/hosts entries.
101
102 '';
103 };
104
105 openFirewall = mkOption {
106 type = types.bool;
107 default = false;
108 description = lib.mdDoc ''
109 Open firewall ports for HBase master.
110 '';
111 };
112 };
113 };
114 };
115
116 config = mkMerge [
117 (mkIf cfg.hbase.master.enable {
118 services.hadoop.gatewayRole = {
119 enable = true;
120 enableHbaseCli = mkDefault true;
121 };
122
123 systemd.services.hbase-master = {
124 description = "HBase master";
125 wantedBy = [ "multi-user.target" ];
126
127 preStart = mkIf cfg.hbase.master.initHDFS ''
128 HADOOP_USER_NAME=hdfs ${cfg.package}/bin/hdfs --config ${hadoopConf} dfsadmin -safemode wait
129 HADOOP_USER_NAME=hdfs ${cfg.package}/bin/hdfs --config ${hadoopConf} dfs -mkdir -p ${cfg.hbase.rootdir}
130 HADOOP_USER_NAME=hdfs ${cfg.package}/bin/hdfs --config ${hadoopConf} dfs -chown hbase ${cfg.hbase.rootdir}
131 '';
132
133 serviceConfig = {
134 User = "hbase";
135 SyslogIdentifier = "hbase-master";
136 ExecStart = "${cfg.hbase.package}/bin/hbase --config ${hadoopConf} " +
137 "master start";
138 Restart = "always";
139 };
140 };
141
142 services.hadoop.hbaseSiteInternal."hbase.rootdir" = cfg.hbase.rootdir;
143
144 networking.firewall.allowedTCPPorts = (mkIf cfg.hbase.master.openFirewall [
145 16000 16010
146 ]);
147
148 })
149
150 (mkIf cfg.hbase.regionServer.enable {
151 services.hadoop.gatewayRole = {
152 enable = true;
153 enableHbaseCli = mkDefault true;
154 };
155
156 systemd.services.hbase-regionserver = {
157 description = "HBase RegionServer";
158 wantedBy = [ "multi-user.target" ];
159 serviceConfig = {
160 User = "hbase";
161 SyslogIdentifier = "hbase-regionserver";
162 ExecStart = "${cfg.hbase.package}/bin/hbase --config /etc/hadoop-conf/ " +
163 "regionserver start";
164 Restart = "always";
165 };
166 };
167
168 services.hadoop.hbaseSiteInternal."hbase.rootdir" = cfg.hbase.rootdir;
169
170 networking = {
171 firewall.allowedTCPPorts = (mkIf cfg.hbase.regionServer.openFirewall [
172 16020 16030
173 ]);
174 hosts = mkIf cfg.hbase.regionServer.overrideHosts {
175 "127.0.0.2" = mkForce [ ];
176 "::1" = mkForce [ ];
177 };
178 };
179 })
180
181 (mkIf cfg.gatewayRole.enable {
182
183 environment.systemPackages = mkIf cfg.gatewayRole.enableHbaseCli [ cfg.hbase.package ];
184
185 services.hadoop.hbaseSiteInternal = with cfg.hbase; {
186 "hbase.zookeeper.quorum" = mkIfNotNull zookeeperQuorum;
187 };
188
189 users.users.hbase = {
190 description = "Hadoop HBase user";
191 group = "hadoop";
192 isSystemUser = true;
193 };
194 })
195 ];
196}