pyrox.dev
1{
2 config,
3 lib,
4 options,
5 pkgs,
6 ...
7}:
8let
9 cfg = config.services.hadoop;
10 opt = options.services.hadoop;
11in
12{
13 imports = [
14 ./yarn.nix
15 ./hdfs.nix
16 ./hbase.nix
17 ];
18
19 options.services.hadoop = {
20 coreSite = lib.mkOption {
21 default = { };
22 type = lib.types.attrsOf lib.types.anything;
23 example = lib.literalExpression ''
24 {
25 "fs.defaultFS" = "hdfs://localhost";
26 }
27 '';
28 description = ''
29 Hadoop core-site.xml definition
30 <https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/core-default.xml>
31 '';
32 };
33 coreSiteInternal = lib.mkOption {
34 default = { };
35 type = lib.types.attrsOf lib.types.anything;
36 internal = true;
37 description = ''
38 Internal option to add configs to core-site.xml based on module options
39 '';
40 };
41
42 hdfsSiteDefault = lib.mkOption {
43 default = {
44 "dfs.namenode.rpc-bind-host" = "0.0.0.0";
45 "dfs.namenode.http-address" = "0.0.0.0:9870";
46 "dfs.namenode.servicerpc-bind-host" = "0.0.0.0";
47 "dfs.namenode.http-bind-host" = "0.0.0.0";
48 };
49 type = lib.types.attrsOf lib.types.anything;
50 description = ''
51 Default options for hdfs-site.xml
52 '';
53 };
54 hdfsSite = lib.mkOption {
55 default = { };
56 type = lib.types.attrsOf lib.types.anything;
57 example = lib.literalExpression ''
58 {
59 "dfs.nameservices" = "namenode1";
60 }
61 '';
62 description = ''
63 Additional options and overrides for hdfs-site.xml
64 <https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/hdfs-default.xml>
65 '';
66 };
67 hdfsSiteInternal = lib.mkOption {
68 default = { };
69 type = lib.types.attrsOf lib.types.anything;
70 internal = true;
71 description = ''
72 Internal option to add configs to hdfs-site.xml based on module options
73 '';
74 };
75
76 mapredSiteDefault = lib.mkOption {
77 default = {
78 "mapreduce.framework.name" = "yarn";
79 "yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=${cfg.package}";
80 "mapreduce.map.env" = "HADOOP_MAPRED_HOME=${cfg.package}";
81 "mapreduce.reduce.env" = "HADOOP_MAPRED_HOME=${cfg.package}";
82 };
83 defaultText = lib.literalExpression ''
84 {
85 "mapreduce.framework.name" = "yarn";
86 "yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}";
87 "mapreduce.map.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}";
88 "mapreduce.reduce.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}";
89 }
90 '';
91 type = lib.types.attrsOf lib.types.anything;
92 description = ''
93 Default options for mapred-site.xml
94 '';
95 };
96 mapredSite = lib.mkOption {
97 default = { };
98 type = lib.types.attrsOf lib.types.anything;
99 example = lib.literalExpression ''
100 {
101 "mapreduce.map.java.opts" = "-Xmx900m -XX:+UseParallelGC";
102 }
103 '';
104 description = ''
105 Additional options and overrides for mapred-site.xml
106 <https://hadoop.apache.org/docs/current/hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml>
107 '';
108 };
109
110 yarnSiteDefault = lib.mkOption {
111 default = {
112 "yarn.nodemanager.admin-env" = "PATH=$PATH";
113 "yarn.nodemanager.aux-services" = "mapreduce_shuffle";
114 "yarn.nodemanager.aux-services.mapreduce_shuffle.class" = "org.apache.hadoop.mapred.ShuffleHandler";
115 "yarn.nodemanager.bind-host" = "0.0.0.0";
116 "yarn.nodemanager.container-executor.class" =
117 "org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor";
118 "yarn.nodemanager.env-whitelist" =
119 "JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,LANG,TZ";
120 "yarn.nodemanager.linux-container-executor.group" = "hadoop";
121 "yarn.nodemanager.linux-container-executor.path" =
122 "/run/wrappers/yarn-nodemanager/bin/container-executor";
123 "yarn.nodemanager.log-dirs" = "/var/log/hadoop/yarn/nodemanager";
124 "yarn.resourcemanager.bind-host" = "0.0.0.0";
125 "yarn.resourcemanager.scheduler.class" =
126 "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler";
127 };
128 type = lib.types.attrsOf lib.types.anything;
129 description = ''
130 Default options for yarn-site.xml
131 '';
132 };
133 yarnSite = lib.mkOption {
134 default = { };
135 type = lib.types.attrsOf lib.types.anything;
136 example = lib.literalExpression ''
137 {
138 "yarn.resourcemanager.hostname" = "''${config.networking.hostName}";
139 }
140 '';
141 description = ''
142 Additional options and overrides for yarn-site.xml
143 <https://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-common/yarn-default.xml>
144 '';
145 };
146 yarnSiteInternal = lib.mkOption {
147 default = { };
148 type = lib.types.attrsOf lib.types.anything;
149 internal = true;
150 description = ''
151 Internal option to add configs to yarn-site.xml based on module options
152 '';
153 };
154
155 httpfsSite = lib.mkOption {
156 default = { };
157 type = lib.types.attrsOf lib.types.anything;
158 example = lib.literalExpression ''
159 {
160 "hadoop.http.max.threads" = 500;
161 }
162 '';
163 description = ''
164 Hadoop httpfs-site.xml definition
165 <https://hadoop.apache.org/docs/current/hadoop-hdfs-httpfs/httpfs-default.html>
166 '';
167 };
168
169 log4jProperties = lib.mkOption {
170 default = "${cfg.package}/etc/hadoop/log4j.properties";
171 defaultText = lib.literalExpression ''
172 "''${config.${opt.package}}/etc/hadoop/log4j.properties"
173 '';
174 type = lib.types.path;
175 example = lib.literalExpression ''
176 "''${pkgs.hadoop}/etc/hadoop/log4j.properties";
177 '';
178 description = "log4j.properties file added to HADOOP_CONF_DIR";
179 };
180
181 containerExecutorCfg = lib.mkOption {
182 default = {
183 # must be the same as yarn.nodemanager.linux-container-executor.group in yarnSite
184 "yarn.nodemanager.linux-container-executor.group" = "hadoop";
185 "min.user.id" = 1000;
186 "feature.terminal.enabled" = 1;
187 "feature.mount-cgroup.enabled" = 1;
188 };
189 type = lib.types.attrsOf lib.types.anything;
190 example = lib.literalExpression ''
191 options.services.hadoop.containerExecutorCfg.default // {
192 "feature.terminal.enabled" = 0;
193 }
194 '';
195 description = ''
196 Yarn container-executor.cfg definition
197 <https://hadoop.apache.org/docs/r2.7.2/hadoop-yarn/hadoop-yarn-site/SecureContainer.html>
198 '';
199 };
200
201 extraConfDirs = lib.mkOption {
202 default = [ ];
203 type = lib.types.listOf lib.types.path;
204 example = lib.literalExpression ''
205 [
206 ./extraHDFSConfs
207 ./extraYARNConfs
208 ]
209 '';
210 description = "Directories containing additional config files to be added to HADOOP_CONF_DIR";
211 };
212
213 gatewayRole.enable = lib.mkEnableOption "gateway role for deploying hadoop configs";
214
215 package = lib.mkPackageOption pkgs "hadoop" { };
216 };
217
218 config = lib.mkIf cfg.gatewayRole.enable {
219 users.groups.hadoop = {
220 gid = config.ids.gids.hadoop;
221 };
222 environment = {
223 systemPackages = [ cfg.package ];
224 etc."hadoop-conf".source =
225 let
226 hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/";
227 in
228 "${hadoopConf}";
229 variables.HADOOP_CONF_DIR = "/etc/hadoop-conf/";
230 };
231 };
232}