pyrox.dev
1{ config, lib, pkgs, ...}:
2with lib;
3let
4 cfg = config.services.hadoop;
5 hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/";
6 restartIfChanged = mkOption {
7 type = types.bool;
8 description = ''
9 Automatically restart the service on config change.
10 This can be set to false to defer restarts on clusters running critical applications.
11 Please consider the security implications of inadvertently running an older version,
12 and the possibility of unexpected behavior caused by inconsistent versions across a cluster when disabling this option.
13 '';
14 default = false;
15 };
16 extraFlags = mkOption{
17 type = with types; listOf str;
18 default = [];
19 description = "Extra command line flags to pass to the service";
20 example = [
21 "-Dcom.sun.management.jmxremote"
22 "-Dcom.sun.management.jmxremote.port=8010"
23 ];
24 };
25 extraEnv = mkOption{
26 type = with types; attrsOf str;
27 default = {};
28 description = "Extra environment variables";
29 };
30in
31{
32 options.services.hadoop.yarn = {
33 resourcemanager = {
34 enable = mkEnableOption "Hadoop YARN ResourceManager";
35 inherit restartIfChanged extraFlags extraEnv;
36
37 openFirewall = mkOption {
38 type = types.bool;
39 default = false;
40 description = ''
41 Open firewall ports for resourcemanager
42 '';
43 };
44 };
45 nodemanager = {
46 enable = mkEnableOption "Hadoop YARN NodeManager";
47 inherit restartIfChanged extraFlags extraEnv;
48
49 resource = {
50 cpuVCores = mkOption {
51 description = "Number of vcores that can be allocated for containers.";
52 type = with types; nullOr ints.positive;
53 default = null;
54 };
55 maximumAllocationVCores = mkOption {
56 description = "The maximum virtual CPU cores any container can be allocated.";
57 type = with types; nullOr ints.positive;
58 default = null;
59 };
60 memoryMB = mkOption {
61 description = "Amount of physical memory, in MB, that can be allocated for containers.";
62 type = with types; nullOr ints.positive;
63 default = null;
64 };
65 maximumAllocationMB = mkOption {
66 description = "The maximum physical memory any container can be allocated.";
67 type = with types; nullOr ints.positive;
68 default = null;
69 };
70 };
71
72 useCGroups = mkOption {
73 type = types.bool;
74 default = true;
75 description = ''
76 Use cgroups to enforce resource limits on containers
77 '';
78 };
79
80 localDir = mkOption {
81 description = "List of directories to store localized files in.";
82 type = with types; nullOr (listOf path);
83 example = [ "/var/lib/hadoop/yarn/nm" ];
84 default = null;
85 };
86
87 addBinBash = mkOption {
88 type = types.bool;
89 default = true;
90 description = ''
91 Add /bin/bash. This is needed by the linux container executor's launch script.
92 '';
93 };
94 openFirewall = mkOption {
95 type = types.bool;
96 default = false;
97 description = ''
98 Open firewall ports for nodemanager.
99 Because containers can listen on any ephemeral port, TCP ports 1024–65535 will be opened.
100 '';
101 };
102 };
103 };
104
105 config = mkMerge [
106 (mkIf cfg.gatewayRole.enable {
107 users.users.yarn = {
108 description = "Hadoop YARN user";
109 group = "hadoop";
110 uid = config.ids.uids.yarn;
111 };
112 })
113
114 (mkIf cfg.yarn.resourcemanager.enable {
115 systemd.services.yarn-resourcemanager = {
116 description = "Hadoop YARN ResourceManager";
117 wantedBy = [ "multi-user.target" ];
118 inherit (cfg.yarn.resourcemanager) restartIfChanged;
119 environment = cfg.yarn.resourcemanager.extraEnv;
120
121 serviceConfig = {
122 User = "yarn";
123 SyslogIdentifier = "yarn-resourcemanager";
124 ExecStart = "${cfg.package}/bin/yarn --config ${hadoopConf} " +
125 " resourcemanager ${escapeShellArgs cfg.yarn.resourcemanager.extraFlags}";
126 Restart = "always";
127 };
128 };
129
130 services.hadoop.gatewayRole.enable = true;
131
132 networking.firewall.allowedTCPPorts = (mkIf cfg.yarn.resourcemanager.openFirewall [
133 8088 # resourcemanager.webapp.address
134 8030 # resourcemanager.scheduler.address
135 8031 # resourcemanager.resource-tracker.address
136 8032 # resourcemanager.address
137 8033 # resourcemanager.admin.address
138 ]);
139 })
140
141 (mkIf cfg.yarn.nodemanager.enable {
142 # Needed because yarn hardcodes /bin/bash in container start scripts
143 # These scripts can't be patched, they are generated at runtime
144 systemd.tmpfiles.rules = [
145 (mkIf cfg.yarn.nodemanager.addBinBash "L /bin/bash - - - - /run/current-system/sw/bin/bash")
146 ];
147
148 systemd.services.yarn-nodemanager = {
149 description = "Hadoop YARN NodeManager";
150 wantedBy = [ "multi-user.target" ];
151 inherit (cfg.yarn.nodemanager) restartIfChanged;
152 environment = cfg.yarn.nodemanager.extraEnv;
153
154 preStart = ''
155 # create log dir
156 mkdir -p /var/log/hadoop/yarn/nodemanager
157 chown yarn:hadoop /var/log/hadoop/yarn/nodemanager
158
159 # set up setuid container executor binary
160 umount /run/wrappers/yarn-nodemanager/cgroup/cpu || true
161 rm -rf /run/wrappers/yarn-nodemanager/ || true
162 mkdir -p /run/wrappers/yarn-nodemanager/{bin,etc/hadoop,cgroup/cpu}
163 cp ${cfg.package}/bin/container-executor /run/wrappers/yarn-nodemanager/bin/
164 chgrp hadoop /run/wrappers/yarn-nodemanager/bin/container-executor
165 chmod 6050 /run/wrappers/yarn-nodemanager/bin/container-executor
166 cp ${hadoopConf}/container-executor.cfg /run/wrappers/yarn-nodemanager/etc/hadoop/
167 '';
168
169 serviceConfig = {
170 User = "yarn";
171 SyslogIdentifier = "yarn-nodemanager";
172 PermissionsStartOnly = true;
173 ExecStart = "${cfg.package}/bin/yarn --config ${hadoopConf} " +
174 " nodemanager ${escapeShellArgs cfg.yarn.nodemanager.extraFlags}";
175 Restart = "always";
176 };
177 };
178
179 services.hadoop.gatewayRole.enable = true;
180
181 services.hadoop.yarnSiteInternal = with cfg.yarn.nodemanager; mkMerge [ ({
182 "yarn.nodemanager.local-dirs" = mkIf (localDir!= null) (concatStringsSep "," localDir);
183 "yarn.scheduler.maximum-allocation-vcores" = resource.maximumAllocationVCores;
184 "yarn.scheduler.maximum-allocation-mb" = resource.maximumAllocationMB;
185 "yarn.nodemanager.resource.cpu-vcores" = resource.cpuVCores;
186 "yarn.nodemanager.resource.memory-mb" = resource.memoryMB;
187 }) (mkIf useCGroups {
188 "yarn.nodemanager.linux-container-executor.cgroups.hierarchy" = "/hadoop-yarn";
189 "yarn.nodemanager.linux-container-executor.resources-handler.class" = "org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler";
190 "yarn.nodemanager.linux-container-executor.cgroups.mount" = "true";
191 "yarn.nodemanager.linux-container-executor.cgroups.mount-path" = "/run/wrappers/yarn-nodemanager/cgroup";
192 })];
193
194 networking.firewall.allowedTCPPortRanges = [
195 (mkIf (cfg.yarn.nodemanager.openFirewall) {from = 1024; to = 65535;})
196 ];
197 })
198
199 ];
200}