pyrox.dev
1{
2 config,
3 lib,
4 pkgs,
5 ...
6}:
7let
8 cfg = config.services.hadoop;
9 hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/";
10 restartIfChanged = lib.mkOption {
11 type = lib.types.bool;
12 description = ''
13 Automatically restart the service on config change.
14 This can be set to false to defer restarts on clusters running critical applications.
15 Please consider the security implications of inadvertently running an older version,
16 and the possibility of unexpected behavior caused by inconsistent versions across a cluster when disabling this option.
17 '';
18 default = false;
19 };
20 extraFlags = lib.mkOption {
21 type = with lib.types; listOf str;
22 default = [ ];
23 description = "Extra command line flags to pass to the service";
24 example = [
25 "-Dcom.sun.management.jmxremote"
26 "-Dcom.sun.management.jmxremote.port=8010"
27 ];
28 };
29 extraEnv = lib.mkOption {
30 type = with lib.types; attrsOf str;
31 default = { };
32 description = "Extra environment variables";
33 };
34in
35{
36 options.services.hadoop.yarn = {
37 resourcemanager = {
38 enable = lib.mkEnableOption "Hadoop YARN ResourceManager";
39 inherit restartIfChanged extraFlags extraEnv;
40
41 openFirewall = lib.mkOption {
42 type = lib.types.bool;
43 default = false;
44 description = ''
45 Open firewall ports for resourcemanager
46 '';
47 };
48 };
49 nodemanager = {
50 enable = lib.mkEnableOption "Hadoop YARN NodeManager";
51 inherit restartIfChanged extraFlags extraEnv;
52
53 resource = {
54 cpuVCores = lib.mkOption {
55 description = "Number of vcores that can be allocated for containers.";
56 type = with lib.types; nullOr ints.positive;
57 default = null;
58 };
59 maximumAllocationVCores = lib.mkOption {
60 description = "The maximum virtual CPU cores any container can be allocated.";
61 type = with lib.types; nullOr ints.positive;
62 default = null;
63 };
64 memoryMB = lib.mkOption {
65 description = "Amount of physical memory, in MB, that can be allocated for containers.";
66 type = with lib.types; nullOr ints.positive;
67 default = null;
68 };
69 maximumAllocationMB = lib.mkOption {
70 description = "The maximum physical memory any container can be allocated.";
71 type = with lib.types; nullOr ints.positive;
72 default = null;
73 };
74 };
75
76 useCGroups = lib.mkOption {
77 type = lib.types.bool;
78 default = true;
79 description = ''
80 Use cgroups to enforce resource limits on containers
81 '';
82 };
83
84 localDir = lib.mkOption {
85 description = "List of directories to store localized files in.";
86 type = with lib.types; nullOr (listOf path);
87 example = [ "/var/lib/hadoop/yarn/nm" ];
88 default = null;
89 };
90
91 addBinBash = lib.mkOption {
92 type = lib.types.bool;
93 default = true;
94 description = ''
95 Add /bin/bash. This is needed by the linux container executor's launch script.
96 '';
97 };
98 openFirewall = lib.mkOption {
99 type = lib.types.bool;
100 default = false;
101 description = ''
102 Open firewall ports for nodemanager.
103 Because containers can listen on any ephemeral port, TCP ports 1024–65535 will be opened.
104 '';
105 };
106 };
107 };
108
109 config = lib.mkMerge [
110 (lib.mkIf cfg.gatewayRole.enable {
111 users.users.yarn = {
112 description = "Hadoop YARN user";
113 group = "hadoop";
114 uid = config.ids.uids.yarn;
115 };
116 })
117
118 (lib.mkIf cfg.yarn.resourcemanager.enable {
119 systemd.services.yarn-resourcemanager = {
120 description = "Hadoop YARN ResourceManager";
121 wantedBy = [ "multi-user.target" ];
122 inherit (cfg.yarn.resourcemanager) restartIfChanged;
123 environment = cfg.yarn.resourcemanager.extraEnv;
124
125 serviceConfig = {
126 User = "yarn";
127 SyslogIdentifier = "yarn-resourcemanager";
128 ExecStart =
129 "${cfg.package}/bin/yarn --config ${hadoopConf} "
130 + " resourcemanager ${lib.escapeShellArgs cfg.yarn.resourcemanager.extraFlags}";
131 Restart = "always";
132 };
133 };
134
135 services.hadoop.gatewayRole.enable = true;
136
137 networking.firewall.allowedTCPPorts = (
138 lib.mkIf cfg.yarn.resourcemanager.openFirewall [
139 8088 # resourcemanager.webapp.address
140 8030 # resourcemanager.scheduler.address
141 8031 # resourcemanager.resource-tracker.address
142 8032 # resourcemanager.address
143 8033 # resourcemanager.admin.address
144 ]
145 );
146 })
147
148 (lib.mkIf cfg.yarn.nodemanager.enable {
149 # Needed because yarn hardcodes /bin/bash in container start scripts
150 # These scripts can't be patched, they are generated at runtime
151 systemd.tmpfiles.rules = [
152 (lib.mkIf cfg.yarn.nodemanager.addBinBash "L /bin/bash - - - - /run/current-system/sw/bin/bash")
153 ];
154
155 systemd.services.yarn-nodemanager = {
156 description = "Hadoop YARN NodeManager";
157 wantedBy = [ "multi-user.target" ];
158 inherit (cfg.yarn.nodemanager) restartIfChanged;
159 environment = cfg.yarn.nodemanager.extraEnv;
160
161 preStart = ''
162 # create log dir
163 mkdir -p /var/log/hadoop/yarn/nodemanager
164 chown yarn:hadoop /var/log/hadoop/yarn/nodemanager
165
166 # set up setuid container executor binary
167 umount /run/wrappers/yarn-nodemanager/cgroup/cpu || true
168 rm -rf /run/wrappers/yarn-nodemanager/ || true
169 mkdir -p /run/wrappers/yarn-nodemanager/{bin,etc/hadoop,cgroup/cpu}
170 cp ${cfg.package}/bin/container-executor /run/wrappers/yarn-nodemanager/bin/
171 chgrp hadoop /run/wrappers/yarn-nodemanager/bin/container-executor
172 chmod 6050 /run/wrappers/yarn-nodemanager/bin/container-executor
173 cp ${hadoopConf}/container-executor.cfg /run/wrappers/yarn-nodemanager/etc/hadoop/
174 '';
175
176 serviceConfig = {
177 User = "yarn";
178 SyslogIdentifier = "yarn-nodemanager";
179 PermissionsStartOnly = true;
180 ExecStart =
181 "${cfg.package}/bin/yarn --config ${hadoopConf} "
182 + " nodemanager ${lib.escapeShellArgs cfg.yarn.nodemanager.extraFlags}";
183 Restart = "always";
184 };
185 };
186
187 services.hadoop.gatewayRole.enable = true;
188
189 services.hadoop.yarnSiteInternal =
190 with cfg.yarn.nodemanager;
191 lib.mkMerge [
192 ({
193 "yarn.nodemanager.local-dirs" = lib.mkIf (localDir != null) (concatStringsSep "," localDir);
194 "yarn.scheduler.maximum-allocation-vcores" = resource.maximumAllocationVCores;
195 "yarn.scheduler.maximum-allocation-mb" = resource.maximumAllocationMB;
196 "yarn.nodemanager.resource.cpu-vcores" = resource.cpuVCores;
197 "yarn.nodemanager.resource.memory-mb" = resource.memoryMB;
198 })
199 (lib.mkIf useCGroups (
200 lib.warnIf (lib.versionOlder cfg.package.version "3.5.0")
201 ''
202 hadoop < 3.5.0 does not support cgroup v2
203 setting `services.hadoop.yarn.nodemanager.useCGroups = false` is recommended
204 see: https://issues.apache.org/jira/browse/YARN-11669
205 ''
206 {
207 "yarn.nodemanager.linux-container-executor.cgroups.hierarchy" = "/hadoop-yarn";
208 "yarn.nodemanager.linux-container-executor.resources-handler.class" =
209 "org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler";
210 "yarn.nodemanager.linux-container-executor.cgroups.mount" = "true";
211 "yarn.nodemanager.linux-container-executor.cgroups.mount-path" =
212 "/run/wrappers/yarn-nodemanager/cgroup";
213 }
214 ))
215 ];
216
217 networking.firewall.allowedTCPPortRanges = [
218 (lib.mkIf (cfg.yarn.nodemanager.openFirewall) {
219 from = 1024;
220 to = 65535;
221 })
222 ];
223 })
224
225 ];
226}