commit c62ec80d1fc0cc4a665f3dca74a4d1292a8820e5 · pyrox.dev/nixpkgs

+16 -12

nixos/modules/services/cluster/hadoop/yarn.nix

···

       196
       196
        
                   "yarn.nodemanager.resource.cpu-vcores" = resource.cpuVCores;

     

       197
       197
        
                   "yarn.nodemanager.resource.memory-mb" = resource.memoryMB;

     

       198
       198
        
                 })

     

       199
       199
       -
                 (lib.mkIf useCGroups ( lib.warnIf (lib.versionOlder cfg.package.version "3.5.0") ''

     

       200
       200
       -
                   hadoop < 3.5.0 does not support cgroup v2

     

       201
       201
       -
                   setting `services.hadoop.yarn.nodemanager.useCGroups = false` is recommended

     

       202
       202
       -
                   see: https://issues.apache.org/jira/browse/YARN-11669

     

       203
       203
       -
                 '' {

     

       204
       204
       -
                   "yarn.nodemanager.linux-container-executor.cgroups.hierarchy" = "/hadoop-yarn";

     

       205
       205
       -
                   "yarn.nodemanager.linux-container-executor.resources-handler.class" =

     

       206
       206
       -
                     "org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler";

     

       207
       207
       -
                   "yarn.nodemanager.linux-container-executor.cgroups.mount" = "true";

     

       208
       208
       -
                   "yarn.nodemanager.linux-container-executor.cgroups.mount-path" =

     

       209
       209
       -
                     "/run/wrappers/yarn-nodemanager/cgroup";

     

       210
       210
       -
                 }))

     

       199
       199
       +
                 (lib.mkIf useCGroups (

     

       200
       200
       +
                   lib.warnIf (lib.versionOlder cfg.package.version "3.5.0")

     

       201
       201
       +
                     ''

     

       202
       202
       +
                       hadoop < 3.5.0 does not support cgroup v2

     

       203
       203
       +
                       setting `services.hadoop.yarn.nodemanager.useCGroups = false` is recommended

     

       204
       204
       +
                       see: https://issues.apache.org/jira/browse/YARN-11669

     

       205
       205
       +
                     ''

     

       206
       206
       +
                     {

     

       207
       207
       +
                       "yarn.nodemanager.linux-container-executor.cgroups.hierarchy" = "/hadoop-yarn";

     

       208
       208
       +
                       "yarn.nodemanager.linux-container-executor.resources-handler.class" =

     

       209
       209
       +
                         "org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler";

     

       210
       210
       +
                       "yarn.nodemanager.linux-container-executor.cgroups.mount" = "true";

     

       211
       211
       +
                       "yarn.nodemanager.linux-container-executor.cgroups.mount-path" =

     

       212
       212
       +
                         "/run/wrappers/yarn-nodemanager/cgroup";

     

       213
       213
       +
                     }

     

       214
       214
       +
                 ))

     

       211
       215
        
               ];

     

       212
       216
        
       

     

       213
       217
        
             networking.firewall.allowedTCPPortRanges = [

+12 -4

nixos/tests/hadoop/default.nix

···

       1
       1
        
       { handleTestOn, package, ... }:

     

       2
       2
        
       

     

       3
       3
        
       {

     

       4
       4
       -
         all = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hadoop.nix { inherit package; };

     

       5
       5
       -
         hdfs = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hdfs.nix { inherit package; };

     

       6
       6
       -
         yarn = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./yarn.nix { inherit package; };

     

       7
       7
       -
         hbase = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hbase.nix { inherit package; };

     

       4
       4
       +
         all = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hadoop.nix {

     

       5
       5
       +
           inherit package;

     

       6
       6
       +
         };

     

       7
       7
       +
         hdfs = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hdfs.nix {

     

       8
       8
       +
           inherit package;

     

       9
       9
       +
         };

     

       10
       10
       +
         yarn = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./yarn.nix {

     

       11
       11
       +
           inherit package;

     

       12
       12
       +
         };

     

       13
       13
       +
         hbase = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hbase.nix {

     

       14
       14
       +
           inherit package;

     

       15
       15
       +
         };

     

       8
       16
        
       }

+9 -4

pkgs/applications/networking/cluster/hadoop/default.nix

···

       46
       46
        
                 "mirror://apache/hadoop/common/hadoop-${finalAttrs.version}/hadoop-${finalAttrs.version}"

     

       47
       47
        
                 + lib.optionalString stdenv.hostPlatform.isAarch64 "-aarch64"

     

       48
       48
        
                 + ".tar.gz";

     

       49
       49
       -
               inherit (platformAttrs.${stdenv.system} or (throw "Unsupported system: ${stdenv.system}")) hash;

     

       49
       49
       +
               inherit (platformAttrs.${stdenv.system} or (throw "Unsupported system: ${stdenv.system}"))

     

       50
       50
       +
                 hash

     

       51
       51
       +
                 ;

     

       50
       52
        
             };

     

       51
       53
        
             doCheck = true;

     

       52
       54
        
       

     
···

       101
       103
        
                 # hadoop 3.3+ depends on protobuf 3.18, 3.2 depends on 3.8

     

       102
       104
        
                 find $out/lib/native -name 'libhdfspp.so*' | \

     

       103
       105
        
                   xargs -r -n1 patchelf --replace-needed libprotobuf.so.${

     

       104
       104
       -
                     if (lib.versionAtLeast finalAttrs.version "3.4.1") then "32"

     

       105
       105
       -
                     else if (lib.versionAtLeast finalAttrs.version "3.3") then "18"

     

       106
       106
       -
                     else "8"

     

       106
       106
       +
                     if (lib.versionAtLeast finalAttrs.version "3.4.1") then

     

       107
       107
       +
                       "32"

     

       108
       108
       +
                     else if (lib.versionAtLeast finalAttrs.version "3.3") then

     

       109
       109
       +
                       "18"

     

       110
       110
       +
                     else

     

       111
       111
       +
                       "8"

     

       107
       112
        
                   } libprotobuf.so

     

       108
       113
        
       

     

       109
       114
        
                 patchelf --replace-needed libcrypto.so.1.1 libcrypto.so \