commit 0c10b2baa6bf61c8ddaed7cdb6c2f2dbaab42662 · pyrox.dev/nixpkgs

nixos/modules/misc/ids.nix

···

       317
       317
        
             restic = 291;

     

       318
       318
        
             openvpn = 292;

     

       319
       319
        
             meguca = 293;

     

       320
       320
       +
             yarn = 294;

     

       321
       321
       +
             hdfs = 295;

     

       322
       322
       +
             mapred = 296;

     

       323
       323
       +
             hadoop = 297;

     

       320
       324
        
       

     

       321
       325
        
             # When adding a uid, make sure it doesn't match an existing gid. And don't use uids above 399!

     

       322
       326
        
       

     
···

       594
       598
        
             restic = 291;

     

       595
       599
        
             openvpn = 292;

     

       596
       600
        
             meguca = 293;

     

       601
       601
       +
             yarn = 294;

     

       602
       602
       +
             hdfs = 295;

     

       603
       603
       +
             mapred = 296;

     

       604
       604
       +
             hadoop = 297;

     

       597
       605
        
       

     

       598
       606
        
             # When adding a gid, make sure it doesn't match an existing

     

       599
       607
        
             # uid. Users and groups with the same name should have equal

nixos/modules/module-list.nix

···

       178
       178
        
         ./services/backup/rsnapshot.nix

     

       179
       179
        
         ./services/backup/tarsnap.nix

     

       180
       180
        
         ./services/backup/znapzend.nix

     

       181
       181
       +
         ./services/cluster/hadoop/default.nix

     

       181
       182
        
         ./services/cluster/kubernetes/default.nix

     

       182
       183
        
         ./services/cluster/kubernetes/dns.nix

     

       183
       184
        
         ./services/cluster/kubernetes/dashboard.nix

+31

nixos/modules/services/cluster/hadoop/conf.nix

···

       1
       1
       +
       { hadoop, pkgs }:

     

       2
       2
       +
       let

     

       3
       3
       +
         propertyXml = name: value: ''

     

       4
       4
       +
           <property>

     

       5
       5
       +
             <name>${name}</name>

     

       6
       6
       +
             <value>${builtins.toString value}</value>

     

       7
       7
       +
           </property>

     

       8
       8
       +
         '';

     

       9
       9
       +
         siteXml = fileName: properties: pkgs.writeTextDir fileName ''

     

       10
       10
       +
           <?xml version="1.0" encoding="UTF-8" standalone="no"?>

     

       11
       11
       +
           <!-- generated by NixOS -->

     

       12
       12
       +
           <configuration>

     

       13
       13
       +
             ${builtins.concatStringsSep "\n" (pkgs.lib.mapAttrsToList propertyXml properties)}

     

       14
       14
       +
           </configuration>

     

       15
       15
       +
         '';

     

       16
       16
       +
         userFunctions = ''

     

       17
       17
       +
           hadoop_verify_logdir() {

     

       18
       18
       +
             echo Skipping verification of log directory

     

       19
       19
       +
           }

     

       20
       20
       +
         '';

     

       21
       21
       +
       in

     

       22
       22
       +
       pkgs.buildEnv {

     

       23
       23
       +
         name = "hadoop-conf";

     

       24
       24
       +
         paths = [

     

       25
       25
       +
           (siteXml "core-site.xml" hadoop.coreSite)

     

       26
       26
       +
           (siteXml "hdfs-site.xml" hadoop.hdfsSite)

     

       27
       27
       +
           (siteXml "mapred-site.xml" hadoop.mapredSite)

     

       28
       28
       +
           (siteXml "yarn-site.xml" hadoop.yarnSite)

     

       29
       29
       +
           (pkgs.writeTextDir "hadoop-user-functions.sh" userFunctions)

     

       30
       30
       +
         ];

     

       31
       31
       +
       }

+63

nixos/modules/services/cluster/hadoop/default.nix

···

       1
       1
       +
       { config, lib, pkgs, ...}:

     

       2
       2
       +
       let 

     

       3
       3
       +
         cfg = config.services.hadoop;

     

       4
       4
       +
         hadoopConf = import ./conf.nix { hadoop = cfg; pkgs = pkgs; };

     

       5
       5
       +
       in

     

       6
       6
       +
       with lib;

     

       7
       7
       +
       {

     

       8
       8
       +
         imports = [ ./yarn.nix ./hdfs.nix ];

     

       9
       9
       +
       

     

       10
       10
       +
         options.services.hadoop = {

     

       11
       11
       +
           coreSite = mkOption {

     

       12
       12
       +
             default = {};

     

       13
       13
       +
             example = {

     

       14
       14
       +
               "fs.defaultFS" = "hdfs://localhost";

     

       15
       15
       +
             };

     

       16
       16
       +
             description = "Hadoop core-site.xml definition";

     

       17
       17
       +
           };

     

       18
       18
       +
       

     

       19
       19
       +
           hdfsSite = mkOption {

     

       20
       20
       +
             default = {};

     

       21
       21
       +
             example = {

     

       22
       22
       +
               "dfs.nameservices" = "namenode1";

     

       23
       23
       +
             };

     

       24
       24
       +
             description = "Hadoop hdfs-site.xml definition";

     

       25
       25
       +
           };

     

       26
       26
       +
       

     

       27
       27
       +
           mapredSite = mkOption {

     

       28
       28
       +
             default = {};

     

       29
       29
       +
             example = {

     

       30
       30
       +
               "mapreduce.map.cpu.vcores" = "1";

     

       31
       31
       +
             };

     

       32
       32
       +
             description = "Hadoop mapred-site.xml definition";

     

       33
       33
       +
           };

     

       34
       34
       +
       

     

       35
       35
       +
           yarnSite = mkOption {

     

       36
       36
       +
             default = {};

     

       37
       37
       +
             example = {

     

       38
       38
       +
               "yarn.resourcemanager.ha.id" = "resourcemanager1";

     

       39
       39
       +
             };

     

       40
       40
       +
             description = "Hadoop yarn-site.xml definition";

     

       41
       41
       +
           };

     

       42
       42
       +
       

     

       43
       43
       +
           package = mkOption {

     

       44
       44
       +
             type = types.package;

     

       45
       45
       +
             default = pkgs.hadoop;

     

       46
       46
       +
             defaultText = "pkgs.hadoop";

     

       47
       47
       +
             example = literalExample "pkgs.hadoop";

     

       48
       48
       +
             description = ''

     

       49
       49
       +
             '';

     

       50
       50
       +
           };

     

       51
       51
       +
         };

     

       52
       52
       +
       

     

       53
       53
       +
       

     

       54
       54
       +
         config = mkMerge [

     

       55
       55
       +
           (mkIf (builtins.hasAttr "yarn" config.users.extraUsers ||

     

       56
       56
       +
                  builtins.hasAttr "hdfs" config.users.extraUsers ) {

     

       57
       57
       +
             users.extraGroups.hadoop = {

     

       58
       58
       +
               gid = config.ids.gids.hadoop;

     

       59
       59
       +
             };

     

       60
       60
       +
           })

     

       61
       61
       +
       

     

       62
       62
       +
         ];

     

       63
       63
       +
       }

+73

nixos/modules/services/cluster/hadoop/hdfs.nix

···

       1
       1
       +
       { config, lib, pkgs, ...}:

     

       2
       2
       +
       let

     

       3
       3
       +
         cfg = config.services.hadoop;

     

       4
       4
       +
         hadoopConf = import ./conf.nix { hadoop = cfg; pkgs = pkgs; };

     

       5
       5
       +
       in

     

       6
       6
       +
       with lib;

     

       7
       7
       +
       {

     

       8
       8
       +
         options.services.hadoop.hdfs = {

     

       9
       9
       +
           namenode.enabled = mkOption {

     

       10
       10
       +
             type = types.bool;

     

       11
       11
       +
             default = false;

     

       12
       12
       +
             description = ''

     

       13
       13
       +
               Whether to run the Hadoop YARN NameNode

     

       14
       14
       +
             '';

     

       15
       15
       +
           };

     

       16
       16
       +
           datanode.enabled = mkOption {

     

       17
       17
       +
             type = types.bool;

     

       18
       18
       +
             default = false;

     

       19
       19
       +
             description = ''

     

       20
       20
       +
               Whether to run the Hadoop YARN DataNode

     

       21
       21
       +
             '';

     

       22
       22
       +
           };

     

       23
       23
       +
         };

     

       24
       24
       +
       

     

       25
       25
       +
         config = mkMerge [

     

       26
       26
       +
           (mkIf cfg.hdfs.namenode.enabled {

     

       27
       27
       +
             systemd.services."hdfs-namenode" = {

     

       28
       28
       +
               description = "Hadoop HDFS NameNode";

     

       29
       29
       +
               wantedBy = [ "multi-user.target" ];

     

       30
       30
       +
       

     

       31
       31
       +
               environment = {

     

       32
       32
       +
                 HADOOP_HOME = "${cfg.package}";

     

       33
       33
       +
               };

     

       34
       34
       +
       

     

       35
       35
       +
               preStart = ''

     

       36
       36
       +
                 ${cfg.package}/bin/hdfs --config ${hadoopConf} namenode -format -nonInteractive || true

     

       37
       37
       +
               '';

     

       38
       38
       +
       

     

       39
       39
       +
               serviceConfig = {

     

       40
       40
       +
                 User = "hdfs";

     

       41
       41
       +
                 SyslogIdentifier = "hdfs-namenode";

     

       42
       42
       +
                 ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} namenode";

     

       43
       43
       +
               };

     

       44
       44
       +
             };

     

       45
       45
       +
           })

     

       46
       46
       +
           (mkIf cfg.hdfs.datanode.enabled {

     

       47
       47
       +
             systemd.services."hdfs-datanode" = {

     

       48
       48
       +
               description = "Hadoop HDFS DataNode";

     

       49
       49
       +
               wantedBy = [ "multi-user.target" ];

     

       50
       50
       +
       

     

       51
       51
       +
               environment = {

     

       52
       52
       +
                 HADOOP_HOME = "${cfg.package}";

     

       53
       53
       +
               };

     

       54
       54
       +
       

     

       55
       55
       +
               serviceConfig = {

     

       56
       56
       +
                 User = "hdfs";

     

       57
       57
       +
                 SyslogIdentifier = "hdfs-datanode";

     

       58
       58
       +
                 ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} datanode";

     

       59
       59
       +
               };

     

       60
       60
       +
             };

     

       61
       61
       +
           })

     

       62
       62
       +
           (mkIf (

     

       63
       63
       +
               cfg.hdfs.namenode.enabled || cfg.hdfs.datanode.enabled

     

       64
       64
       +
           ) {

     

       65
       65
       +
             users.extraUsers.hdfs = {

     

       66
       66
       +
               description = "Hadoop HDFS user";

     

       67
       67
       +
               group = "hadoop";

     

       68
       68
       +
               uid = config.ids.uids.hdfs;

     

       69
       69
       +
             };

     

       70
       70
       +
           })

     

       71
       71
       +
       

     

       72
       72
       +
         ];

     

       73
       73
       +
       }

+74

nixos/modules/services/cluster/hadoop/yarn.nix

···

       1
       1
       +
       { config, lib, pkgs, ...}:

     

       2
       2
       +
       let

     

       3
       3
       +
         cfg = config.services.hadoop;

     

       4
       4
       +
         hadoopConf = import ./conf.nix { hadoop = cfg; pkgs = pkgs; };

     

       5
       5
       +
       in

     

       6
       6
       +
       with lib;

     

       7
       7
       +
       {

     

       8
       8
       +
         options.services.hadoop.yarn = {

     

       9
       9
       +
           resourcemanager.enabled = mkOption {

     

       10
       10
       +
             type = types.bool;

     

       11
       11
       +
             default = false;

     

       12
       12
       +
             description = ''

     

       13
       13
       +
               Whether to run the Hadoop YARN ResourceManager

     

       14
       14
       +
             '';

     

       15
       15
       +
           };

     

       16
       16
       +
           nodemanager.enabled = mkOption {

     

       17
       17
       +
             type = types.bool;

     

       18
       18
       +
             default = false;

     

       19
       19
       +
             description = ''

     

       20
       20
       +
               Whether to run the Hadoop YARN NodeManager

     

       21
       21
       +
             '';

     

       22
       22
       +
           };

     

       23
       23
       +
         };

     

       24
       24
       +
       

     

       25
       25
       +
         config = mkMerge [

     

       26
       26
       +
           (mkIf (

     

       27
       27
       +
               cfg.yarn.resourcemanager.enabled || cfg.yarn.nodemanager.enabled

     

       28
       28
       +
           ) {

     

       29
       29
       +
       

     

       30
       30
       +
             users.extraUsers.yarn = {

     

       31
       31
       +
               description = "Hadoop YARN user";

     

       32
       32
       +
               group = "hadoop";

     

       33
       33
       +
               uid = config.ids.uids.yarn;

     

       34
       34
       +
             };

     

       35
       35
       +
           })

     

       36
       36
       +
       

     

       37
       37
       +
           (mkIf cfg.yarn.resourcemanager.enabled {

     

       38
       38
       +
             systemd.services."yarn-resourcemanager" = {

     

       39
       39
       +
               description = "Hadoop YARN ResourceManager";

     

       40
       40
       +
               wantedBy = [ "multi-user.target" ];

     

       41
       41
       +
       

     

       42
       42
       +
               environment = {

     

       43
       43
       +
                 HADOOP_HOME = "${cfg.package}";

     

       44
       44
       +
               };

     

       45
       45
       +
       

     

       46
       46
       +
               serviceConfig = {

     

       47
       47
       +
                 User = "yarn";

     

       48
       48
       +
                 SyslogIdentifier = "yarn-resourcemanager";

     

       49
       49
       +
                 ExecStart = "${cfg.package}/bin/yarn --config ${hadoopConf} " +

     

       50
       50
       +
                             " resourcemanager";

     

       51
       51
       +
               };

     

       52
       52
       +
             };

     

       53
       53
       +
           })

     

       54
       54
       +
       

     

       55
       55
       +
           (mkIf cfg.yarn.nodemanager.enabled {

     

       56
       56
       +
             systemd.services."yarn-nodemanager" = {

     

       57
       57
       +
               description = "Hadoop YARN NodeManager";

     

       58
       58
       +
               wantedBy = [ "multi-user.target" ];

     

       59
       59
       +
       

     

       60
       60
       +
               environment = {

     

       61
       61
       +
                 HADOOP_HOME = "${cfg.package}";

     

       62
       62
       +
               };

     

       63
       63
       +
       

     

       64
       64
       +
               serviceConfig = {

     

       65
       65
       +
                 User = "yarn";

     

       66
       66
       +
                 SyslogIdentifier = "yarn-nodemanager";

     

       67
       67
       +
                 ExecStart = "${cfg.package}/bin/yarn --config ${hadoopConf} " +

     

       68
       68
       +
                             " nodemanager";

     

       69
       69
       +
               };

     

       70
       70
       +
             };

     

       71
       71
       +
           })

     

       72
       72
       +
       

     

       73
       73
       +
         ];

     

       74
       74
       +
       }

nixos/release.nix

···

       299
       299
        
         tests.gnome3-gdm = callTest tests/gnome3-gdm.nix {};

     

       300
       300
        
         tests.grafana = callTest tests/grafana.nix {};

     

       301
       301
        
         tests.graphite = callTest tests/graphite.nix {};

     

       302
       302
       +
         tests.hadoop.hdfs = callTestOnMatchingSystems [ "x86_64-linux" ] tests/hadoop/hdfs.nix {};

     

       303
       303
       +
         tests.hadoop.yarn = callTestOnMatchingSystems [ "x86_64-linux" ] tests/hadoop/yarn.nix {};

     

       302
       304
        
         tests.hardened = callTest tests/hardened.nix { };

     

       303
       305
        
         tests.haproxy = callTest tests/haproxy.nix {};

     

       304
       306
        
         tests.hibernate = callTest tests/hibernate.nix {};

+54

nixos/tests/hadoop/hdfs.nix

···

       1
       1
       +
       import ../make-test.nix ({pkgs, ...}: {

     

       2
       2
       +
         nodes = {

     

       3
       3
       +
           namenode = {pkgs, config, ...}: {

     

       4
       4
       +
             services.hadoop = {

     

       5
       5
       +
               package = pkgs.hadoop_3_1;

     

       6
       6
       +
               hdfs.namenode.enabled = true;

     

       7
       7
       +
               coreSite = {

     

       8
       8
       +
                 "fs.defaultFS" = "hdfs://namenode:8020";

     

       9
       9
       +
               };

     

       10
       10
       +
               hdfsSite = {

     

       11
       11
       +
                 "dfs.replication" = 1;

     

       12
       12
       +
                 "dfs.namenode.rpc-bind-host" = "0.0.0.0";

     

       13
       13
       +
                 "dfs.namenode.http-bind-host" = "0.0.0.0";

     

       14
       14
       +
               };

     

       15
       15
       +
             };

     

       16
       16
       +
             networking.firewall.allowedTCPPorts = [

     

       17
       17
       +
               9870 # namenode.http-address

     

       18
       18
       +
               8020 # namenode.rpc-address

     

       19
       19
       +
             ];

     

       20
       20
       +
           };

     

       21
       21
       +
           datanode = {pkgs, config, ...}: {

     

       22
       22
       +
             services.hadoop = {

     

       23
       23
       +
               package = pkgs.hadoop_3_1;

     

       24
       24
       +
               hdfs.datanode.enabled = true;

     

       25
       25
       +
               coreSite = {

     

       26
       26
       +
                 "fs.defaultFS" = "hdfs://namenode:8020";

     

       27
       27
       +
               };

     

       28
       28
       +
             };

     

       29
       29
       +
             networking.firewall.allowedTCPPorts = [

     

       30
       30
       +
               9864 # datanode.http.address

     

       31
       31
       +
               9866 # datanode.address

     

       32
       32
       +
               9867 # datanode.ipc.address

     

       33
       33
       +
             ];

     

       34
       34
       +
           };

     

       35
       35
       +
         };

     

       36
       36
       +
       

     

       37
       37
       +
         testScript = ''

     

       38
       38
       +
           startAll

     

       39
       39
       +
       

     

       40
       40
       +
           $namenode->waitForUnit("hdfs-namenode");

     

       41
       41
       +
           $namenode->waitForUnit("network.target");

     

       42
       42
       +
           $namenode->waitForOpenPort(8020);

     

       43
       43
       +
           $namenode->waitForOpenPort(9870);

     

       44
       44
       +
       

     

       45
       45
       +
           $datanode->waitForUnit("hdfs-datanode");

     

       46
       46
       +
           $datanode->waitForUnit("network.target");

     

       47
       47
       +
           $datanode->waitForOpenPort(9864);

     

       48
       48
       +
           $datanode->waitForOpenPort(9866);

     

       49
       49
       +
           $datanode->waitForOpenPort(9867);

     

       50
       50
       +
       

     

       51
       51
       +
           $namenode->succeed("curl http://namenode:9870");

     

       52
       52
       +
           $datanode->succeed("curl http://datanode:9864");

     

       53
       53
       +
         '';

     

       54
       54
       +
       })

+46

nixos/tests/hadoop/yarn.nix

···

       1
       1
       +
       import ../make-test.nix ({pkgs, ...}: {

     

       2
       2
       +
         nodes = {

     

       3
       3
       +
           resourcemanager = {pkgs, config, ...}: {

     

       4
       4
       +
             services.hadoop.package = pkgs.hadoop_3_1;

     

       5
       5
       +
             services.hadoop.yarn.resourcemanager.enabled = true;

     

       6
       6
       +
             services.hadoop.yarnSite = {

     

       7
       7
       +
               "yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler";

     

       8
       8
       +
             };

     

       9
       9
       +
             networking.firewall.allowedTCPPorts = [

     

       10
       10
       +
               8088 # resourcemanager.webapp.address

     

       11
       11
       +
               8031 # resourcemanager.resource-tracker.address

     

       12
       12
       +
             ];

     

       13
       13
       +
           };

     

       14
       14
       +
           nodemanager = {pkgs, config, ...}: {

     

       15
       15
       +
             services.hadoop.package = pkgs.hadoop_3_1;

     

       16
       16
       +
             services.hadoop.yarn.nodemanager.enabled = true;

     

       17
       17
       +
             services.hadoop.yarnSite = {

     

       18
       18
       +
               "yarn.resourcemanager.hostname" = "resourcemanager";

     

       19
       19
       +
               "yarn.nodemanager.log-dirs" = "/tmp/userlogs";

     

       20
       20
       +
               "yarn.nodemanager.address" = "0.0.0.0:8041";

     

       21
       21
       +
             };

     

       22
       22
       +
             networking.firewall.allowedTCPPorts = [

     

       23
       23
       +
               8042 # nodemanager.webapp.address

     

       24
       24
       +
               8041 # nodemanager.address

     

       25
       25
       +
             ];

     

       26
       26
       +
           };

     

       27
       27
       +
       

     

       28
       28
       +
         };

     

       29
       29
       +
       

     

       30
       30
       +
         testScript = ''

     

       31
       31
       +
           startAll;

     

       32
       32
       +
       

     

       33
       33
       +
           $resourcemanager->waitForUnit("yarn-resourcemanager");

     

       34
       34
       +
           $resourcemanager->waitForUnit("network.target");

     

       35
       35
       +
           $resourcemanager->waitForOpenPort(8031);

     

       36
       36
       +
           $resourcemanager->waitForOpenPort(8088);

     

       37
       37
       +
       

     

       38
       38
       +
           $nodemanager->waitForUnit("yarn-nodemanager");

     

       39
       39
       +
           $nodemanager->waitForUnit("network.target");

     

       40
       40
       +
           $nodemanager->waitForOpenPort(8042);

     

       41
       41
       +
           $nodemanager->waitForOpenPort(8041);

     

       42
       42
       +
       

     

       43
       43
       +
           $resourcemanager->succeed("curl http://localhost:8088");

     

       44
       44
       +
           $nodemanager->succeed("curl http://localhost:8042");

     

       45
       45
       +
         '';

     

       46
       46
       +
       })