commit c3d147f507e81455ae467f8aa589972af3449f77 · pyrox.dev/nixpkgs

+12 -12

nixos/modules/services/cluster/hadoop/hdfs.nix

···

       17
       17
        
       {

     

       18
       18
        
         options.services.hadoop.hdfs = {

     

       19
       19
        
           namenode = {

     

       20
       20
       -
             enabled = mkOption {

     

       20
       20
       +
             enable = mkOption {

     

       21
       21
        
               type = types.bool;

     

       22
       22
        
               default = false;

     

       23
       23
        
               description = ''

     
···

       43
       43
        
             };

     

       44
       44
        
           };

     

       45
       45
        
           datanode = {

     

       46
       46
       -
             enabled = mkOption {

     

       46
       46
       +
             enable = mkOption {

     

       47
       47
        
               type = types.bool;

     

       48
       48
        
               default = false;

     

       49
       49
        
               description = ''

     
···

       60
       60
        
             };

     

       61
       61
        
           };

     

       62
       62
        
           journalnode = {

     

       63
       63
       -
             enabled = mkOption {

     

       63
       63
       +
             enable = mkOption {

     

       64
       64
        
               type = types.bool;

     

       65
       65
        
               default = false;

     

       66
       66
        
               description = ''

     
···

       77
       77
        
             };

     

       78
       78
        
           };

     

       79
       79
        
           zkfc = {

     

       80
       80
       -
             enabled = mkOption {

     

       80
       80
       +
             enable = mkOption {

     

       81
       81
        
               type = types.bool;

     

       82
       82
        
               default = false;

     

       83
       83
        
               description = ''

     
···

       87
       87
        
             inherit restartIfChanged;

     

       88
       88
        
           };

     

       89
       89
        
           httpfs = {

     

       90
       90
       -
             enabled = mkOption {

     

       90
       90
       +
             enable = mkOption {

     

       91
       91
        
               type = types.bool;

     

       92
       92
        
               default = false;

     

       93
       93
        
               description = ''

     
···

       106
       106
        
         };

     

       107
       107
        
       

     

       108
       108
        
         config = mkMerge [

     

       109
       109
       -
           (mkIf cfg.hdfs.namenode.enabled {

     

       109
       109
       +
           (mkIf cfg.hdfs.namenode.enable {

     

       110
       110
        
             systemd.services.hdfs-namenode = {

     

       111
       111
        
               description = "Hadoop HDFS NameNode";

     

       112
       112
        
               wantedBy = [ "multi-user.target" ];

     
···

       130
       130
        
               8022 # namenode. servicerpc-address

     

       131
       131
        
             ]);

     

       132
       132
        
           })

     

       133
       133
       -
           (mkIf cfg.hdfs.datanode.enabled {

     

       133
       133
       +
           (mkIf cfg.hdfs.datanode.enable {

     

       134
       134
        
             systemd.services.hdfs-datanode = {

     

       135
       135
        
               description = "Hadoop HDFS DataNode";

     

       136
       136
        
               wantedBy = [ "multi-user.target" ];

     
···

       150
       150
        
               9867 # datanode.ipc.address

     

       151
       151
        
             ]);

     

       152
       152
        
           })

     

       153
       153
       -
           (mkIf cfg.hdfs.journalnode.enabled {

     

       153
       153
       +
           (mkIf cfg.hdfs.journalnode.enable {

     

       154
       154
        
             systemd.services.hdfs-journalnode = {

     

       155
       155
        
               description = "Hadoop HDFS JournalNode";

     

       156
       156
        
               wantedBy = [ "multi-user.target" ];

     
···

       169
       169
        
               8485 # dfs.journalnode.rpc-address

     

       170
       170
        
             ]);

     

       171
       171
        
           })

     

       172
       172
       -
           (mkIf cfg.hdfs.zkfc.enabled {

     

       172
       172
       +
           (mkIf cfg.hdfs.zkfc.enable {

     

       173
       173
        
             systemd.services.hdfs-zkfc = {

     

       174
       174
        
               description = "Hadoop HDFS ZooKeeper failover controller";

     

       175
       175
        
               wantedBy = [ "multi-user.target" ];

     
···

       183
       183
        
               };

     

       184
       184
        
             };

     

       185
       185
        
           })

     

       186
       186
       -
           (mkIf cfg.hdfs.httpfs.enabled {

     

       186
       186
       +
           (mkIf cfg.hdfs.httpfs.enable {

     

       187
       187
        
             systemd.services.hdfs-httpfs = {

     

       188
       188
        
               description = "Hadoop httpfs";

     

       189
       189
        
               wantedBy = [ "multi-user.target" ];

     
···

       209
       209
        
             ]);

     

       210
       210
        
           })

     

       211
       211
        
           (mkIf (

     

       212
       212
       -
               cfg.hdfs.namenode.enabled || cfg.hdfs.datanode.enabled || cfg.hdfs.journalnode.enabled || cfg.hdfs.zkfc.enabled

     

       212
       212
       +
               cfg.hdfs.namenode.enable || cfg.hdfs.datanode.enable || cfg.hdfs.journalnode.enable || cfg.hdfs.zkfc.enable

     

       213
       213
        
           ) {

     

       214
       214
        
             users.users.hdfs = {

     

       215
       215
        
               description = "Hadoop HDFS user";

     
···

       217
       217
        
               uid = config.ids.uids.hdfs;

     

       218
       218
        
             };

     

       219
       219
        
           })

     

       220
       220
       -
           (mkIf cfg.hdfs.httpfs.enabled {

     

       220
       220
       +
           (mkIf cfg.hdfs.httpfs.enable {

     

       221
       221
        
             users.users.httpfs = {

     

       222
       222
        
               description = "Hadoop HTTPFS user";

     

       223
       223
        
               group = "hadoop";

+5 -5

nixos/modules/services/cluster/hadoop/yarn.nix

···

       17
       17
        
       {

     

       18
       18
        
         options.services.hadoop.yarn = {

     

       19
       19
        
           resourcemanager = {

     

       20
       20
       -
             enabled = mkOption {

     

       20
       20
       +
             enable = mkOption {

     

       21
       21
        
               type = types.bool;

     

       22
       22
        
               default = false;

     

       23
       23
        
               description = ''

     
···

       34
       34
        
             };

     

       35
       35
        
           };

     

       36
       36
        
           nodemanager = {

     

       37
       37
       -
             enabled = mkOption {

     

       37
       37
       +
             enable = mkOption {

     

       38
       38
        
               type = types.bool;

     

       39
       39
        
               default = false;

     

       40
       40
        
               description = ''

     
···

       62
       62
        
       

     

       63
       63
        
         config = mkMerge [

     

       64
       64
        
           (mkIf (

     

       65
       65
       -
               cfg.yarn.resourcemanager.enabled || cfg.yarn.nodemanager.enabled

     

       65
       65
       +
               cfg.yarn.resourcemanager.enable || cfg.yarn.nodemanager.enable

     

       66
       66
        
           ) {

     

       67
       67
        
       

     

       68
       68
        
             users.users.yarn = {

     
···

       72
       72
        
             };

     

       73
       73
        
           })

     

       74
       74
        
       

     

       75
       75
       -
           (mkIf cfg.yarn.resourcemanager.enabled {

     

       75
       75
       +
           (mkIf cfg.yarn.resourcemanager.enable {

     

       76
       76
        
             systemd.services.yarn-resourcemanager = {

     

       77
       77
        
               description = "Hadoop YARN ResourceManager";

     

       78
       78
        
               wantedBy = [ "multi-user.target" ];

     
···

       95
       95
        
             ]);

     

       96
       96
        
           })

     

       97
       97
        
       

     

       98
       98
       -
           (mkIf cfg.yarn.nodemanager.enabled {

     

       98
       98
       +
           (mkIf cfg.yarn.nodemanager.enable {

     

       99
       99
        
             # Needed because yarn hardcodes /bin/bash in container start scripts

     

       100
       100
        
             # These scripts can't be patched, they are generated at runtime

     

       101
       101
        
             systemd.tmpfiles.rules = [

+11 -11

nixos/tests/hadoop/hadoop.nix

···

       55
       55
        
           nn1 = {pkgs, options, ...}: {

     

       56
       56
        
             services.hadoop = {

     

       57
       57
        
               inherit package coreSite hdfsSite;

     

       58
       58
       -
               hdfs.namenode.enabled = true;

     

       59
       59
       -
               hdfs.zkfc.enabled = true;

     

       58
       58
       +
               hdfs.namenode.enable = true;

     

       59
       59
       +
               hdfs.zkfc.enable = true;

     

       60
       60
        
             };

     

       61
       61
        
           };

     

       62
       62
        
           nn2 = {pkgs, options, ...}: {

     

       63
       63
        
             services.hadoop = {

     

       64
       64
        
               inherit package coreSite hdfsSite;

     

       65
       65
       -
               hdfs.namenode.enabled = true;

     

       66
       66
       -
               hdfs.zkfc.enabled = true;

     

       65
       65
       +
               hdfs.namenode.enable = true;

     

       66
       66
       +
               hdfs.zkfc.enable = true;

     

       67
       67
        
             };

     

       68
       68
        
           };

     

       69
       69
        
       

     

       70
       70
        
           jn1 = {pkgs, options, ...}: {

     

       71
       71
        
             services.hadoop = {

     

       72
       72
        
               inherit package coreSite hdfsSite;

     

       73
       73
       -
               hdfs.journalnode.enabled = true;

     

       73
       73
       +
               hdfs.journalnode.enable = true;

     

       74
       74
        
             };

     

       75
       75
        
           };

     

       76
       76
        
           jn2 = {pkgs, options, ...}: {

     

       77
       77
        
             services.hadoop = {

     

       78
       78
        
               inherit package coreSite hdfsSite;

     

       79
       79
       -
               hdfs.journalnode.enabled = true;

     

       79
       79
       +
               hdfs.journalnode.enable = true;

     

       80
       80
        
             };

     

       81
       81
        
           };

     

       82
       82
        
           jn3 = {pkgs, options, ...}: {

     

       83
       83
        
             services.hadoop = {

     

       84
       84
        
               inherit package coreSite hdfsSite;

     

       85
       85
       -
               hdfs.journalnode.enabled = true;

     

       85
       85
       +
               hdfs.journalnode.enable = true;

     

       86
       86
        
             };

     

       87
       87
        
           };

     

       88
       88
        
       

     

       89
       89
        
           dn1 = {pkgs, options, ...}: {

     

       90
       90
        
             services.hadoop = {

     

       91
       91
        
               inherit package coreSite hdfsSite;

     

       92
       92
       -
               hdfs.datanode.enabled = true;

     

       92
       92
       +
               hdfs.datanode.enable = true;

     

       93
       93
        
             };

     

       94
       94
        
           };

     

       95
       95
        
       

     
···

       99
       99
        
             services.hadoop = {

     

       100
       100
        
               inherit package coreSite hdfsSite;

     

       101
       101
        
               yarnSite = options.services.hadoop.yarnSite.default // yarnSiteHA;

     

       102
       102
       -
               yarn.resourcemanager.enabled = true;

     

       102
       102
       +
               yarn.resourcemanager.enable = true;

     

       103
       103
        
             };

     

       104
       104
        
           };

     

       105
       105
        
           rm2 = {pkgs, options, ...}: {

     
···

       107
       107
        
             services.hadoop = {

     

       108
       108
        
               inherit package coreSite hdfsSite;

     

       109
       109
        
               yarnSite = options.services.hadoop.yarnSite.default // yarnSiteHA;

     

       110
       110
       -
               yarn.resourcemanager.enabled = true;

     

       110
       110
       +
               yarn.resourcemanager.enable = true;

     

       111
       111
        
             };

     

       112
       112
        
           };

     

       113
       113
        
           nm1 = {pkgs, options, ...}: {

     
···

       115
       115
        
             services.hadoop = {

     

       116
       116
        
               inherit package coreSite hdfsSite;

     

       117
       117
        
               yarnSite = options.services.hadoop.yarnSite.default // yarnSiteHA;

     

       118
       118
       -
               yarn.nodemanager.enabled = true;

     

       118
       118
       +
               yarn.nodemanager.enable = true;

     

       119
       119
        
             };

     

       120
       120
        
           };

     

       121
       121
        
         };

+3 -3

nixos/tests/hadoop/hdfs.nix

···

       7
       7
        
               package = pkgs.hadoop;

     

       8
       8
        
               hdfs = {

     

       9
       9
        
                 namenode = {

     

       10
       10
       -
                   enabled = true;

     

       10
       10
       +
                   enable = true;

     

       11
       11
        
                   formatOnInit = true;

     

       12
       12
        
                 };

     

       13
       13
       -
                 httpfs.enabled = true;

     

       13
       13
       +
                 httpfs.enable = true;

     

       14
       14
        
               };

     

       15
       15
        
               coreSite = {

     

       16
       16
        
                 "fs.defaultFS" = "hdfs://namenode:8020";

     
···

       22
       22
        
           datanode = {pkgs, ...}: {

     

       23
       23
        
             services.hadoop = {

     

       24
       24
        
               package = pkgs.hadoop;

     

       25
       25
       -
               hdfs.datanode.enabled = true;

     

       25
       25
       +
               hdfs.datanode.enable = true;

     

       26
       26
        
               coreSite = {

     

       27
       27
        
                 "fs.defaultFS" = "hdfs://namenode:8020";

     

       28
       28
        
                 "hadoop.proxyuser.httpfs.groups" = "*";

+2 -2

nixos/tests/hadoop/yarn.nix

···

       3
       3
        
         nodes = {

     

       4
       4
        
           resourcemanager = {pkgs, ...}: {

     

       5
       5
        
             services.hadoop.package = pkgs.hadoop;

     

       6
       6
       -
             services.hadoop.yarn.resourcemanager.enabled = true;

     

       6
       6
       +
             services.hadoop.yarn.resourcemanager.enable = true;

     

       7
       7
        
             services.hadoop.yarnSite = {

     

       8
       8
        
               "yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler";

     

       9
       9
        
             };

     

       10
       10
        
           };

     

       11
       11
        
           nodemanager = {pkgs, ...}: {

     

       12
       12
        
             services.hadoop.package = pkgs.hadoop;

     

       13
       13
       -
             services.hadoop.yarn.nodemanager.enabled = true;

     

       13
       13
       +
             services.hadoop.yarn.nodemanager.enable = true;

     

       14
       14
        
             services.hadoop.yarnSite = {

     

       15
       15
        
               "yarn.resourcemanager.hostname" = "resourcemanager";

     

       16
       16
        
               "yarn.nodemanager.log-dirs" = "/tmp/userlogs";