commit 565f22d27aba3c53fcf4155dba639f2071d08ed6 · pyrox.dev/nixpkgs

lib/maintainers.nix

···

       380
       380
        
         ledif = "Adam Fidel <refuse@gmail.com>";

     

       381
       381
        
         leemachin = "Lee Machin <me@mrl.ee>";

     

       382
       382
        
         leenaars = "Michiel Leenaars <ml.software@leenaa.rs>";

     

       383
       383
       +
         lejonet = "Daniel Kuehn <daniel@kuehn.se>";

     

       383
       384
        
         leonardoce = "Leonardo Cecchi <leonardo.cecchi@gmail.com>";

     

       384
       385
        
         lethalman = "Luca Bruno <lucabru@src.gnome.org>";

     

       385
       386
        
         lewo = "Antoine Eiche <lewo@abesis.fr>";

nixos/modules/misc/ids.nix

···

       304
       304
        
             mighttpd2 = 285;

     

       305
       305
        
             hass = 286;

     

       306
       306
        
             monero = 287;

     

       307
       307
       +
             ceph = 288;

     

       307
       308
        
       

     

       308
       309
        
             # When adding a uid, make sure it doesn't match an existing gid. And don't use uids above 399!

     

       309
       310
        
       

     
···

       576
       577
        
             mighttpd2 = 285;

     

       577
       578
        
             hass = 286;

     

       578
       579
        
             monero = 287;

     

       580
       580
       +
             ceph = 288;

     

       579
       581
        
       

     

       580
       582
        
             # When adding a gid, make sure it doesn't match an existing

     

       581
       583
        
             # uid. Users and groups with the same name should have equal

nixos/modules/module-list.nix

···

       439
       439
        
         ./services/network-filesystems/u9fs.nix

     

       440
       440
        
         ./services/network-filesystems/yandex-disk.nix

     

       441
       441
        
         ./services/network-filesystems/xtreemfs.nix

     

       442
       442
       +
         ./services/network-filesystems/ceph.nix

     

       442
       443
        
         ./services/networking/amuled.nix

     

       443
       444
        
         ./services/networking/aria2.nix

     

       444
       445
        
         ./services/networking/asterisk.nix

+371

nixos/modules/services/network-filesystems/ceph.nix

···

       1
       1
       +
       { config, lib, pkgs, ... }:

     

       2
       2
       +
       

     

       3
       3
       +
       with lib;

     

       4
       4
       +
       

     

       5
       5
       +
       let

     

       6
       6
       +
         ceph = pkgs.ceph;

     

       7
       7
       +
         cfg  = config.services.ceph;

     

       8
       8
       +
         # function that translates "camelCaseOptions" to "camel case options", credits to tilpner in #nixos@freenode

     

       9
       9
       +
         translateOption = replaceStrings upperChars (map (s: " ${s}") lowerChars);

     

       10
       10
       +
         generateDaemonList = (daemonType: daemons: extraServiceConfig:

     

       11
       11
       +
           mkMerge (

     

       12
       12
       +
             map (daemon: 

     

       13
       13
       +
               { "ceph-${daemonType}-${daemon}" = generateServiceFile daemonType daemon cfg.global.clusterName ceph extraServiceConfig; }

     

       14
       14
       +
             ) daemons

     

       15
       15
       +
           )

     

       16
       16
       +
         );

     

       17
       17
       +
         generateServiceFile = (daemonType: daemonId: clusterName: ceph: extraServiceConfig: {

     

       18
       18
       +
           enable = true;

     

       19
       19
       +
           description = "Ceph ${builtins.replaceStrings lowerChars upperChars daemonType} daemon ${daemonId}";

     

       20
       20
       +
           after = [ "network-online.target" "local-fs.target" "time-sync.target" ] ++ optional (daemonType == "osd") "ceph-mon.target";

     

       21
       21
       +
           wants = [ "network-online.target" "local-fs.target" "time-sync.target" ];

     

       22
       22
       +
           partOf = [ "ceph-${daemonType}.target" ];

     

       23
       23
       +
           wantedBy = [ "ceph-${daemonType}.target" ];

     

       24
       24
       +
       

     

       25
       25
       +
           serviceConfig = {

     

       26
       26
       +
             LimitNOFILE = 1048576;

     

       27
       27
       +
             LimitNPROC = 1048576;

     

       28
       28
       +
             Environment = "CLUSTER=${clusterName}";

     

       29
       29
       +
             ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";

     

       30
       30
       +
             PrivateDevices = "yes";

     

       31
       31
       +
             PrivateTmp = "true";

     

       32
       32
       +
             ProtectHome = "true";

     

       33
       33
       +
             ProtectSystem = "full";

     

       34
       34
       +
             Restart = "on-failure";

     

       35
       35
       +
             StartLimitBurst = "5";

     

       36
       36
       +
             StartLimitInterval = "30min";

     

       37
       37
       +
             ExecStart = "${ceph.out}/bin/${if daemonType == "rgw" then "radosgw" else "ceph-${daemonType}"} -f --cluster ${clusterName} --id ${if daemonType == "rgw" then "client.${daemonId}" else daemonId} --setuser ceph --setgroup ceph";

     

       38
       38
       +
           } // extraServiceConfig

     

       39
       39
       +
             // optionalAttrs (daemonType == "osd") { ExecStartPre = "${ceph.out}/libexec/ceph/ceph-osd-prestart.sh --id ${daemonId} --cluster ${clusterName}"; };

     

       40
       40
       +
           } // optionalAttrs (builtins.elem daemonType [ "mds" "mon" "rgw" "mgr" ]) { preStart = ''

     

       41
       41
       +
               daemonPath="/var/lib/ceph/${if daemonType == "rgw" then "radosgw" else daemonType}/${clusterName}-${daemonId}"

     

       42
       42
       +
               if [ ! -d ''$daemonPath ]; then

     

       43
       43
       +
                 mkdir -m 755 -p ''$daemonPath

     

       44
       44
       +
                 chown -R ceph:ceph ''$daemonPath 

     

       45
       45
       +
               fi

     

       46
       46
       +
             '';

     

       47
       47
       +
           } // optionalAttrs (daemonType == "osd") { path = [ pkgs.getopt ]; }

     

       48
       48
       +
         );

     

       49
       49
       +
         generateTargetFile = (daemonType:

     

       50
       50
       +
           {

     

       51
       51
       +
             "ceph-${daemonType}" = {

     

       52
       52
       +
               description = "Ceph target allowing to start/stop all ceph-${daemonType} services at once";

     

       53
       53
       +
               partOf = [ "ceph.target" ];

     

       54
       54
       +
               before = [ "ceph.target" ];

     

       55
       55
       +
             };

     

       56
       56
       +
           }

     

       57
       57
       +
         );

     

       58
       58
       +
       in 

     

       59
       59
       +
       {

     

       60
       60
       +
         options.services.ceph = {

     

       61
       61
       +
           # Ceph has a monolithic configuration file but different sections for

     

       62
       62
       +
           # each daemon, a separate client section and a global section

     

       63
       63
       +
           enable = mkEnableOption "Ceph global configuration";

     

       64
       64
       +
       

     

       65
       65
       +
           global = {

     

       66
       66
       +
             fsid = mkOption {

     

       67
       67
       +
               type = types.str;

     

       68
       68
       +
               example = ''

     

       69
       69
       +
                 433a2193-4f8a-47a0-95d2-209d7ca2cca5

     

       70
       70
       +
               '';

     

       71
       71
       +
               description = ''

     

       72
       72
       +
                 Filesystem ID, a generated uuid, its must be generated and set before

     

       73
       73
       +
                 attempting to start a cluster

     

       74
       74
       +
               '';

     

       75
       75
       +
             };

     

       76
       76
       +
       

     

       77
       77
       +
             clusterName = mkOption {

     

       78
       78
       +
               type = types.str;

     

       79
       79
       +
               default = "ceph";

     

       80
       80
       +
               description = ''

     

       81
       81
       +
                 Name of cluster

     

       82
       82
       +
               '';

     

       83
       83
       +
             };

     

       84
       84
       +
       

     

       85
       85
       +
             monInitialMembers = mkOption {

     

       86
       86
       +
               type = with types; nullOr commas;

     

       87
       87
       +
               default = null;

     

       88
       88
       +
               example = ''

     

       89
       89
       +
                 node0, node1, node2 

     

       90
       90
       +
               '';

     

       91
       91
       +
               description = ''

     

       92
       92
       +
                 List of hosts that will be used as monitors at startup.

     

       93
       93
       +
               '';

     

       94
       94
       +
             };

     

       95
       95
       +
       

     

       96
       96
       +
             monHost = mkOption {

     

       97
       97
       +
               type = with types; nullOr commas;

     

       98
       98
       +
               default = null;

     

       99
       99
       +
               example = ''

     

       100
       100
       +
                 10.10.0.1, 10.10.0.2, 10.10.0.3

     

       101
       101
       +
               '';

     

       102
       102
       +
               description = ''

     

       103
       103
       +
                 List of hostname shortnames/IP addresses of the initial monitors.

     

       104
       104
       +
               '';

     

       105
       105
       +
             };

     

       106
       106
       +
       

     

       107
       107
       +
             maxOpenFiles = mkOption {

     

       108
       108
       +
               type = types.int;

     

       109
       109
       +
               default = 131072;

     

       110
       110
       +
               description = ''

     

       111
       111
       +
                 Max open files for each OSD daemon.

     

       112
       112
       +
               '';

     

       113
       113
       +
             };

     

       114
       114
       +
       

     

       115
       115
       +
             authClusterRequired = mkOption {

     

       116
       116
       +
               type = types.enum [ "cephx" "none" ];

     

       117
       117
       +
               default = "cephx";

     

       118
       118
       +
               description = ''

     

       119
       119
       +
                 Enables requiring daemons to authenticate with eachother in the cluster.

     

       120
       120
       +
               '';

     

       121
       121
       +
             };

     

       122
       122
       +
       

     

       123
       123
       +
             authServiceRequired = mkOption {

     

       124
       124
       +
               type = types.enum [ "cephx" "none" ];

     

       125
       125
       +
               default = "cephx";

     

       126
       126
       +
               description = ''

     

       127
       127
       +
                 Enables requiring clients to authenticate with the cluster to access services in the cluster (e.g. radosgw, mds or osd).

     

       128
       128
       +
               '';

     

       129
       129
       +
             };

     

       130
       130
       +
       

     

       131
       131
       +
             authClientRequired = mkOption {

     

       132
       132
       +
               type = types.enum [ "cephx" "none" ];

     

       133
       133
       +
               default = "cephx";

     

       134
       134
       +
               description = ''

     

       135
       135
       +
                 Enables requiring the cluster to authenticate itself to the client.

     

       136
       136
       +
               '';

     

       137
       137
       +
             };

     

       138
       138
       +
       

     

       139
       139
       +
             publicNetwork = mkOption {

     

       140
       140
       +
               type = with types; nullOr commas;

     

       141
       141
       +
               default = null;

     

       142
       142
       +
               example = ''

     

       143
       143
       +
                 10.20.0.0/24, 192.168.1.0/24

     

       144
       144
       +
               '';

     

       145
       145
       +
               description = ''

     

       146
       146
       +
                 A comma-separated list of subnets that will be used as public networks in the cluster.

     

       147
       147
       +
               '';

     

       148
       148
       +
             };

     

       149
       149
       +
       

     

       150
       150
       +
             clusterNetwork = mkOption {

     

       151
       151
       +
               type = with types; nullOr commas;

     

       152
       152
       +
               default = null;

     

       153
       153
       +
               example = ''

     

       154
       154
       +
                 10.10.0.0/24, 192.168.0.0/24

     

       155
       155
       +
               '';

     

       156
       156
       +
               description = ''

     

       157
       157
       +
                 A comma-separated list of subnets that will be used as cluster networks in the cluster.

     

       158
       158
       +
               '';

     

       159
       159
       +
             };

     

       160
       160
       +
           };

     

       161
       161
       +
       

     

       162
       162
       +
           mgr = {

     

       163
       163
       +
             enable = mkEnableOption "Ceph MGR daemon";

     

       164
       164
       +
             daemons = mkOption {

     

       165
       165
       +
               type = with types; listOf str;

     

       166
       166
       +
               default = [];

     

       167
       167
       +
               example = ''

     

       168
       168
       +
                 [ "name1" "name2" ];

     

       169
       169
       +
               '';

     

       170
       170
       +
               description = ''

     

       171
       171
       +
                 A list of names for manager daemons that should have a service created. The names correspond

     

       172
       172
       +
                 to the id part in ceph i.e. [ "name1" ] would result in mgr.name1

     

       173
       173
       +
               '';

     

       174
       174
       +
             };

     

       175
       175
       +
             extraConfig = mkOption {

     

       176
       176
       +
               type = with types; attrsOf str;

     

       177
       177
       +
               default = {};

     

       178
       178
       +
               description = ''

     

       179
       179
       +
                 Extra configuration to add to the global section for manager daemons.

     

       180
       180
       +
               '';

     

       181
       181
       +
             };

     

       182
       182
       +
           };

     

       183
       183
       +
       

     

       184
       184
       +
           mon = {

     

       185
       185
       +
             enable = mkEnableOption "Ceph MON daemon";

     

       186
       186
       +
             daemons = mkOption {

     

       187
       187
       +
               type = with types; listOf str;

     

       188
       188
       +
               default = [];

     

       189
       189
       +
               example = ''

     

       190
       190
       +
                 [ "name1" "name2" ];

     

       191
       191
       +
               '';

     

       192
       192
       +
               description = ''

     

       193
       193
       +
                 A list of monitor daemons that should have a service created. The names correspond

     

       194
       194
       +
                 to the id part in ceph i.e. [ "name1" ] would result in mon.name1

     

       195
       195
       +
               '';

     

       196
       196
       +
             };

     

       197
       197
       +
             extraConfig = mkOption {

     

       198
       198
       +
               type = with types; attrsOf str;

     

       199
       199
       +
               default = {};

     

       200
       200
       +
               description = ''

     

       201
       201
       +
                 Extra configuration to add to the monitor section.

     

       202
       202
       +
               '';

     

       203
       203
       +
             };

     

       204
       204
       +
           };

     

       205
       205
       +
       

     

       206
       206
       +
           osd = {

     

       207
       207
       +
             enable = mkEnableOption "Ceph OSD daemon";

     

       208
       208
       +
             daemons = mkOption {

     

       209
       209
       +
               type = with types; listOf str;

     

       210
       210
       +
               default = [];

     

       211
       211
       +
               example = ''

     

       212
       212
       +
                 [ "name1" "name2" ];

     

       213
       213
       +
               '';

     

       214
       214
       +
               description = ''

     

       215
       215
       +
                 A list of OSD daemons that should have a service created. The names correspond

     

       216
       216
       +
                 to the id part in ceph i.e. [ "name1" ] would result in osd.name1

     

       217
       217
       +
               '';

     

       218
       218
       +
             };

     

       219
       219
       +
             extraConfig = mkOption {

     

       220
       220
       +
               type = with types; attrsOf str;

     

       221
       221
       +
               default = {

     

       222
       222
       +
                 "osd journal size" = "10000";

     

       223
       223
       +
                 "osd pool default size" = "3";

     

       224
       224
       +
                 "osd pool default min size" = "2";

     

       225
       225
       +
                 "osd pool default pg num" = "200";

     

       226
       226
       +
                 "osd pool default pgp num" = "200";

     

       227
       227
       +
                 "osd crush chooseleaf type" = "1";

     

       228
       228
       +
               };

     

       229
       229
       +
               description = ''

     

       230
       230
       +
                 Extra configuration to add to the OSD section.

     

       231
       231
       +
               '';

     

       232
       232
       +
             };

     

       233
       233
       +
           };

     

       234
       234
       +
       

     

       235
       235
       +
           mds = {

     

       236
       236
       +
             enable = mkEnableOption "Ceph MDS daemon";

     

       237
       237
       +
             daemons = mkOption {

     

       238
       238
       +
               type = with types; listOf str;

     

       239
       239
       +
               default = [];

     

       240
       240
       +
               example = ''

     

       241
       241
       +
                 [ "name1" "name2" ];

     

       242
       242
       +
               '';

     

       243
       243
       +
               description = ''

     

       244
       244
       +
                 A list of metadata service daemons that should have a service created. The names correspond

     

       245
       245
       +
                 to the id part in ceph i.e. [ "name1" ] would result in mds.name1

     

       246
       246
       +
               '';

     

       247
       247
       +
             };

     

       248
       248
       +
             extraConfig = mkOption {

     

       249
       249
       +
               type = with types; attrsOf str;

     

       250
       250
       +
               default = {};

     

       251
       251
       +
               description = ''

     

       252
       252
       +
                 Extra configuration to add to the MDS section.

     

       253
       253
       +
               '';

     

       254
       254
       +
             };

     

       255
       255
       +
           };

     

       256
       256
       +
       

     

       257
       257
       +
           rgw = {

     

       258
       258
       +
             enable = mkEnableOption "Ceph RadosGW daemon";

     

       259
       259
       +
             daemons = mkOption {

     

       260
       260
       +
               type = with types; listOf str;

     

       261
       261
       +
               default = [];

     

       262
       262
       +
               example = ''

     

       263
       263
       +
                 [ "name1" "name2" ];

     

       264
       264
       +
               '';

     

       265
       265
       +
               description = ''

     

       266
       266
       +
                 A list of rados gateway daemons that should have a service created. The names correspond

     

       267
       267
       +
                 to the id part in ceph i.e. [ "name1" ] would result in client.name1, radosgw daemons

     

       268
       268
       +
                 aren't daemons to cluster in the sense that OSD, MGR or MON daemons are. They are simply

     

       269
       269
       +
                 daemons, from ceph, that uses the cluster as a backend.

     

       270
       270
       +
               '';

     

       271
       271
       +
             };

     

       272
       272
       +
           };

     

       273
       273
       +
       

     

       274
       274
       +
           client = {

     

       275
       275
       +
             enable = mkEnableOption "Ceph client configuration";

     

       276
       276
       +
             extraConfig = mkOption {

     

       277
       277
       +
               type = with types; attrsOf str;

     

       278
       278
       +
               default = {};

     

       279
       279
       +
               example = ''

     

       280
       280
       +
                 {

     

       281
       281
       +
                   # This would create a section for a radosgw daemon named node0 and related

     

       282
       282
       +
                   # configuration for it

     

       283
       283
       +
                   "client.radosgw.node0" = { "some config option" = "true"; };

     

       284
       284
       +
                 };

     

       285
       285
       +
               '';

     

       286
       286
       +
               description = ''

     

       287
       287
       +
                 Extra configuration to add to the client section. Configuration for rados gateways

     

       288
       288
       +
                 would be added here, with their own sections, see example.

     

       289
       289
       +
               '';

     

       290
       290
       +
             };

     

       291
       291
       +
           };

     

       292
       292
       +
         };

     

       293
       293
       +
       

     

       294
       294
       +
         config = mkIf config.services.ceph.enable {

     

       295
       295
       +
           assertions = [

     

       296
       296
       +
             { assertion = cfg.global.fsid != "";

     

       297
       297
       +
               message = "fsid has to be set to a valid uuid for the cluster to function";

     

       298
       298
       +
             }

     

       299
       299
       +
             { assertion = cfg.mgr.enable == true;

     

       300
       300
       +
               message = "ceph 12.x requires atleast 1 MGR daemon enabled for the cluster to function";

     

       301
       301
       +
             }

     

       302
       302
       +
             { assertion = cfg.mon.enable == true -> cfg.mon.daemons != [];

     

       303
       303
       +
               message = "have to set id of atleast one MON if you're going to enable Monitor";

     

       304
       304
       +
             }

     

       305
       305
       +
             { assertion = cfg.mds.enable == true -> cfg.mds.daemons != [];

     

       306
       306
       +
               message = "have to set id of atleast one MDS if you're going to enable Metadata Service";

     

       307
       307
       +
             }

     

       308
       308
       +
             { assertion = cfg.osd.enable == true -> cfg.osd.daemons != [];

     

       309
       309
       +
               message = "have to set id of atleast one OSD if you're going to enable OSD";

     

       310
       310
       +
             }

     

       311
       311
       +
             { assertion = cfg.mgr.enable == true -> cfg.mgr.daemons != [];

     

       312
       312
       +
               message = "have to set id of atleast one MGR if you're going to enable MGR";

     

       313
       313
       +
             }

     

       314
       314
       +
           ];

     

       315
       315
       +
       

     

       316
       316
       +
           warnings = optional (cfg.global.monInitialMembers == null) 

     

       317
       317
       +
             ''Not setting up a list of members in monInitialMembers requires that you set the host variable for each mon daemon or else the cluster won't function'';

     

       318
       318
       +
           

     

       319
       319
       +
           environment.etc."ceph/ceph.conf".text = let

     

       320
       320
       +
             # Translate camelCaseOptions to the expected camel case option for ceph.conf

     

       321
       321
       +
             translatedGlobalConfig = mapAttrs' (name: value: nameValuePair (translateOption name) value) cfg.global;

     

       322
       322
       +
             # Merge the extraConfig set for mgr daemons, as mgr don't have their own section

     

       323
       323
       +
             globalAndMgrConfig = translatedGlobalConfig // optionalAttrs cfg.mgr.enable cfg.mgr.extraConfig;

     

       324
       324
       +
             # Remove all name-value pairs with null values from the attribute set to avoid making empty sections in the ceph.conf

     

       325
       325
       +
             globalConfig = mapAttrs' (name: value: nameValuePair (translateOption name) value) (filterAttrs (name: value: value != null) globalAndMgrConfig);

     

       326
       326
       +
             totalConfig = {

     

       327
       327
       +
                 "global" = globalConfig;

     

       328
       328
       +
               } // optionalAttrs (cfg.mon.enable && cfg.mon.extraConfig != {}) { "mon" = cfg.mon.extraConfig; }

     

       329
       329
       +
                 // optionalAttrs (cfg.mds.enable && cfg.mds.extraConfig != {}) { "mds" = cfg.mds.extraConfig; }

     

       330
       330
       +
                 // optionalAttrs (cfg.osd.enable && cfg.osd.extraConfig != {}) { "osd" = cfg.osd.extraConfig; }

     

       331
       331
       +
                 // optionalAttrs (cfg.client.enable && cfg.client.extraConfig != {})  cfg.client.extraConfig;

     

       332
       332
       +
             in

     

       333
       333
       +
               generators.toINI {} totalConfig;

     

       334
       334
       +
       

     

       335
       335
       +
           users.extraUsers = singleton {

     

       336
       336
       +
             name = "ceph";

     

       337
       337
       +
             uid = config.ids.uids.ceph;

     

       338
       338
       +
             description = "Ceph daemon user";

     

       339
       339
       +
           };

     

       340
       340
       +
       

     

       341
       341
       +
           users.extraGroups = singleton {

     

       342
       342
       +
             name = "ceph";

     

       343
       343
       +
             gid = config.ids.gids.ceph;

     

       344
       344
       +
           };

     

       345
       345
       +
       

     

       346
       346
       +
           systemd.services = let

     

       347
       347
       +
             services = [] 

     

       348
       348
       +
               ++ optional cfg.mon.enable (generateDaemonList "mon" cfg.mon.daemons { RestartSec = "10"; }) 

     

       349
       349
       +
               ++ optional cfg.mds.enable (generateDaemonList "mds" cfg.mds.daemons { StartLimitBurst = "3"; })

     

       350
       350
       +
               ++ optional cfg.osd.enable (generateDaemonList "osd" cfg.osd.daemons { StartLimitBurst = "30"; RestartSec = "20s"; })

     

       351
       351
       +
               ++ optional cfg.rgw.enable (generateDaemonList "rgw" cfg.rgw.daemons { })

     

       352
       352
       +
               ++ optional cfg.mgr.enable (generateDaemonList "mgr" cfg.mgr.daemons { StartLimitBurst = "3"; });

     

       353
       353
       +
             in 

     

       354
       354
       +
               mkMerge services;

     

       355
       355
       +
       

     

       356
       356
       +
           systemd.targets = let

     

       357
       357
       +
             targets = [

     

       358
       358
       +
               { "ceph" = { description = "Ceph target allowing to start/stop all ceph service instances at once"; }; }

     

       359
       359
       +
             ] ++ optional cfg.mon.enable (generateTargetFile "mon")

     

       360
       360
       +
               ++ optional cfg.mds.enable (generateTargetFile "mds")

     

       361
       361
       +
               ++ optional cfg.osd.enable (generateTargetFile "osd")

     

       362
       362
       +
               ++ optional cfg.rgw.enable (generateTargetFile "rgw")

     

       363
       363
       +
               ++ optional cfg.mgr.enable (generateTargetFile "mgr");

     

       364
       364
       +
             in

     

       365
       365
       +
               mkMerge targets;

     

       366
       366
       +
       

     

       367
       367
       +
           systemd.tmpfiles.rules = [

     

       368
       368
       +
             "d /run/ceph 0770 ceph ceph -"

     

       369
       369
       +
           ];

     

       370
       370
       +
         };

     

       371
       371
       +
       }

nixos/release.nix

···

       230
       230
        
         tests.borgbackup = callTest tests/borgbackup.nix {};

     

       231
       231
        
         tests.buildbot = callTest tests/buildbot.nix {};

     

       232
       232
        
         tests.cadvisor = callTestOnTheseSystems ["x86_64-linux"] tests/cadvisor.nix {};

     

       233
       233
       +
         tests.ceph = callTestOnTheseSystems ["x86_64-linux"] tests/ceph.nix {};

     

       233
       234
        
         tests.chromium = (callSubTestsOnTheseSystems ["x86_64-linux"] tests/chromium.nix {}).stable;

     

       234
       235
        
         tests.cjdns = callTest tests/cjdns.nix {};

     

       235
       236
        
         tests.cloud-init = callTest tests/cloud-init.nix {};

+140

nixos/tests/ceph.nix

···

       1
       1
       +
       import ./make-test.nix ({pkgs, ...}: rec {

     

       2
       2
       +
         name = "All-in-one-basic-ceph-cluster";

     

       3
       3
       +
         meta = with pkgs.stdenv.lib.maintainers; {

     

       4
       4
       +
           maintainers = [ lejonet ];

     

       5
       5
       +
         };

     

       6
       6
       +
       

     

       7
       7
       +
         nodes = {

     

       8
       8
       +
           aio = { config, pkgs, ... }: {

     

       9
       9
       +
             virtualisation = {

     

       10
       10
       +
               emptyDiskImages = [ 20480 20480 ];

     

       11
       11
       +
               vlans = [ 1 ];

     

       12
       12
       +
             };

     

       13
       13
       +
             

     

       14
       14
       +
             networking = {

     

       15
       15
       +
               firewall.allowPing = true;

     

       16
       16
       +
               useDHCP = false;

     

       17
       17
       +
               interfaces.eth1.ipv4.addresses = pkgs.lib.mkOverride 0 [

     

       18
       18
       +
                 { address = "192.168.1.1"; prefixLength = 24; }

     

       19
       19
       +
               ];

     

       20
       20
       +
             };

     

       21
       21
       +
       

     

       22
       22
       +
             environment.systemPackages = with pkgs; [

     

       23
       23
       +
               bash

     

       24
       24
       +
               sudo

     

       25
       25
       +
               ceph

     

       26
       26
       +
               xfsprogs

     

       27
       27
       +
             ];

     

       28
       28
       +
             nixpkgs.config.packageOverrides = super: {

     

       29
       29
       +
               ceph = super.ceph.override({ nss = super.nss; libxfs = super.libxfs; libaio = super.libaio; jemalloc = super.jemalloc; });

     

       30
       30
       +
             };

     

       31
       31
       +
       

     

       32
       32
       +
             boot.kernelModules = [ "xfs" ];

     

       33
       33
       +
       

     

       34
       34
       +
             services.ceph.enable = true;

     

       35
       35
       +
             services.ceph.global = {

     

       36
       36
       +
               fsid = "066ae264-2a5d-4729-8001-6ad265f50b03";

     

       37
       37
       +
               monInitialMembers = "aio";

     

       38
       38
       +
               monHost = "192.168.1.1";

     

       39
       39
       +
             };

     

       40
       40
       +
       

     

       41
       41
       +
             services.ceph.mon = {

     

       42
       42
       +
               enable = true;

     

       43
       43
       +
               daemons = [ "aio" ];

     

       44
       44
       +
             };

     

       45
       45
       +
       

     

       46
       46
       +
             services.ceph.mgr = {

     

       47
       47
       +
               enable = true;

     

       48
       48
       +
               daemons = [ "aio" ];

     

       49
       49
       +
             };

     

       50
       50
       +
       

     

       51
       51
       +
             services.ceph.osd = {

     

       52
       52
       +
               enable = true;

     

       53
       53
       +
               daemons = [ "0" "1" ];

     

       54
       54
       +
             };

     

       55
       55
       +
           };

     

       56
       56
       +
         };

     

       57
       57
       +
         

     

       58
       58
       +
         testScript = { nodes, ... }: ''

     

       59
       59
       +
           startAll;

     

       60
       60
       +
       

     

       61
       61
       +
           $aio->waitForUnit("network.target");

     

       62
       62
       +
       

     

       63
       63
       +
           # Create the ceph-related directories

     

       64
       64
       +
           $aio->mustSucceed(

     

       65
       65
       +
             "mkdir -p /var/lib/ceph/mgr/ceph-aio/",

     

       66
       66
       +
             "mkdir -p /var/lib/ceph/mon/ceph-aio/",

     

       67
       67
       +
             "mkdir -p /var/lib/ceph/osd/ceph-{0..1}/",

     

       68
       68
       +
             "chown ceph:ceph -R /var/lib/ceph/"

     

       69
       69
       +
           );

     

       70
       70
       +
       

     

       71
       71
       +
           # Bootstrap ceph-mon daemon

     

       72
       72
       +
           $aio->mustSucceed(

     

       73
       73
       +
             "mkdir -p /var/lib/ceph/bootstrap-osd && chown ceph:ceph /var/lib/ceph/bootstrap-osd",

     

       74
       74
       +
             "sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'",

     

       75
       75
       +
             "ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'",

     

       76
       76
       +
             "ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring",

     

       77
       77
       +
                   "monmaptool --create --add aio 192.168.1.1 --fsid 066ae264-2a5d-4729-8001-6ad265f50b03 /tmp/monmap",

     

       78
       78
       +
             "sudo -u ceph ceph-mon --mkfs -i aio --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring",

     

       79
       79
       +
             "touch /var/lib/ceph/mon/ceph-aio/done",

     

       80
       80
       +
             "systemctl start ceph-mon-aio"

     

       81
       81
       +
           );

     

       82
       82
       +
           $aio->waitForUnit("ceph-mon-aio");

     

       83
       83
       +
       

     

       84
       84
       +
           # Can't check ceph status until a mon is up

     

       85
       85
       +
           $aio->succeed("ceph -s | grep 'mon: 1 daemons'");

     

       86
       86
       +
                 

     

       87
       87
       +
           # Start the ceph-mgr daemon, it has no deps and hardly any setup

     

       88
       88
       +
           $aio->mustSucceed(

     

       89
       89
       +
             "ceph auth get-or-create mgr.aio mon 'allow profile mgr' osd 'allow *' mds 'allow *' > /var/lib/ceph/mgr/ceph-aio/keyring",

     

       90
       90
       +
             "systemctl start ceph-mgr-aio"

     

       91
       91
       +
           );

     

       92
       92
       +
           $aio->waitForUnit("ceph-mgr-aio");

     

       93
       93
       +
           $aio->waitUntilSucceeds("ceph -s | grep 'quorum aio'");

     

       94
       94
       +
       

     

       95
       95
       +
           # Bootstrap both OSDs

     

       96
       96
       +
           $aio->mustSucceed(

     

       97
       97
       +
             "mkfs.xfs /dev/vdb",

     

       98
       98
       +
             "mkfs.xfs /dev/vdc",

     

       99
       99
       +
             "mount /dev/vdb /var/lib/ceph/osd/ceph-0",

     

       100
       100
       +
             "mount /dev/vdc /var/lib/ceph/osd/ceph-1",

     

       101
       101
       +
             "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-0/keyring --name osd.0 --add-key AQBCEJNa3s8nHRAANvdsr93KqzBznuIWm2gOGg==",

     

       102
       102
       +
             "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-1/keyring --name osd.1 --add-key AQBEEJNac00kExAAXEgy943BGyOpVH1LLlHafQ==",

     

       103
       103
       +
             "echo '{\"cephx_secret\": \"AQBCEJNa3s8nHRAANvdsr93KqzBznuIWm2gOGg==\"}' | ceph osd new 55ba2294-3e24-478f-bee0-9dca4c231dd9 -i -",

     

       104
       104
       +
             "echo '{\"cephx_secret\": \"AQBEEJNac00kExAAXEgy943BGyOpVH1LLlHafQ==\"}' | ceph osd new 5e97a838-85b6-43b0-8950-cb56d554d1e5 -i -"

     

       105
       105
       +
           );

     

       106
       106
       +
       

     

       107
       107
       +
           # Initialize the OSDs with regular filestore

     

       108
       108
       +
           $aio->mustSucceed(

     

       109
       109
       +
             "ceph-osd -i 0 --mkfs --osd-uuid 55ba2294-3e24-478f-bee0-9dca4c231dd9",

     

       110
       110
       +
             "ceph-osd -i 1 --mkfs --osd-uuid 5e97a838-85b6-43b0-8950-cb56d554d1e5",

     

       111
       111
       +
             "chown -R ceph:ceph /var/lib/ceph/osd",

     

       112
       112
       +
             "systemctl start ceph-osd-0",

     

       113
       113
       +
             "systemctl start ceph-osd-1"

     

       114
       114
       +
           );

     

       115
       115
       +
       

     

       116
       116
       +
           $aio->waitUntilSucceeds("ceph osd stat | grep '2 osds: 2 up, 2 in'");

     

       117
       117
       +
           $aio->waitUntilSucceeds("ceph -s | grep 'mgr: aio(active)'");

     

       118
       118
       +
           $aio->waitUntilSucceeds("ceph -s | grep 'HEALTH_OK'");

     

       119
       119
       +
       

     

       120
       120
       +
           $aio->mustSucceed(

     

       121
       121
       +
             "ceph osd pool create aio-test 100 100",

     

       122
       122
       +
             "ceph osd pool ls | grep 'aio-test'",

     

       123
       123
       +
             "ceph osd pool rename aio-test aio-other-test",

     

       124
       124
       +
             "ceph osd pool ls | grep 'aio-other-test'",

     

       125
       125
       +
             "ceph -s | grep '1 pools, 100 pgs'",

     

       126
       126
       +
             "ceph osd getcrushmap -o crush",

     

       127
       127
       +
             "crushtool -d crush -o decrushed",

     

       128
       128
       +
             "sed 's/step chooseleaf firstn 0 type host/step chooseleaf firstn 0 type osd/' decrushed > modcrush",

     

       129
       129
       +
             "crushtool -c modcrush -o recrushed",

     

       130
       130
       +
             "ceph osd setcrushmap -i recrushed",

     

       131
       131
       +
             "ceph osd pool set aio-other-test size 2"

     

       132
       132
       +
           );

     

       133
       133
       +
           $aio->waitUntilSucceeds("ceph -s | grep 'HEALTH_OK'");

     

       134
       134
       +
           $aio->waitUntilSucceeds("ceph -s | grep '100 active+clean'");

     

       135
       135
       +
           $aio->mustFail(

     

       136
       136
       +
             "ceph osd pool ls | grep 'aio-test'",

     

       137
       137
       +
             "ceph osd pool delete aio-other-test aio-other-test --yes-i-really-really-mean-it"

     

       138
       138
       +
           );

     

       139
       139
       +
         '';

     

       140
       140
       +
       })