···

       
301
       
 
       
    "aarch64-linux"

     

       
302
       
 
       
    "x86_64-linux"

     

       
303
       
 
       
  ] ./ceph-single-node-bluestore.nix;

     

       
304
       
-
       
  ceph-single-node-bluestore-dmcrypt = handleTestOn [

     

       
305
       
 
       
    "aarch64-linux"

     

       
306
       
 
       
    "x86_64-linux"

     

       
307
       
-
       
  ] ./ceph-single-node-bluestore-dmcrypt.nix { };

     

       
308
       
 
       
  certmgr = handleTest ./certmgr.nix { };

     

       
309
       
 
       
  cfssl = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./cfssl.nix { };

     

       
310
       
 
       
  cgit = runTest ./cgit.nix;

     

···

       
301
       
 
       
    "aarch64-linux"

     

       
302
       
 
       
    "x86_64-linux"

     

       
303
       
 
       
  ] ./ceph-single-node-bluestore.nix;

     

       
304
       
+
       
  ceph-single-node-bluestore-dmcrypt = runTestOn [

     

       
305
       
 
       
    "aarch64-linux"

     

       
306
       
 
       
    "x86_64-linux"

     

       
307
       
+
       
  ] ./ceph-single-node-bluestore-dmcrypt.nix;

     

       
308
       
 
       
  certmgr = handleTest ./certmgr.nix { };

     

       
309
       
 
       
  cfssl = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./cfssl.nix { };

     

       
310
       
 
       
  cgit = runTest ./cgit.nix;

     

···

       
1
       
-
       
import ./make-test-python.nix (

     

       
2
       
-
       
  { pkgs, lib, ... }:

     

       
3
       
 
       


     

       
4
       
-
       
  let

     

       
5
       
-
       
    # the single node ipv6 address

     

       
6
       
-
       
    ip = "2001:db8:ffff::";

     

       
7
       
-
       
    # the global ceph cluster id

     

       
8
       
-
       
    cluster = "54465b37-b9d8-4539-a1f9-dd33c75ee45a";

     

       
9
       
-
       
    # the fsids of OSDs

     

       
10
       
-
       
    osd-fsid-map = {

     

       
11
       
-
       
      "0" = "1c1b7ea9-06bf-4d30-9a01-37ac3a0254aa";

     

       
12
       
-
       
      "1" = "bd5a6f49-69d5-428c-ac25-a99f0c44375c";

     

       
13
       
-
       
      "2" = "c90de6c7-86c6-41da-9694-e794096dfc5c";

     

       
14
       
-
       
    };

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
15
       
 
       


     

       
16
       
-
       
  in

     

       
17
       
-
       
  {

     

       
18
       
-
       
    name = "basic-single-node-ceph-cluster-bluestore-dmcrypt";

     

       
19
       
-
       
    meta = {

     

       
20
       
-
       
      maintainers = with lib.maintainers; [

     

       
21
       
-
       
        benaryorg

     

       
22
       
-
       
        nh2

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
23
       
 
       
      ];

     

       
24
       
-
       
    };

     

       
25
       
 
       


     

       
26
       
-
       
    nodes = {

     

       
27
       
-
       
      ceph =

     

       
28
       
-
       
        { pkgs, config, ... }:

     

       
29
       
-
       
        {

     

       
30
       
-
       
          # disks for bluestore

     

       
31
       
-
       
          virtualisation.emptyDiskImages = [

     

       
32
       
-
       
            20480

     

       
33
       
-
       
            20480

     

       
34
       
-
       
            20480

     

       
35
       
-
       
          ];

     

       
36
       
-
       


     

       
37
       
-
       
          # networking setup (no external connectivity required, only local IPv6)

     

       
38
       
-
       
          networking.useDHCP = false;

     

       
39
       
-
       
          systemd.network = {

     

       
40
       
 
       
            enable = true;

     

       
41
       
-
       
            wait-online.extraArgs = [

     

       
42
       
-
       
              "-i"

     

       
43
       
-
       
              "lo"

     

       
44
       
-
       
            ];

     

       
45
       
-
       
            networks = {

     

       
46
       
-
       
              "40-loopback" = {

     

       
47
       
-
       
                enable = true;

     

       
48
       
-
       
                name = "lo";

     

       
49
       
-
       
                DHCP = "no";

     

       
50
       
-
       
                addresses = [ { Address = "${ip}/128"; } ];

     

       
51
       
-
       
              };

     

       
52
       
-
       
            };

     

       
53
       
 
       
          };

     

       
0
       
 
       

     

       
0
       
 
       

     

       
54
       
 
       


     

       
55
       
-
       
          # do not start the ceph target by default so we can format the disks first

     

       
56
       
-
       
          systemd.targets.ceph.wantedBy = lib.mkForce [ ];

     

       
57
       
 
       


     

       
58
       
-
       
          # add the packages to systemPackages so the testscript doesn't run into any unexpected issues

     

       
59
       
-
       
          # this shouldn't be required on production systems which have their required packages in the unit paths only

     

       
60
       
-
       
          # but it helps in case one needs to actually run the tooling anyway

     

       
61
       
-
       
          environment.systemPackages = with pkgs; [

     

       
62
       
-
       
            ceph

     

       
63
       
-
       
            cryptsetup

     

       
64
       
-
       
            lvm2

     

       
65
       
-
       
          ];

     

       
66
       
 
       


     

       
67
       
-
       
          services.ceph = {

     

       
68
       
-
       
            enable = true;

     

       
69
       
-
       
            client.enable = true;

     

       
70
       
-
       
            extraConfig = {

     

       
71
       
-
       
              public_addr = ip;

     

       
72
       
-
       
              cluster_addr = ip;

     

       
73
       
-
       
              # ipv6

     

       
74
       
-
       
              ms_bind_ipv4 = "false";

     

       
75
       
-
       
              ms_bind_ipv6 = "true";

     

       
76
       
-
       
              # msgr2 settings

     

       
77
       
-
       
              ms_cluster_mode = "secure";

     

       
78
       
-
       
              ms_service_mode = "secure";

     

       
79
       
-
       
              ms_client_mode = "secure";

     

       
80
       
-
       
              ms_mon_cluster_mode = "secure";

     

       
81
       
-
       
              ms_mon_service_mode = "secure";

     

       
82
       
-
       
              ms_mon_client_mode = "secure";

     

       
83
       
-
       
              # less default modules, cuts down on memory and startup time in the tests

     

       
84
       
-
       
              mgr_initial_modules = "";

     

       
85
       
-
       
              # distribute by OSD, not by host, as per https://docs.ceph.com/en/reef/cephadm/install/#single-host

     

       
86
       
-
       
              osd_crush_chooseleaf_type = "0";

     

       
87
       
-
       
            };

     

       
88
       
-
       
            client.extraConfig."mon.0" = {

     

       
89
       
-
       
              host = "ceph";

     

       
90
       
-
       
              mon_addr = "v2:[${ip}]:3300";

     

       
91
       
-
       
              public_addr = "v2:[${ip}]:3300";

     

       
92
       
-
       
            };

     

       
93
       
-
       
            global = {

     

       
94
       
-
       
              fsid = cluster;

     

       
95
       
-
       
              clusterNetwork = "${ip}/64";

     

       
96
       
-
       
              publicNetwork = "${ip}/64";

     

       
97
       
-
       
              monInitialMembers = "0";

     

       
98
       
-
       
            };

     

       
99
       
 
       


     

       
100
       
-
       
            mon = {

     

       
101
       
-
       
              enable = true;

     

       
102
       
-
       
              daemons = [ "0" ];

     

       
103
       
-
       
            };

     

       
104
       
 
       


     

       
105
       
-
       
            osd = {

     

       
106
       
-
       
              enable = true;

     

       
107
       
-
       
              daemons = builtins.attrNames osd-fsid-map;

     

       
108
       
-
       
            };

     

       
109
       
 
       


     

       
110
       
-
       
            mgr = {

     

       
111
       
-
       
              enable = true;

     

       
112
       
-
       
              daemons = [ "ceph" ];

     

       
113
       
-
       
            };

     

       
114
       
-
       
          };

     

       
115
       
 
       


     

       
116
       
-
       
          systemd.services =

     

       
117
       
-
       
            let

     

       
118
       
-
       
              osd-name = id: "ceph-osd-${id}";

     

       
119
       
-
       
              osd-pre-start = id: [

     

       
120
       
-
       
                "!${config.services.ceph.osd.package.out}/bin/ceph-volume lvm activate --bluestore ${id} ${osd-fsid-map.${id}} --no-systemd"

     

       
121
       
-
       
                "${config.services.ceph.osd.package.lib}/libexec/ceph/ceph-osd-prestart.sh --id ${id} --cluster ${config.services.ceph.global.clusterName}"

     

       
122
       
-
       
              ];

     

       
123
       
-
       
              osd-post-stop = id: [

     

       
124
       
-
       
                "!${config.services.ceph.osd.package.out}/bin/ceph-volume lvm deactivate ${id}"

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
125
       
 
       
              ];

     

       
126
       
-
       
              map-osd = id: {

     

       
127
       
-
       
                name = osd-name id;

     

       
128
       
-
       
                value = {

     

       
129
       
-
       
                  serviceConfig.ExecStartPre = lib.mkForce (osd-pre-start id);

     

       
130
       
-
       
                  serviceConfig.ExecStopPost = osd-post-stop id;

     

       
131
       
-
       
                  unitConfig.ConditionPathExists = lib.mkForce [ ];

     

       
132
       
-
       
                  unitConfig.StartLimitBurst = lib.mkForce 4;

     

       
133
       
-
       
                  path = with pkgs; [

     

       
134
       
-
       
                    util-linux

     

       
135
       
-
       
                    lvm2

     

       
136
       
-
       
                    cryptsetup

     

       
137
       
-
       
                  ];

     

       
138
       
-
       
                };

     

       
139
       
-
       
              };

     

       
140
       
-
       
            in

     

       
141
       
-
       
            lib.pipe config.services.ceph.osd.daemons [

     

       
142
       
-
       
              (builtins.map map-osd)

     

       
143
       
-
       
              builtins.listToAttrs

     

       
144
       
-
       
            ];

     

       
145
       
-
       
        };

     

       
146
       
 
       
    };

     

       
147
       
 
       


     

       
148
       
-
       
    testScript =

     

       
149
       
-
       
      { ... }:

     

       
150
       
-
       
      ''

     

       
151
       
-
       
        start_all()

     

       
152
       
 
       


     

       
153
       
-
       
        ceph.wait_for_unit("default.target")

     

       
154
       
 
       


     

       
155
       
-
       
        # Bootstrap ceph-mon daemon

     

       
156
       
-
       
        ceph.succeed(

     

       
157
       
-
       
            "mkdir -p /var/lib/ceph/bootstrap-osd",

     

       
158
       
-
       
            "ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'",

     

       
159
       
-
       
            "ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'",

     

       
160
       
-
       
            "ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r'",

     

       
161
       
-
       
            "ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring",

     

       
162
       
-
       
            "ceph-authtool /tmp/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring",

     

       
163
       
-
       
            "monmaptool --create --fsid ${cluster} --addv 0 'v2:[${ip}]:3300/0' --clobber /tmp/ceph.initial-monmap",

     

       
164
       
-
       
            "mkdir -p /var/lib/ceph/mon/ceph-0",

     

       
165
       
-
       
            "ceph-mon --mkfs -i 0 --monmap /tmp/ceph.initial-monmap --keyring /tmp/ceph.mon.keyring",

     

       
166
       
-
       
            "chown ceph:ceph -R /tmp/ceph.mon.keyring /var/lib/ceph",

     

       
167
       
-
       
            "systemctl start ceph-mon-0.service",

     

       
168
       
-
       
        )

     

       
169
       
 
       


     

       
170
       
-
       
        ceph.wait_for_unit("ceph-mon-0.service")

     

       
171
       
-
       
        # should the mon not start or bind for some reason this gives us a better error message than the config commands running into a timeout

     

       
172
       
-
       
        ceph.wait_for_open_port(3300, "${ip}")

     

       
173
       
-
       
        ceph.succeed(

     

       
174
       
-
       
            # required for HEALTH_OK

     

       
175
       
-
       
            "ceph config set mon auth_allow_insecure_global_id_reclaim false",

     

       
176
       
-
       
            # IPv6

     

       
177
       
-
       
            "ceph config set global ms_bind_ipv4 false",

     

       
178
       
-
       
            "ceph config set global ms_bind_ipv6 true",

     

       
179
       
-
       
            # the new (secure) protocol

     

       
180
       
-
       
            "ceph config set global ms_bind_msgr1 false",

     

       
181
       
-
       
            "ceph config set global ms_bind_msgr2 true",

     

       
182
       
-
       
            # just a small little thing

     

       
183
       
-
       
            "ceph config set mon mon_compact_on_start true",

     

       
184
       
-
       
        )

     

       
185
       
 
       


     

       
186
       
-
       
        # Can't check ceph status until a mon is up

     

       
187
       
-
       
        ceph.succeed("ceph -s | grep 'mon: 1 daemons'")

     

       
188
       
 
       


     

       
189
       
-
       
        # Bootstrap OSDs (do this before starting the mgr because cryptsetup and the mgr both eat a lot of memory)

     

       
190
       
-
       
        ceph.succeed(

     

       
191
       
-
       
            # this will automatically do what's required for LVM, cryptsetup, and stores all the data in Ceph's internal databases

     

       
192
       
-
       
            "ceph-volume lvm prepare --bluestore --data /dev/vdb --dmcrypt --no-systemd --osd-id 0 --osd-fsid ${osd-fsid-map."0"}",

     

       
193
       
-
       
            "ceph-volume lvm prepare --bluestore --data /dev/vdc --dmcrypt --no-systemd --osd-id 1 --osd-fsid ${osd-fsid-map."1"}",

     

       
194
       
-
       
            "ceph-volume lvm prepare --bluestore --data /dev/vdd --dmcrypt --no-systemd --osd-id 2 --osd-fsid ${osd-fsid-map."2"}",

     

       
195
       
-
       
            "sudo ceph-volume lvm deactivate 0",

     

       
196
       
-
       
            "sudo ceph-volume lvm deactivate 1",

     

       
197
       
-
       
            "sudo ceph-volume lvm deactivate 2",

     

       
198
       
-
       
            "chown -R ceph:ceph /var/lib/ceph",

     

       
199
       
-
       
        )

     

       
200
       
 
       


     

       
201
       
-
       
        # Start OSDs (again, argon2id eats memory, so this happens before starting the mgr)

     

       
202
       
-
       
        ceph.succeed(

     

       
203
       
-
       
            "systemctl start ceph-osd-0.service",

     

       
204
       
-
       
            "systemctl start ceph-osd-1.service",

     

       
205
       
-
       
            "systemctl start ceph-osd-2.service",

     

       
206
       
-
       
        )

     

       
207
       
-
       
        ceph.wait_until_succeeds("ceph -s | grep 'quorum 0'")

     

       
208
       
-
       
        ceph.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")

     

       
209
       
 
       


     

       
210
       
-
       
        # Start the ceph-mgr daemon, after copying in the keyring

     

       
211
       
-
       
        ceph.succeed(

     

       
212
       
-
       
            "mkdir -p /var/lib/ceph/mgr/ceph-ceph/",

     

       
213
       
-
       
            "ceph auth get-or-create -o /var/lib/ceph/mgr/ceph-ceph/keyring mgr.ceph mon 'allow profile mgr' osd 'allow *' mds 'allow *'",

     

       
214
       
-
       
            "chown -R ceph:ceph /var/lib/ceph/mgr/ceph-ceph/",

     

       
215
       
-
       
            "systemctl start ceph-mgr-ceph.service",

     

       
216
       
-
       
        )

     

       
217
       
-
       
        ceph.wait_for_unit("ceph-mgr-ceph")

     

       
218
       
-
       
        ceph.wait_until_succeeds("ceph -s | grep 'quorum 0'")

     

       
219
       
-
       
        ceph.wait_until_succeeds("ceph -s | grep 'mgr: ceph(active,'")

     

       
220
       
-
       
        ceph.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")

     

       
221
       
-
       
        ceph.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")

     

       
222
       
 
       


     

       
223
       
-
       
        # test the actual storage

     

       
224
       
-
       
        ceph.succeed(

     

       
225
       
-
       
            "ceph osd pool create single-node-test 32 32",

     

       
226
       
-
       
            "ceph osd pool ls | grep 'single-node-test'",

     

       
227
       
 
       


     

       
228
       
-
       
            # We need to enable an application on the pool, otherwise it will

     

       
229
       
-
       
            # stay unhealthy in state POOL_APP_NOT_ENABLED.

     

       
230
       
-
       
            # Creating a CephFS would do this automatically, but we haven't done that here.

     

       
231
       
-
       
            # See: https://docs.ceph.com/en/reef/rados/operations/pools/#associating-a-pool-with-an-application

     

       
232
       
-
       
            # We use the custom application name "nixos-test" for this.

     

       
233
       
-
       
            "ceph osd pool application enable single-node-test nixos-test",

     

       
234
       
 
       


     

       
235
       
-
       
            "ceph osd pool rename single-node-test single-node-other-test",

     

       
236
       
-
       
            "ceph osd pool ls | grep 'single-node-other-test'",

     

       
237
       
-
       
        )

     

       
238
       
-
       
        ceph.wait_until_succeeds("ceph -s | grep '2 pools, 33 pgs'")

     

       
239
       
-
       
        ceph.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")

     

       
240
       
-
       
        ceph.wait_until_succeeds("ceph -s | grep '33 active+clean'")

     

       
241
       
-
       
        ceph.fail(

     

       
242
       
-
       
            # the old pool should be gone

     

       
243
       
-
       
            "ceph osd pool ls | grep 'multi-node-test'",

     

       
244
       
-
       
            # deleting the pool should fail without setting mon_allow_pool_delete

     

       
245
       
-
       
            "ceph osd pool delete single-node-other-test single-node-other-test --yes-i-really-really-mean-it",

     

       
246
       
-
       
        )

     

       
247
       
 
       


     

       
248
       
-
       
        # rebooting gets rid of any potential tmpfs mounts or device-mapper devices

     

       
249
       
-
       
        ceph.shutdown()

     

       
250
       
-
       
        ceph.start()

     

       
251
       
-
       
        ceph.wait_for_unit("default.target")

     

       
252
       
 
       


     

       
253
       
-
       
        # Start it up (again OSDs first due to memory constraints of cryptsetup and mgr)

     

       
254
       
-
       
        ceph.systemctl("start ceph-mon-0.service")

     

       
255
       
-
       
        ceph.wait_for_unit("ceph-mon-0")

     

       
256
       
-
       
        ceph.systemctl("start ceph-osd-0.service")

     

       
257
       
-
       
        ceph.wait_for_unit("ceph-osd-0")

     

       
258
       
-
       
        ceph.systemctl("start ceph-osd-1.service")

     

       
259
       
-
       
        ceph.wait_for_unit("ceph-osd-1")

     

       
260
       
-
       
        ceph.systemctl("start ceph-osd-2.service")

     

       
261
       
-
       
        ceph.wait_for_unit("ceph-osd-2")

     

       
262
       
-
       
        ceph.systemctl("start ceph-mgr-ceph.service")

     

       
263
       
-
       
        ceph.wait_for_unit("ceph-mgr-ceph")

     

       
264
       
 
       


     

       
265
       
-
       
        # Ensure the cluster comes back up again

     

       
266
       
-
       
        ceph.succeed("ceph -s | grep 'mon: 1 daemons'")

     

       
267
       
-
       
        ceph.wait_until_succeeds("ceph -s | grep 'quorum 0'")

     

       
268
       
-
       
        ceph.wait_until_succeeds("ceph osd stat | grep -E '3 osds: 3 up[^,]*, 3 in'")

     

       
269
       
-
       
        ceph.wait_until_succeeds("ceph -s | grep 'mgr: ceph(active,'")

     

       
270
       
-
       
        ceph.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")

     

       
271
       
-
       
      '';

     

       
272
       
-
       
  }

     

       
273
       
-
       
)

     

···

       
1
       
+
       
{ lib, ... }:

     

       
0
       
 
       

     

       
2
       
 
       


     

       
3
       
+
       
let

     

       
4
       
+
       
  # the single node ipv6 address

     

       
5
       
+
       
  ip = "2001:db8:ffff::";

     

       
6
       
+
       
  # the global ceph cluster id

     

       
7
       
+
       
  cluster = "54465b37-b9d8-4539-a1f9-dd33c75ee45a";

     

       
8
       
+
       
  # the fsids of OSDs

     

       
9
       
+
       
  osd-fsid-map = {

     

       
10
       
+
       
    "0" = "1c1b7ea9-06bf-4d30-9a01-37ac3a0254aa";

     

       
11
       
+
       
    "1" = "bd5a6f49-69d5-428c-ac25-a99f0c44375c";

     

       
12
       
+
       
    "2" = "c90de6c7-86c6-41da-9694-e794096dfc5c";

     

       
13
       
+
       
  };

     

       
14
       
+
       
in

     

       
15
       
+
       
{

     

       
16
       
+
       
  name = "basic-single-node-ceph-cluster-bluestore-dmcrypt";

     

       
17
       
+
       
  meta.maintainers = with lib.maintainers; [

     

       
18
       
+
       
    benaryorg

     

       
19
       
+
       
    nh2

     

       
20
       
+
       
  ];

     

       
21
       
 
       


     

       
22
       
+
       
  nodes.ceph =

     

       
23
       
+
       
    {

     

       
24
       
+
       
      lib,

     

       
25
       
+
       
      pkgs,

     

       
26
       
+
       
      config,

     

       
27
       
+
       
      ...

     

       
28
       
+
       
    }:

     

       
29
       
+
       
    {

     

       
30
       
+
       
      # disks for bluestore

     

       
31
       
+
       
      virtualisation.emptyDiskImages = [

     

       
32
       
+
       
        20480

     

       
33
       
+
       
        20480

     

       
34
       
+
       
        20480

     

       
35
       
 
       
      ];

     

       
0
       
 
       

     

       
36
       
 
       


     

       
37
       
+
       
      # networking setup (no external connectivity required, only local IPv6)

     

       
38
       
+
       
      networking.useDHCP = false;

     

       
39
       
+
       
      systemd.network = {

     

       
40
       
+
       
        enable = true;

     

       
41
       
+
       
        wait-online.extraArgs = [

     

       
42
       
+
       
          "-i"

     

       
43
       
+
       
          "lo"

     

       
44
       
+
       
        ];

     

       
45
       
+
       
        networks = {

     

       
46
       
+
       
          "40-loopback" = {

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
47
       
 
       
            enable = true;

     

       
48
       
+
       
            name = "lo";

     

       
49
       
+
       
            DHCP = "no";

     

       
50
       
+
       
            addresses = [ { Address = "${ip}/128"; } ];

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
51
       
 
       
          };

     

       
52
       
+
       
        };

     

       
53
       
+
       
      };

     

       
54
       
 
       


     

       
55
       
+
       
      # do not start the ceph target by default so we can format the disks first

     

       
56
       
+
       
      systemd.targets.ceph.wantedBy = lib.mkForce [ ];

     

       
57
       
 
       


     

       
58
       
+
       
      # add the packages to systemPackages so the testscript doesn't run into any unexpected issues

     

       
59
       
+
       
      # this shouldn't be required on production systems which have their required packages in the unit paths only

     

       
60
       
+
       
      # but it helps in case one needs to actually run the tooling anyway

     

       
61
       
+
       
      environment.systemPackages = with pkgs; [

     

       
62
       
+
       
        ceph

     

       
63
       
+
       
        cryptsetup

     

       
64
       
+
       
        lvm2

     

       
65
       
+
       
      ];

     

       
66
       
 
       


     

       
67
       
+
       
      services.ceph = {

     

       
68
       
+
       
        enable = true;

     

       
69
       
+
       
        client.enable = true;

     

       
70
       
+
       
        extraConfig = {

     

       
71
       
+
       
          public_addr = ip;

     

       
72
       
+
       
          cluster_addr = ip;

     

       
73
       
+
       
          # ipv6

     

       
74
       
+
       
          ms_bind_ipv4 = "false";

     

       
75
       
+
       
          ms_bind_ipv6 = "true";

     

       
76
       
+
       
          # msgr2 settings

     

       
77
       
+
       
          ms_cluster_mode = "secure";

     

       
78
       
+
       
          ms_service_mode = "secure";

     

       
79
       
+
       
          ms_client_mode = "secure";

     

       
80
       
+
       
          ms_mon_cluster_mode = "secure";

     

       
81
       
+
       
          ms_mon_service_mode = "secure";

     

       
82
       
+
       
          ms_mon_client_mode = "secure";

     

       
83
       
+
       
          # less default modules, cuts down on memory and startup time in the tests

     

       
84
       
+
       
          mgr_initial_modules = "";

     

       
85
       
+
       
          # distribute by OSD, not by host, as per https://docs.ceph.com/en/reef/cephadm/install/#single-host

     

       
86
       
+
       
          osd_crush_chooseleaf_type = "0";

     

       
87
       
+
       
        };

     

       
88
       
+
       
        client.extraConfig."mon.0" = {

     

       
89
       
+
       
          host = "ceph";

     

       
90
       
+
       
          mon_addr = "v2:[${ip}]:3300";

     

       
91
       
+
       
          public_addr = "v2:[${ip}]:3300";

     

       
92
       
+
       
        };

     

       
93
       
+
       
        global = {

     

       
94
       
+
       
          fsid = cluster;

     

       
95
       
+
       
          clusterNetwork = "${ip}/64";

     

       
96
       
+
       
          publicNetwork = "${ip}/64";

     

       
97
       
+
       
          monInitialMembers = "0";

     

       
98
       
+
       
        };

     

       
99
       
 
       


     

       
100
       
+
       
        mon = {

     

       
101
       
+
       
          enable = true;

     

       
102
       
+
       
          daemons = [ "0" ];

     

       
103
       
+
       
        };

     

       
104
       
 
       


     

       
105
       
+
       
        osd = {

     

       
106
       
+
       
          enable = true;

     

       
107
       
+
       
          daemons = builtins.attrNames osd-fsid-map;

     

       
108
       
+
       
        };

     

       
109
       
 
       


     

       
110
       
+
       
        mgr = {

     

       
111
       
+
       
          enable = true;

     

       
112
       
+
       
          daemons = [ "ceph" ];

     

       
113
       
+
       
        };

     

       
114
       
+
       
      };

     

       
115
       
 
       


     

       
116
       
+
       
      systemd.services =

     

       
117
       
+
       
        let

     

       
118
       
+
       
          osd-name = id: "ceph-osd-${id}";

     

       
119
       
+
       
          osd-pre-start = id: [

     

       
120
       
+
       
            "!${config.services.ceph.osd.package.out}/bin/ceph-volume lvm activate --bluestore ${id} ${osd-fsid-map.${id}} --no-systemd"

     

       
121
       
+
       
            "${config.services.ceph.osd.package.lib}/libexec/ceph/ceph-osd-prestart.sh --id ${id} --cluster ${config.services.ceph.global.clusterName}"

     

       
122
       
+
       
          ];

     

       
123
       
+
       
          osd-post-stop = id: [

     

       
124
       
+
       
            "!${config.services.ceph.osd.package.out}/bin/ceph-volume lvm deactivate ${id}"

     

       
125
       
+
       
          ];

     

       
126
       
+
       
          map-osd = id: {

     

       
127
       
+
       
            name = osd-name id;

     

       
128
       
+
       
            value = {

     

       
129
       
+
       
              serviceConfig.ExecStartPre = lib.mkForce (osd-pre-start id);

     

       
130
       
+
       
              serviceConfig.ExecStopPost = osd-post-stop id;

     

       
131
       
+
       
              unitConfig.ConditionPathExists = lib.mkForce [ ];

     

       
132
       
+
       
              unitConfig.StartLimitBurst = lib.mkForce 4;

     

       
133
       
+
       
              path = with pkgs; [

     

       
134
       
+
       
                util-linux

     

       
135
       
+
       
                lvm2

     

       
136
       
+
       
                cryptsetup

     

       
137
       
 
       
              ];

     

       
138
       
+
       
            };

     

       
139
       
+
       
          };

     

       
140
       
+
       
        in

     

       
141
       
+
       
        lib.pipe config.services.ceph.osd.daemons [

     

       
142
       
+
       
          (builtins.map map-osd)

     

       
143
       
+
       
          builtins.listToAttrs

     

       
144
       
+
       
        ];

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
145
       
 
       
    };

     

       
146
       
 
       


     

       
147
       
+
       
  testScript = ''

     

       
148
       
+
       
    start_all()

     

       
0
       
 
       

     

       
0
       
 
       

     

       
149
       
 
       


     

       
150
       
+
       
    ceph.wait_for_unit("default.target")

     

       
151
       
 
       


     

       
152
       
+
       
    # Bootstrap ceph-mon daemon

     

       
153
       
+
       
    ceph.succeed(

     

       
154
       
+
       
        "mkdir -p /var/lib/ceph/bootstrap-osd",

     

       
155
       
+
       
        "ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'",

     

       
156
       
+
       
        "ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'",

     

       
157
       
+
       
        "ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r'",

     

       
158
       
+
       
        "ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring",

     

       
159
       
+
       
        "ceph-authtool /tmp/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring",

     

       
160
       
+
       
        "monmaptool --create --fsid ${cluster} --addv 0 'v2:[${ip}]:3300/0' --clobber /tmp/ceph.initial-monmap",

     

       
161
       
+
       
        "mkdir -p /var/lib/ceph/mon/ceph-0",

     

       
162
       
+
       
        "ceph-mon --mkfs -i 0 --monmap /tmp/ceph.initial-monmap --keyring /tmp/ceph.mon.keyring",

     

       
163
       
+
       
        "chown ceph:ceph -R /tmp/ceph.mon.keyring /var/lib/ceph",

     

       
164
       
+
       
        "systemctl start ceph-mon-0.service",

     

       
165
       
+
       
    )

     

       
166
       
 
       


     

       
167
       
+
       
    ceph.wait_for_unit("ceph-mon-0.service")

     

       
168
       
+
       
    # should the mon not start or bind for some reason this gives us a better error message than the config commands running into a timeout

     

       
169
       
+
       
    ceph.wait_for_open_port(3300, "${ip}")

     

       
170
       
+
       
    ceph.succeed(

     

       
171
       
+
       
        # required for HEALTH_OK

     

       
172
       
+
       
        "ceph config set mon auth_allow_insecure_global_id_reclaim false",

     

       
173
       
+
       
        # IPv6

     

       
174
       
+
       
        "ceph config set global ms_bind_ipv4 false",

     

       
175
       
+
       
        "ceph config set global ms_bind_ipv6 true",

     

       
176
       
+
       
        # the new (secure) protocol

     

       
177
       
+
       
        "ceph config set global ms_bind_msgr1 false",

     

       
178
       
+
       
        "ceph config set global ms_bind_msgr2 true",

     

       
179
       
+
       
        # just a small little thing

     

       
180
       
+
       
        "ceph config set mon mon_compact_on_start true",

     

       
181
       
+
       
    )

     

       
182
       
 
       


     

       
183
       
+
       
    # Can't check ceph status until a mon is up

     

       
184
       
+
       
    ceph.succeed("ceph -s | grep 'mon: 1 daemons'")

     

       
185
       
 
       


     

       
186
       
+
       
    # Bootstrap OSDs (do this before starting the mgr because cryptsetup and the mgr both eat a lot of memory)

     

       
187
       
+
       
    ceph.succeed(

     

       
188
       
+
       
        # this will automatically do what's required for LVM, cryptsetup, and stores all the data in Ceph's internal databases

     

       
189
       
+
       
        "ceph-volume lvm prepare --bluestore --data /dev/vdb --dmcrypt --no-systemd --osd-id 0 --osd-fsid ${osd-fsid-map."0"}",

     

       
190
       
+
       
        "ceph-volume lvm prepare --bluestore --data /dev/vdc --dmcrypt --no-systemd --osd-id 1 --osd-fsid ${osd-fsid-map."1"}",

     

       
191
       
+
       
        "ceph-volume lvm prepare --bluestore --data /dev/vdd --dmcrypt --no-systemd --osd-id 2 --osd-fsid ${osd-fsid-map."2"}",

     

       
192
       
+
       
        "sudo ceph-volume lvm deactivate 0",

     

       
193
       
+
       
        "sudo ceph-volume lvm deactivate 1",

     

       
194
       
+
       
        "sudo ceph-volume lvm deactivate 2",

     

       
195
       
+
       
        "chown -R ceph:ceph /var/lib/ceph",

     

       
196
       
+
       
    )

     

       
197
       
 
       


     

       
198
       
+
       
    # Start OSDs (again, argon2id eats memory, so this happens before starting the mgr)

     

       
199
       
+
       
    ceph.succeed(

     

       
200
       
+
       
        "systemctl start ceph-osd-0.service",

     

       
201
       
+
       
        "systemctl start ceph-osd-1.service",

     

       
202
       
+
       
        "systemctl start ceph-osd-2.service",

     

       
203
       
+
       
    )

     

       
204
       
+
       
    ceph.wait_until_succeeds("ceph -s | grep 'quorum 0'")

     

       
205
       
+
       
    ceph.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")

     

       
206
       
 
       


     

       
207
       
+
       
    # Start the ceph-mgr daemon, after copying in the keyring

     

       
208
       
+
       
    ceph.succeed(

     

       
209
       
+
       
        "mkdir -p /var/lib/ceph/mgr/ceph-ceph/",

     

       
210
       
+
       
        "ceph auth get-or-create -o /var/lib/ceph/mgr/ceph-ceph/keyring mgr.ceph mon 'allow profile mgr' osd 'allow *' mds 'allow *'",

     

       
211
       
+
       
        "chown -R ceph:ceph /var/lib/ceph/mgr/ceph-ceph/",

     

       
212
       
+
       
        "systemctl start ceph-mgr-ceph.service",

     

       
213
       
+
       
    )

     

       
214
       
+
       
    ceph.wait_for_unit("ceph-mgr-ceph")

     

       
215
       
+
       
    ceph.wait_until_succeeds("ceph -s | grep 'quorum 0'")

     

       
216
       
+
       
    ceph.wait_until_succeeds("ceph -s | grep 'mgr: ceph(active,'")

     

       
217
       
+
       
    ceph.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")

     

       
218
       
+
       
    ceph.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")

     

       
219
       
 
       


     

       
220
       
+
       
    # test the actual storage

     

       
221
       
+
       
    ceph.succeed(

     

       
222
       
+
       
        "ceph osd pool create single-node-test 32 32",

     

       
223
       
+
       
        "ceph osd pool ls | grep 'single-node-test'",

     

       
224
       
 
       


     

       
225
       
+
       
        # We need to enable an application on the pool, otherwise it will

     

       
226
       
+
       
        # stay unhealthy in state POOL_APP_NOT_ENABLED.

     

       
227
       
+
       
        # Creating a CephFS would do this automatically, but we haven't done that here.

     

       
228
       
+
       
        # See: https://docs.ceph.com/en/reef/rados/operations/pools/#associating-a-pool-with-an-application

     

       
229
       
+
       
        # We use the custom application name "nixos-test" for this.

     

       
230
       
+
       
        "ceph osd pool application enable single-node-test nixos-test",

     

       
231
       
 
       


     

       
232
       
+
       
        "ceph osd pool rename single-node-test single-node-other-test",

     

       
233
       
+
       
        "ceph osd pool ls | grep 'single-node-other-test'",

     

       
234
       
+
       
    )

     

       
235
       
+
       
    ceph.wait_until_succeeds("ceph -s | grep '2 pools, 33 pgs'")

     

       
236
       
+
       
    ceph.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")

     

       
237
       
+
       
    ceph.wait_until_succeeds("ceph -s | grep '33 active+clean'")

     

       
238
       
+
       
    ceph.fail(

     

       
239
       
+
       
        # the old pool should be gone

     

       
240
       
+
       
        "ceph osd pool ls | grep 'multi-node-test'",

     

       
241
       
+
       
        # deleting the pool should fail without setting mon_allow_pool_delete

     

       
242
       
+
       
        "ceph osd pool delete single-node-other-test single-node-other-test --yes-i-really-really-mean-it",

     

       
243
       
+
       
    )

     

       
244
       
 
       


     

       
245
       
+
       
    # rebooting gets rid of any potential tmpfs mounts or device-mapper devices

     

       
246
       
+
       
    ceph.shutdown()

     

       
247
       
+
       
    ceph.start()

     

       
248
       
+
       
    ceph.wait_for_unit("default.target")

     

       
249
       
 
       


     

       
250
       
+
       
    # Start it up (again OSDs first due to memory constraints of cryptsetup and mgr)

     

       
251
       
+
       
    ceph.systemctl("start ceph-mon-0.service")

     

       
252
       
+
       
    ceph.wait_for_unit("ceph-mon-0")

     

       
253
       
+
       
    ceph.systemctl("start ceph-osd-0.service")

     

       
254
       
+
       
    ceph.wait_for_unit("ceph-osd-0")

     

       
255
       
+
       
    ceph.systemctl("start ceph-osd-1.service")

     

       
256
       
+
       
    ceph.wait_for_unit("ceph-osd-1")

     

       
257
       
+
       
    ceph.systemctl("start ceph-osd-2.service")

     

       
258
       
+
       
    ceph.wait_for_unit("ceph-osd-2")

     

       
259
       
+
       
    ceph.systemctl("start ceph-mgr-ceph.service")

     

       
260
       
+
       
    ceph.wait_for_unit("ceph-mgr-ceph")

     

       
261
       
 
       


     

       
262
       
+
       
    # Ensure the cluster comes back up again

     

       
263
       
+
       
    ceph.succeed("ceph -s | grep 'mon: 1 daemons'")

     

       
264
       
+
       
    ceph.wait_until_succeeds("ceph -s | grep 'quorum 0'")

     

       
265
       
+
       
    ceph.wait_until_succeeds("ceph osd stat | grep -E '3 osds: 3 up[^,]*, 3 in'")

     

       
266
       
+
       
    ceph.wait_until_succeeds("ceph -s | grep 'mgr: ceph(active,'")

     

       
267
       
+
       
    ceph.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")

     

       
268
       
+
       
  '';

     

       
269
       
+
       
}

     

       
0