pyrox.dev
1import ./make-test-python.nix (
2 { pkgs, lib, ... }:
3
4 let
5 # the single node ipv6 address
6 ip = "2001:db8:ffff::";
7 # the global ceph cluster id
8 cluster = "54465b37-b9d8-4539-a1f9-dd33c75ee45a";
9 # the fsids of OSDs
10 osd-fsid-map = {
11 "0" = "1c1b7ea9-06bf-4d30-9a01-37ac3a0254aa";
12 "1" = "bd5a6f49-69d5-428c-ac25-a99f0c44375c";
13 "2" = "c90de6c7-86c6-41da-9694-e794096dfc5c";
14 };
15
16 in
17 {
18 name = "basic-single-node-ceph-cluster-bluestore-dmcrypt";
19 meta = {
20 maintainers = with lib.maintainers; [
21 benaryorg
22 nh2
23 ];
24 };
25
26 nodes = {
27 ceph =
28 { pkgs, config, ... }:
29 {
30 # disks for bluestore
31 virtualisation.emptyDiskImages = [
32 20480
33 20480
34 20480
35 ];
36
37 # networking setup (no external connectivity required, only local IPv6)
38 networking.useDHCP = false;
39 systemd.network = {
40 enable = true;
41 wait-online.extraArgs = [
42 "-i"
43 "lo"
44 ];
45 networks = {
46 "40-loopback" = {
47 enable = true;
48 name = "lo";
49 DHCP = "no";
50 addresses = [ { Address = "${ip}/128"; } ];
51 };
52 };
53 };
54
55 # do not start the ceph target by default so we can format the disks first
56 systemd.targets.ceph.wantedBy = lib.mkForce [ ];
57
58 # add the packages to systemPackages so the testscript doesn't run into any unexpected issues
59 # this shouldn't be required on production systems which have their required packages in the unit paths only
60 # but it helps in case one needs to actually run the tooling anyway
61 environment.systemPackages = with pkgs; [
62 ceph
63 cryptsetup
64 lvm2
65 ];
66
67 services.ceph = {
68 enable = true;
69 client.enable = true;
70 extraConfig = {
71 public_addr = ip;
72 cluster_addr = ip;
73 # ipv6
74 ms_bind_ipv4 = "false";
75 ms_bind_ipv6 = "true";
76 # msgr2 settings
77 ms_cluster_mode = "secure";
78 ms_service_mode = "secure";
79 ms_client_mode = "secure";
80 ms_mon_cluster_mode = "secure";
81 ms_mon_service_mode = "secure";
82 ms_mon_client_mode = "secure";
83 # less default modules, cuts down on memory and startup time in the tests
84 mgr_initial_modules = "";
85 # distribute by OSD, not by host, as per https://docs.ceph.com/en/reef/cephadm/install/#single-host
86 osd_crush_chooseleaf_type = "0";
87 };
88 client.extraConfig."mon.0" = {
89 host = "ceph";
90 mon_addr = "v2:[${ip}]:3300";
91 public_addr = "v2:[${ip}]:3300";
92 };
93 global = {
94 fsid = cluster;
95 clusterNetwork = "${ip}/64";
96 publicNetwork = "${ip}/64";
97 monInitialMembers = "0";
98 };
99
100 mon = {
101 enable = true;
102 daemons = [ "0" ];
103 };
104
105 osd = {
106 enable = true;
107 daemons = builtins.attrNames osd-fsid-map;
108 };
109
110 mgr = {
111 enable = true;
112 daemons = [ "ceph" ];
113 };
114 };
115
116 systemd.services =
117 let
118 osd-name = id: "ceph-osd-${id}";
119 osd-pre-start = id: [
120 "!${config.services.ceph.osd.package.out}/bin/ceph-volume lvm activate --bluestore ${id} ${osd-fsid-map.${id}} --no-systemd"
121 "${config.services.ceph.osd.package.lib}/libexec/ceph/ceph-osd-prestart.sh --id ${id} --cluster ${config.services.ceph.global.clusterName}"
122 ];
123 osd-post-stop = id: [
124 "!${config.services.ceph.osd.package.out}/bin/ceph-volume lvm deactivate ${id}"
125 ];
126 map-osd = id: {
127 name = osd-name id;
128 value = {
129 serviceConfig.ExecStartPre = lib.mkForce (osd-pre-start id);
130 serviceConfig.ExecStopPost = osd-post-stop id;
131 unitConfig.ConditionPathExists = lib.mkForce [ ];
132 unitConfig.StartLimitBurst = lib.mkForce 4;
133 path = with pkgs; [
134 util-linux
135 lvm2
136 cryptsetup
137 ];
138 };
139 };
140 in
141 lib.pipe config.services.ceph.osd.daemons [
142 (builtins.map map-osd)
143 builtins.listToAttrs
144 ];
145 };
146 };
147
148 testScript =
149 { ... }:
150 ''
151 start_all()
152
153 ceph.wait_for_unit("default.target")
154
155 # Bootstrap ceph-mon daemon
156 ceph.succeed(
157 "mkdir -p /var/lib/ceph/bootstrap-osd",
158 "ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'",
159 "ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'",
160 "ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r'",
161 "ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring",
162 "ceph-authtool /tmp/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring",
163 "monmaptool --create --fsid ${cluster} --addv 0 'v2:[${ip}]:3300/0' --clobber /tmp/ceph.initial-monmap",
164 "mkdir -p /var/lib/ceph/mon/ceph-0",
165 "ceph-mon --mkfs -i 0 --monmap /tmp/ceph.initial-monmap --keyring /tmp/ceph.mon.keyring",
166 "chown ceph:ceph -R /tmp/ceph.mon.keyring /var/lib/ceph",
167 "systemctl start ceph-mon-0.service",
168 )
169
170 ceph.wait_for_unit("ceph-mon-0.service")
171 # should the mon not start or bind for some reason this gives us a better error message than the config commands running into a timeout
172 ceph.wait_for_open_port(3300, "${ip}")
173 ceph.succeed(
174 # required for HEALTH_OK
175 "ceph config set mon auth_allow_insecure_global_id_reclaim false",
176 # IPv6
177 "ceph config set global ms_bind_ipv4 false",
178 "ceph config set global ms_bind_ipv6 true",
179 # the new (secure) protocol
180 "ceph config set global ms_bind_msgr1 false",
181 "ceph config set global ms_bind_msgr2 true",
182 # just a small little thing
183 "ceph config set mon mon_compact_on_start true",
184 )
185
186 # Can't check ceph status until a mon is up
187 ceph.succeed("ceph -s | grep 'mon: 1 daemons'")
188
189 # Bootstrap OSDs (do this before starting the mgr because cryptsetup and the mgr both eat a lot of memory)
190 ceph.succeed(
191 # this will automatically do what's required for LVM, cryptsetup, and stores all the data in Ceph's internal databases
192 "ceph-volume lvm prepare --bluestore --data /dev/vdb --dmcrypt --no-systemd --osd-id 0 --osd-fsid ${osd-fsid-map."0"}",
193 "ceph-volume lvm prepare --bluestore --data /dev/vdc --dmcrypt --no-systemd --osd-id 1 --osd-fsid ${osd-fsid-map."1"}",
194 "ceph-volume lvm prepare --bluestore --data /dev/vdd --dmcrypt --no-systemd --osd-id 2 --osd-fsid ${osd-fsid-map."2"}",
195 "sudo ceph-volume lvm deactivate 0",
196 "sudo ceph-volume lvm deactivate 1",
197 "sudo ceph-volume lvm deactivate 2",
198 "chown -R ceph:ceph /var/lib/ceph",
199 )
200
201 # Start OSDs (again, argon2id eats memory, so this happens before starting the mgr)
202 ceph.succeed(
203 "systemctl start ceph-osd-0.service",
204 "systemctl start ceph-osd-1.service",
205 "systemctl start ceph-osd-2.service",
206 )
207 ceph.wait_until_succeeds("ceph -s | grep 'quorum 0'")
208 ceph.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")
209
210 # Start the ceph-mgr daemon, after copying in the keyring
211 ceph.succeed(
212 "mkdir -p /var/lib/ceph/mgr/ceph-ceph/",
213 "ceph auth get-or-create -o /var/lib/ceph/mgr/ceph-ceph/keyring mgr.ceph mon 'allow profile mgr' osd 'allow *' mds 'allow *'",
214 "chown -R ceph:ceph /var/lib/ceph/mgr/ceph-ceph/",
215 "systemctl start ceph-mgr-ceph.service",
216 )
217 ceph.wait_for_unit("ceph-mgr-ceph")
218 ceph.wait_until_succeeds("ceph -s | grep 'quorum 0'")
219 ceph.wait_until_succeeds("ceph -s | grep 'mgr: ceph(active,'")
220 ceph.wait_until_succeeds("ceph osd stat | grep -e '3 osds: 3 up[^,]*, 3 in'")
221 ceph.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
222
223 # test the actual storage
224 ceph.succeed(
225 "ceph osd pool create single-node-test 32 32",
226 "ceph osd pool ls | grep 'single-node-test'",
227
228 # We need to enable an application on the pool, otherwise it will
229 # stay unhealthy in state POOL_APP_NOT_ENABLED.
230 # Creating a CephFS would do this automatically, but we haven't done that here.
231 # See: https://docs.ceph.com/en/reef/rados/operations/pools/#associating-a-pool-with-an-application
232 # We use the custom application name "nixos-test" for this.
233 "ceph osd pool application enable single-node-test nixos-test",
234
235 "ceph osd pool rename single-node-test single-node-other-test",
236 "ceph osd pool ls | grep 'single-node-other-test'",
237 )
238 ceph.wait_until_succeeds("ceph -s | grep '2 pools, 33 pgs'")
239 ceph.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
240 ceph.wait_until_succeeds("ceph -s | grep '33 active+clean'")
241 ceph.fail(
242 # the old pool should be gone
243 "ceph osd pool ls | grep 'multi-node-test'",
244 # deleting the pool should fail without setting mon_allow_pool_delete
245 "ceph osd pool delete single-node-other-test single-node-other-test --yes-i-really-really-mean-it",
246 )
247
248 # rebooting gets rid of any potential tmpfs mounts or device-mapper devices
249 ceph.shutdown()
250 ceph.start()
251 ceph.wait_for_unit("default.target")
252
253 # Start it up (again OSDs first due to memory constraints of cryptsetup and mgr)
254 ceph.systemctl("start ceph-mon-0.service")
255 ceph.wait_for_unit("ceph-mon-0")
256 ceph.systemctl("start ceph-osd-0.service")
257 ceph.wait_for_unit("ceph-osd-0")
258 ceph.systemctl("start ceph-osd-1.service")
259 ceph.wait_for_unit("ceph-osd-1")
260 ceph.systemctl("start ceph-osd-2.service")
261 ceph.wait_for_unit("ceph-osd-2")
262 ceph.systemctl("start ceph-mgr-ceph.service")
263 ceph.wait_for_unit("ceph-mgr-ceph")
264
265 # Ensure the cluster comes back up again
266 ceph.succeed("ceph -s | grep 'mon: 1 daemons'")
267 ceph.wait_until_succeeds("ceph -s | grep 'quorum 0'")
268 ceph.wait_until_succeeds("ceph osd stat | grep -E '3 osds: 3 up[^,]*, 3 in'")
269 ceph.wait_until_succeeds("ceph -s | grep 'mgr: ceph(active,'")
270 ceph.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
271 '';
272 }
273)