pyrox.dev
1import ./make-test.nix ({pkgs, ...}: rec {
2 name = "All-in-one-basic-ceph-cluster";
3 meta = with pkgs.stdenv.lib.maintainers; {
4 maintainers = [ lejonet ];
5 };
6
7 nodes = {
8 aio = { config, pkgs, ... }: {
9 virtualisation = {
10 emptyDiskImages = [ 20480 20480 ];
11 vlans = [ 1 ];
12 };
13
14 networking = {
15 firewall.allowPing = true;
16 useDHCP = false;
17 interfaces.eth1.ipv4.addresses = pkgs.lib.mkOverride 0 [
18 { address = "192.168.1.1"; prefixLength = 24; }
19 ];
20 };
21
22 environment.systemPackages = with pkgs; [
23 bash
24 sudo
25 ceph
26 xfsprogs
27 ];
28 nixpkgs.config.packageOverrides = super: {
29 ceph = super.ceph.override({ nss = super.nss; libxfs = super.libxfs; libaio = super.libaio; jemalloc = super.jemalloc; });
30 };
31
32 boot.kernelModules = [ "xfs" ];
33
34 services.ceph.enable = true;
35 services.ceph.global = {
36 fsid = "066ae264-2a5d-4729-8001-6ad265f50b03";
37 monInitialMembers = "aio";
38 monHost = "192.168.1.1";
39 };
40
41 services.ceph.mon = {
42 enable = true;
43 daemons = [ "aio" ];
44 };
45
46 services.ceph.mgr = {
47 enable = true;
48 daemons = [ "aio" ];
49 };
50
51 services.ceph.osd = {
52 enable = true;
53 daemons = [ "0" "1" ];
54 };
55 };
56 };
57
58 testScript = { nodes, ... }: ''
59 startAll;
60
61 $aio->waitForUnit("network.target");
62
63 # Create the ceph-related directories
64 $aio->mustSucceed(
65 "mkdir -p /var/lib/ceph/mgr/ceph-aio/",
66 "mkdir -p /var/lib/ceph/mon/ceph-aio/",
67 "mkdir -p /var/lib/ceph/osd/ceph-{0..1}/",
68 "chown ceph:ceph -R /var/lib/ceph/"
69 );
70
71 # Bootstrap ceph-mon daemon
72 $aio->mustSucceed(
73 "mkdir -p /var/lib/ceph/bootstrap-osd && chown ceph:ceph /var/lib/ceph/bootstrap-osd",
74 "sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'",
75 "ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'",
76 "ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring",
77 "monmaptool --create --add aio 192.168.1.1 --fsid 066ae264-2a5d-4729-8001-6ad265f50b03 /tmp/monmap",
78 "sudo -u ceph ceph-mon --mkfs -i aio --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring",
79 "touch /var/lib/ceph/mon/ceph-aio/done",
80 "systemctl start ceph-mon-aio"
81 );
82 $aio->waitForUnit("ceph-mon-aio");
83
84 # Can't check ceph status until a mon is up
85 $aio->succeed("ceph -s | grep 'mon: 1 daemons'");
86
87 # Start the ceph-mgr daemon, it has no deps and hardly any setup
88 $aio->mustSucceed(
89 "ceph auth get-or-create mgr.aio mon 'allow profile mgr' osd 'allow *' mds 'allow *' > /var/lib/ceph/mgr/ceph-aio/keyring",
90 "systemctl start ceph-mgr-aio"
91 );
92 $aio->waitForUnit("ceph-mgr-aio");
93 $aio->waitUntilSucceeds("ceph -s | grep 'quorum aio'");
94
95 # Bootstrap both OSDs
96 $aio->mustSucceed(
97 "mkfs.xfs /dev/vdb",
98 "mkfs.xfs /dev/vdc",
99 "mount /dev/vdb /var/lib/ceph/osd/ceph-0",
100 "mount /dev/vdc /var/lib/ceph/osd/ceph-1",
101 "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-0/keyring --name osd.0 --add-key AQBCEJNa3s8nHRAANvdsr93KqzBznuIWm2gOGg==",
102 "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-1/keyring --name osd.1 --add-key AQBEEJNac00kExAAXEgy943BGyOpVH1LLlHafQ==",
103 "echo '{\"cephx_secret\": \"AQBCEJNa3s8nHRAANvdsr93KqzBznuIWm2gOGg==\"}' | ceph osd new 55ba2294-3e24-478f-bee0-9dca4c231dd9 -i -",
104 "echo '{\"cephx_secret\": \"AQBEEJNac00kExAAXEgy943BGyOpVH1LLlHafQ==\"}' | ceph osd new 5e97a838-85b6-43b0-8950-cb56d554d1e5 -i -"
105 );
106
107 # Initialize the OSDs with regular filestore
108 $aio->mustSucceed(
109 "ceph-osd -i 0 --mkfs --osd-uuid 55ba2294-3e24-478f-bee0-9dca4c231dd9",
110 "ceph-osd -i 1 --mkfs --osd-uuid 5e97a838-85b6-43b0-8950-cb56d554d1e5",
111 "chown -R ceph:ceph /var/lib/ceph/osd",
112 "systemctl start ceph-osd-0",
113 "systemctl start ceph-osd-1"
114 );
115
116 $aio->waitUntilSucceeds("ceph osd stat | grep '2 osds: 2 up, 2 in'");
117 $aio->waitUntilSucceeds("ceph -s | grep 'mgr: aio(active)'");
118 $aio->waitUntilSucceeds("ceph -s | grep 'HEALTH_OK'");
119
120 $aio->mustSucceed(
121 "ceph osd pool create aio-test 100 100",
122 "ceph osd pool ls | grep 'aio-test'",
123 "ceph osd pool rename aio-test aio-other-test",
124 "ceph osd pool ls | grep 'aio-other-test'",
125 "ceph -s | grep '1 pools, 100 pgs'",
126 "ceph osd getcrushmap -o crush",
127 "crushtool -d crush -o decrushed",
128 "sed 's/step chooseleaf firstn 0 type host/step chooseleaf firstn 0 type osd/' decrushed > modcrush",
129 "crushtool -c modcrush -o recrushed",
130 "ceph osd setcrushmap -i recrushed",
131 "ceph osd pool set aio-other-test size 2"
132 );
133 $aio->waitUntilSucceeds("ceph -s | grep 'HEALTH_OK'");
134 $aio->waitUntilSucceeds("ceph -s | grep '100 active+clean'");
135 $aio->mustFail(
136 "ceph osd pool ls | grep 'aio-test'",
137 "ceph osd pool delete aio-other-test aio-other-test --yes-i-really-really-mean-it"
138 );
139 '';
140})