pyrox.dev
1import ./make-test-python.nix (
2 { pkgs, lib, ... }:
3 let
4 initiatorName = "iqn.2020-08.org.linux-iscsi.initiatorhost:example";
5 targetName = "iqn.2003-01.org.linux-iscsi.target.x8664:sn.acf8fd9c23af";
6 in
7 {
8 name = "iscsi";
9 meta = {
10 maintainers = pkgs.lib.teams.deshaw.members;
11 };
12
13 nodes = {
14 target =
15 {
16 config,
17 pkgs,
18 lib,
19 ...
20 }:
21 {
22 virtualisation.vlans = [
23 1
24 2
25 ];
26 services.target = {
27 enable = true;
28 config = {
29 fabric_modules = [ ];
30 storage_objects = [
31 {
32 dev = "/dev/vdb";
33 name = "test";
34 plugin = "block";
35 write_back = true;
36 wwn = "92b17c3f-6b40-4168-b082-ceeb7b495522";
37 }
38 ];
39 targets = [
40 {
41 fabric = "iscsi";
42 tpgs = [
43 {
44 enable = true;
45 attributes = {
46 authentication = 0;
47 generate_node_acls = 1;
48 };
49 luns = [
50 {
51 alias = "94dfe06967";
52 alua_tg_pt_gp_name = "default_tg_pt_gp";
53 index = 0;
54 storage_object = "/backstores/block/test";
55 }
56 ];
57 node_acls = [
58 {
59 mapped_luns = [
60 {
61 alias = "d42f5bdf8a";
62 index = 0;
63 tpg_lun = 0;
64 write_protect = false;
65 }
66 ];
67 node_wwn = initiatorName;
68 }
69 ];
70 portals = [
71 {
72 ip_address = "0.0.0.0";
73 iser = false;
74 offload = false;
75 port = 3260;
76 }
77 ];
78 tag = 1;
79 }
80 ];
81 wwn = targetName;
82 }
83 ];
84 };
85 };
86
87 networking.firewall.allowedTCPPorts = [ 3260 ];
88 networking.firewall.allowedUDPPorts = [ 3260 ];
89
90 virtualisation.memorySize = 2048;
91 virtualisation.emptyDiskImages = [ 2048 ];
92 };
93
94 initiatorAuto =
95 {
96 nodes,
97 config,
98 pkgs,
99 ...
100 }:
101 {
102 virtualisation.vlans = [
103 1
104 2
105 ];
106
107 services.multipath = {
108 enable = true;
109 defaults = ''
110 find_multipaths yes
111 user_friendly_names yes
112 '';
113 pathGroups = [
114 {
115 alias = 123456;
116 wwid = "3600140592b17c3f6b404168b082ceeb7";
117 }
118 ];
119 };
120
121 services.openiscsi = {
122 enable = true;
123 enableAutoLoginOut = true;
124 discoverPortal = "target";
125 name = initiatorName;
126 };
127
128 environment.systemPackages = with pkgs; [
129 xfsprogs
130 ];
131
132 environment.etc."initiator-root-disk-closure".source =
133 nodes.initiatorRootDisk.config.system.build.toplevel;
134
135 nix.settings = {
136 substituters = lib.mkForce [ ];
137 hashed-mirrors = null;
138 connect-timeout = 1;
139 };
140 };
141
142 initiatorRootDisk =
143 {
144 config,
145 pkgs,
146 modulesPath,
147 lib,
148 ...
149 }:
150 {
151 boot.initrd.network.enable = true;
152 boot.loader.grub.enable = false;
153
154 boot.kernelParams = lib.mkOverride 5 ([
155 "boot.shell_on_fail"
156 "console=tty1"
157 "ip=192.168.1.1:::255.255.255.0::ens9:none"
158 "ip=192.168.2.1:::255.255.255.0::ens10:none"
159 ]);
160
161 # defaults to true, puts some code in the initrd that tries to mount an overlayfs on /nix/store
162 virtualisation.writableStore = false;
163 virtualisation.vlans = [
164 1
165 2
166 ];
167
168 services.multipath = {
169 enable = true;
170 defaults = ''
171 find_multipaths yes
172 user_friendly_names yes
173 '';
174 pathGroups = [
175 {
176 alias = 123456;
177 wwid = "3600140592b17c3f6b404168b082ceeb7";
178 }
179 ];
180 };
181
182 fileSystems = lib.mkOverride 5 {
183 "/" = {
184 fsType = "xfs";
185 device = "/dev/mapper/123456";
186 options = [ "_netdev" ];
187 };
188 };
189
190 boot.initrd.extraFiles."etc/multipath/wwids".source =
191 pkgs.writeText "wwids" "/3600140592b17c3f6b404168b082ceeb7/";
192
193 boot.iscsi-initiator = {
194 discoverPortal = "target";
195 name = initiatorName;
196 target = targetName;
197 extraIscsiCommands = ''
198 iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login
199 '';
200 };
201 };
202
203 };
204
205 testScript =
206 { nodes, ... }:
207 ''
208 target.start()
209 target.wait_for_unit("iscsi-target.service")
210
211 initiatorAuto.start()
212
213 initiatorAuto.wait_for_unit("iscsid.service")
214 initiatorAuto.wait_for_unit("iscsi.service")
215 initiatorAuto.get_unit_info("iscsi")
216
217 # Expecting this to fail since we should already know about 192.168.1.3
218 initiatorAuto.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.1.3 --login")
219 # Expecting this to succeed since we don't yet know about 192.168.2.3
220 initiatorAuto.succeed("iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login")
221
222 # /dev/sda is provided by iscsi on target
223 initiatorAuto.succeed("set -x; while ! test -e /dev/sda; do sleep 1; done")
224
225 initiatorAuto.succeed("mkfs.xfs /dev/sda")
226 initiatorAuto.succeed("mkdir /mnt")
227
228 # Start by verifying /dev/sda and /dev/sdb are both the same disk
229 initiatorAuto.succeed("mount /dev/sda /mnt")
230 initiatorAuto.succeed("touch /mnt/hi")
231 initiatorAuto.succeed("umount /mnt")
232
233 initiatorAuto.succeed("mount /dev/sdb /mnt")
234 initiatorAuto.succeed("test -e /mnt/hi")
235 initiatorAuto.succeed("umount /mnt")
236
237 initiatorAuto.succeed("systemctl restart multipathd")
238 initiatorAuto.succeed("systemd-cat multipath -ll")
239
240 # Install our RootDisk machine to 123456, the alias to the device that multipath is now managing
241 initiatorAuto.succeed("mount /dev/mapper/123456 /mnt")
242 initiatorAuto.succeed("mkdir -p /mnt/etc/{multipath,iscsi}")
243 initiatorAuto.succeed("cp -r /etc/multipath/wwids /mnt/etc/multipath/wwids")
244 initiatorAuto.succeed("cp -r /etc/iscsi/{nodes,send_targets} /mnt/etc/iscsi")
245 initiatorAuto.succeed(
246 "nixos-install --no-bootloader --no-root-passwd --system /etc/initiator-root-disk-closure"
247 )
248 initiatorAuto.succeed("umount /mnt")
249 initiatorAuto.shutdown()
250
251 initiatorRootDisk.start()
252 initiatorRootDisk.wait_for_unit("multi-user.target")
253 initiatorRootDisk.wait_for_unit("iscsid")
254
255 # Log in over both nodes
256 initiatorRootDisk.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.1.3 --login")
257 initiatorRootDisk.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login")
258 initiatorRootDisk.succeed("systemctl restart multipathd")
259 initiatorRootDisk.succeed("systemd-cat multipath -ll")
260
261 # Verify we can write and sync the root disk
262 initiatorRootDisk.succeed("mkdir /scratch")
263 initiatorRootDisk.succeed("touch /scratch/both-up")
264 initiatorRootDisk.succeed("sync /scratch")
265
266 # Verify we can write to the root with ens9 (sda, 192.168.1.3) down
267 initiatorRootDisk.succeed("ip link set ens9 down")
268 initiatorRootDisk.succeed("touch /scratch/ens9-down")
269 initiatorRootDisk.succeed("sync /scratch")
270 initiatorRootDisk.succeed("ip link set ens9 up")
271
272 # todo: better way to wait until multipath notices the link is back
273 initiatorRootDisk.succeed("sleep 5")
274 initiatorRootDisk.succeed("touch /scratch/both-down")
275 initiatorRootDisk.succeed("sync /scratch")
276
277 # Verify we can write to the root with ens10 (sdb, 192.168.2.3) down
278 initiatorRootDisk.succeed("ip link set ens10 down")
279 initiatorRootDisk.succeed("touch /scratch/ens10-down")
280 initiatorRootDisk.succeed("sync /scratch")
281 initiatorRootDisk.succeed("ip link set ens10 up")
282 initiatorRootDisk.succeed("touch /scratch/ens10-down")
283 initiatorRootDisk.succeed("sync /scratch")
284
285 initiatorRootDisk.succeed("ip link set ens9 up")
286 initiatorRootDisk.succeed("ip link set ens10 up")
287 initiatorRootDisk.shutdown()
288
289 # Verify we can boot with the target's eth1 down, forcing
290 # it to multipath via the second link
291 target.succeed("ip link set eth1 down")
292 initiatorRootDisk.start()
293 initiatorRootDisk.wait_for_unit("multi-user.target")
294 initiatorRootDisk.wait_for_unit("iscsid")
295 initiatorRootDisk.succeed("test -e /scratch/both-up")
296 '';
297 }
298)