pyrox.dev
1import ./make-test-python.nix (
2 { pkgs, lib, ... }:
3 let
4 initiatorName = "iqn.2020-08.org.linux-iscsi.initiatorhost:example";
5 targetName = "iqn.2003-01.org.linux-iscsi.target.x8664:sn.acf8fd9c23af";
6 in
7 {
8 name = "iscsi";
9 meta = {
10 maintainers = pkgs.lib.teams.deshaw.members;
11 };
12
13 nodes = {
14 target = { config, pkgs, lib, ... }: {
15 virtualisation.vlans = [ 1 2 ];
16 services.target = {
17 enable = true;
18 config = {
19 fabric_modules = [ ];
20 storage_objects = [
21 {
22 dev = "/dev/vdb";
23 name = "test";
24 plugin = "block";
25 write_back = true;
26 wwn = "92b17c3f-6b40-4168-b082-ceeb7b495522";
27 }
28 ];
29 targets = [
30 {
31 fabric = "iscsi";
32 tpgs = [
33 {
34 enable = true;
35 attributes = {
36 authentication = 0;
37 generate_node_acls = 1;
38 };
39 luns = [
40 {
41 alias = "94dfe06967";
42 alua_tg_pt_gp_name = "default_tg_pt_gp";
43 index = 0;
44 storage_object = "/backstores/block/test";
45 }
46 ];
47 node_acls = [
48 {
49 mapped_luns = [
50 {
51 alias = "d42f5bdf8a";
52 index = 0;
53 tpg_lun = 0;
54 write_protect = false;
55 }
56 ];
57 node_wwn = initiatorName;
58 }
59 ];
60 portals = [
61 {
62 ip_address = "0.0.0.0";
63 iser = false;
64 offload = false;
65 port = 3260;
66 }
67 ];
68 tag = 1;
69 }
70 ];
71 wwn = targetName;
72 }
73 ];
74 };
75 };
76
77 networking.firewall.allowedTCPPorts = [ 3260 ];
78 networking.firewall.allowedUDPPorts = [ 3260 ];
79
80 virtualisation.memorySize = 2048;
81 virtualisation.emptyDiskImages = [ 2048 ];
82 };
83
84 initiatorAuto = { nodes, config, pkgs, ... }: {
85 virtualisation.vlans = [ 1 2 ];
86
87 services.multipath = {
88 enable = true;
89 defaults = ''
90 find_multipaths yes
91 user_friendly_names yes
92 '';
93 pathGroups = [
94 {
95 alias = 123456;
96 wwid = "3600140592b17c3f6b404168b082ceeb7";
97 }
98 ];
99 };
100
101 services.openiscsi = {
102 enable = true;
103 enableAutoLoginOut = true;
104 discoverPortal = "target";
105 name = initiatorName;
106 };
107
108 environment.systemPackages = with pkgs; [
109 xfsprogs
110 ];
111
112 environment.etc."initiator-root-disk-closure".source = nodes.initiatorRootDisk.config.system.build.toplevel;
113
114 nix.settings = {
115 substituters = lib.mkForce [ ];
116 hashed-mirrors = null;
117 connect-timeout = 1;
118 };
119 };
120
121 initiatorRootDisk = { config, pkgs, modulesPath, lib, ... }: {
122 boot.initrd.network.enable = true;
123 boot.loader.grub.enable = false;
124
125 boot.kernelParams = lib.mkOverride 5 (
126 [
127 "boot.shell_on_fail"
128 "console=tty1"
129 "ip=192.168.1.1:::255.255.255.0::ens9:none"
130 "ip=192.168.2.1:::255.255.255.0::ens10:none"
131 ]
132 );
133
134 # defaults to true, puts some code in the initrd that tries to mount an overlayfs on /nix/store
135 virtualisation.writableStore = false;
136 virtualisation.vlans = [ 1 2 ];
137
138 services.multipath = {
139 enable = true;
140 defaults = ''
141 find_multipaths yes
142 user_friendly_names yes
143 '';
144 pathGroups = [
145 {
146 alias = 123456;
147 wwid = "3600140592b17c3f6b404168b082ceeb7";
148 }
149 ];
150 };
151
152 fileSystems = lib.mkOverride 5 {
153 "/" = {
154 fsType = "xfs";
155 device = "/dev/mapper/123456";
156 options = [ "_netdev" ];
157 };
158 };
159
160 boot.initrd.extraFiles."etc/multipath/wwids".source = pkgs.writeText "wwids" "/3600140592b17c3f6b404168b082ceeb7/";
161
162 boot.iscsi-initiator = {
163 discoverPortal = "target";
164 name = initiatorName;
165 target = targetName;
166 extraIscsiCommands = ''
167 iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login
168 '';
169 };
170 };
171
172 };
173
174 testScript = { nodes, ... }: ''
175 target.start()
176 target.wait_for_unit("iscsi-target.service")
177
178 initiatorAuto.start()
179
180 initiatorAuto.wait_for_unit("iscsid.service")
181 initiatorAuto.wait_for_unit("iscsi.service")
182 initiatorAuto.get_unit_info("iscsi")
183
184 # Expecting this to fail since we should already know about 192.168.1.3
185 initiatorAuto.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.1.3 --login")
186 # Expecting this to succeed since we don't yet know about 192.168.2.3
187 initiatorAuto.succeed("iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login")
188
189 # /dev/sda is provided by iscsi on target
190 initiatorAuto.succeed("set -x; while ! test -e /dev/sda; do sleep 1; done")
191
192 initiatorAuto.succeed("mkfs.xfs /dev/sda")
193 initiatorAuto.succeed("mkdir /mnt")
194
195 # Start by verifying /dev/sda and /dev/sdb are both the same disk
196 initiatorAuto.succeed("mount /dev/sda /mnt")
197 initiatorAuto.succeed("touch /mnt/hi")
198 initiatorAuto.succeed("umount /mnt")
199
200 initiatorAuto.succeed("mount /dev/sdb /mnt")
201 initiatorAuto.succeed("test -e /mnt/hi")
202 initiatorAuto.succeed("umount /mnt")
203
204 initiatorAuto.succeed("systemctl restart multipathd")
205 initiatorAuto.succeed("multipath -ll | systemd-cat")
206
207 # Install our RootDisk machine to 123456, the alias to the device that multipath is now managing
208 initiatorAuto.succeed("mount /dev/mapper/123456 /mnt")
209 initiatorAuto.succeed("mkdir -p /mnt/etc/{multipath,iscsi}")
210 initiatorAuto.succeed("cp -r /etc/multipath/wwids /mnt/etc/multipath/wwids")
211 initiatorAuto.succeed("cp -r /etc/iscsi/{nodes,send_targets} /mnt/etc/iscsi")
212 initiatorAuto.succeed(
213 "nixos-install --no-bootloader --no-root-passwd --system /etc/initiator-root-disk-closure"
214 )
215 initiatorAuto.succeed("umount /mnt")
216 initiatorAuto.shutdown()
217
218 initiatorRootDisk.start()
219 initiatorRootDisk.wait_for_unit("multi-user.target")
220 initiatorRootDisk.wait_for_unit("iscsid")
221
222 # Log in over both nodes
223 initiatorRootDisk.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.1.3 --login")
224 initiatorRootDisk.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login")
225 initiatorRootDisk.succeed("systemctl restart multipathd")
226 initiatorRootDisk.succeed("multipath -ll | systemd-cat")
227
228 # Verify we can write and sync the root disk
229 initiatorRootDisk.succeed("mkdir /scratch")
230 initiatorRootDisk.succeed("touch /scratch/both-up")
231 initiatorRootDisk.succeed("sync /scratch")
232
233 # Verify we can write to the root with ens9 (sda, 192.168.1.3) down
234 initiatorRootDisk.succeed("ip link set ens9 down")
235 initiatorRootDisk.succeed("touch /scratch/ens9-down")
236 initiatorRootDisk.succeed("sync /scratch")
237 initiatorRootDisk.succeed("ip link set ens9 up")
238
239 # todo: better way to wait until multipath notices the link is back
240 initiatorRootDisk.succeed("sleep 5")
241 initiatorRootDisk.succeed("touch /scratch/both-down")
242 initiatorRootDisk.succeed("sync /scratch")
243
244 # Verify we can write to the root with ens10 (sdb, 192.168.2.3) down
245 initiatorRootDisk.succeed("ip link set ens10 down")
246 initiatorRootDisk.succeed("touch /scratch/ens10-down")
247 initiatorRootDisk.succeed("sync /scratch")
248 initiatorRootDisk.succeed("ip link set ens10 up")
249 initiatorRootDisk.succeed("touch /scratch/ens10-down")
250 initiatorRootDisk.succeed("sync /scratch")
251
252 initiatorRootDisk.succeed("ip link set ens9 up")
253 initiatorRootDisk.succeed("ip link set ens10 up")
254 initiatorRootDisk.shutdown()
255
256 # Verify we can boot with the target's eth1 down, forcing
257 # it to multipath via the second link
258 target.succeed("ip link set eth1 down")
259 initiatorRootDisk.start()
260 initiatorRootDisk.wait_for_unit("multi-user.target")
261 initiatorRootDisk.wait_for_unit("iscsid")
262 initiatorRootDisk.succeed("test -e /scratch/both-up")
263 '';
264 }
265)
266
267