pyrox.dev
1{ pkgs, lib, ... }:
2let
3 initiatorName = "iqn.2020-08.org.linux-iscsi.initiatorhost:example";
4 targetName = "iqn.2003-01.org.linux-iscsi.target.x8664:sn.acf8fd9c23af";
5in
6{
7 name = "iscsi";
8 meta = {
9 maintainers = pkgs.lib.teams.deshaw.members;
10 };
11
12 nodes = {
13 target =
14 {
15 config,
16 pkgs,
17 lib,
18 ...
19 }:
20 {
21 virtualisation.vlans = [
22 1
23 2
24 ];
25 services.target = {
26 enable = true;
27 config = {
28 fabric_modules = [ ];
29 storage_objects = [
30 {
31 dev = "/dev/vdb";
32 name = "test";
33 plugin = "block";
34 write_back = true;
35 wwn = "92b17c3f-6b40-4168-b082-ceeb7b495522";
36 }
37 ];
38 targets = [
39 {
40 fabric = "iscsi";
41 tpgs = [
42 {
43 enable = true;
44 attributes = {
45 authentication = 0;
46 generate_node_acls = 1;
47 };
48 luns = [
49 {
50 alias = "94dfe06967";
51 alua_tg_pt_gp_name = "default_tg_pt_gp";
52 index = 0;
53 storage_object = "/backstores/block/test";
54 }
55 ];
56 node_acls = [
57 {
58 mapped_luns = [
59 {
60 alias = "d42f5bdf8a";
61 index = 0;
62 tpg_lun = 0;
63 write_protect = false;
64 }
65 ];
66 node_wwn = initiatorName;
67 }
68 ];
69 portals = [
70 {
71 ip_address = "0.0.0.0";
72 iser = false;
73 offload = false;
74 port = 3260;
75 }
76 ];
77 tag = 1;
78 }
79 ];
80 wwn = targetName;
81 }
82 ];
83 };
84 };
85
86 networking.firewall.allowedTCPPorts = [ 3260 ];
87 networking.firewall.allowedUDPPorts = [ 3260 ];
88
89 virtualisation.memorySize = 2048;
90 virtualisation.emptyDiskImages = [ 2048 ];
91 };
92
93 initiatorAuto =
94 {
95 nodes,
96 config,
97 pkgs,
98 ...
99 }:
100 {
101 virtualisation.vlans = [
102 1
103 2
104 ];
105
106 services.multipath = {
107 enable = true;
108 defaults = ''
109 find_multipaths yes
110 user_friendly_names yes
111 '';
112 pathGroups = [
113 {
114 alias = 123456;
115 wwid = "3600140592b17c3f6b404168b082ceeb7";
116 }
117 ];
118 };
119
120 services.openiscsi = {
121 enable = true;
122 enableAutoLoginOut = true;
123 discoverPortal = "target";
124 name = initiatorName;
125 };
126
127 environment.systemPackages = with pkgs; [
128 xfsprogs
129 ];
130
131 environment.etc."initiator-root-disk-closure".source =
132 nodes.initiatorRootDisk.config.system.build.toplevel;
133
134 nix.settings = {
135 substituters = lib.mkForce [ ];
136 hashed-mirrors = null;
137 connect-timeout = 1;
138 };
139 };
140
141 initiatorRootDisk =
142 {
143 config,
144 pkgs,
145 modulesPath,
146 lib,
147 ...
148 }:
149 {
150 boot.initrd.network.enable = true;
151 boot.loader.grub.enable = false;
152
153 boot.kernelParams = lib.mkOverride 5 ([
154 "boot.shell_on_fail"
155 "console=tty1"
156 "ip=192.168.1.1:::255.255.255.0::ens9:none"
157 "ip=192.168.2.1:::255.255.255.0::ens10:none"
158 ]);
159
160 # defaults to true, puts some code in the initrd that tries to mount an overlayfs on /nix/store
161 virtualisation.writableStore = false;
162 virtualisation.vlans = [
163 1
164 2
165 ];
166
167 services.multipath = {
168 enable = true;
169 defaults = ''
170 find_multipaths yes
171 user_friendly_names yes
172 '';
173 pathGroups = [
174 {
175 alias = 123456;
176 wwid = "3600140592b17c3f6b404168b082ceeb7";
177 }
178 ];
179 };
180
181 fileSystems = lib.mkOverride 5 {
182 "/" = {
183 fsType = "xfs";
184 device = "/dev/mapper/123456";
185 options = [ "_netdev" ];
186 };
187 };
188
189 boot.initrd.extraFiles."etc/multipath/wwids".source =
190 pkgs.writeText "wwids" "/3600140592b17c3f6b404168b082ceeb7/";
191
192 boot.iscsi-initiator = {
193 discoverPortal = "target";
194 name = initiatorName;
195 target = targetName;
196 extraIscsiCommands = ''
197 iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login
198 '';
199 };
200 };
201
202 };
203
204 testScript =
205 { nodes, ... }:
206 ''
207 target.start()
208 target.wait_for_unit("iscsi-target.service")
209
210 initiatorAuto.start()
211
212 initiatorAuto.wait_for_unit("iscsid.service")
213 initiatorAuto.wait_for_unit("iscsi.service")
214 initiatorAuto.get_unit_info("iscsi")
215
216 # Expecting this to fail since we should already know about 192.168.1.3
217 initiatorAuto.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.1.3 --login")
218 # Expecting this to succeed since we don't yet know about 192.168.2.3
219 initiatorAuto.succeed("iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login")
220
221 # /dev/sda is provided by iscsi on target
222 initiatorAuto.succeed("set -x; while ! test -e /dev/sda; do sleep 1; done")
223
224 initiatorAuto.succeed("mkfs.xfs /dev/sda")
225 initiatorAuto.succeed("mkdir /mnt")
226
227 # Start by verifying /dev/sda and /dev/sdb are both the same disk
228 initiatorAuto.succeed("mount /dev/sda /mnt")
229 initiatorAuto.succeed("touch /mnt/hi")
230 initiatorAuto.succeed("umount /mnt")
231
232 initiatorAuto.succeed("mount /dev/sdb /mnt")
233 initiatorAuto.succeed("test -e /mnt/hi")
234 initiatorAuto.succeed("umount /mnt")
235
236 initiatorAuto.succeed("systemctl restart multipathd")
237 initiatorAuto.succeed("systemd-cat multipath -ll")
238
239 # Install our RootDisk machine to 123456, the alias to the device that multipath is now managing
240 initiatorAuto.succeed("mount /dev/mapper/123456 /mnt")
241 initiatorAuto.succeed("mkdir -p /mnt/etc/{multipath,iscsi}")
242 initiatorAuto.succeed("cp -r /etc/multipath/wwids /mnt/etc/multipath/wwids")
243 initiatorAuto.succeed("cp -r /etc/iscsi/{nodes,send_targets} /mnt/etc/iscsi")
244 initiatorAuto.succeed(
245 "nixos-install --no-bootloader --no-root-passwd --system /etc/initiator-root-disk-closure"
246 )
247 initiatorAuto.succeed("umount /mnt")
248 initiatorAuto.shutdown()
249
250 initiatorRootDisk.start()
251 initiatorRootDisk.wait_for_unit("multi-user.target")
252 initiatorRootDisk.wait_for_unit("iscsid")
253
254 # Log in over both nodes
255 initiatorRootDisk.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.1.3 --login")
256 initiatorRootDisk.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login")
257 initiatorRootDisk.succeed("systemctl restart multipathd")
258 initiatorRootDisk.succeed("systemd-cat multipath -ll")
259
260 # Verify we can write and sync the root disk
261 initiatorRootDisk.succeed("mkdir /scratch")
262 initiatorRootDisk.succeed("touch /scratch/both-up")
263 initiatorRootDisk.succeed("sync /scratch")
264
265 # Verify we can write to the root with ens9 (sda, 192.168.1.3) down
266 initiatorRootDisk.succeed("ip link set ens9 down")
267 initiatorRootDisk.succeed("touch /scratch/ens9-down")
268 initiatorRootDisk.succeed("sync /scratch")
269 initiatorRootDisk.succeed("ip link set ens9 up")
270
271 # todo: better way to wait until multipath notices the link is back
272 initiatorRootDisk.succeed("sleep 5")
273 initiatorRootDisk.succeed("touch /scratch/both-down")
274 initiatorRootDisk.succeed("sync /scratch")
275
276 # Verify we can write to the root with ens10 (sdb, 192.168.2.3) down
277 initiatorRootDisk.succeed("ip link set ens10 down")
278 initiatorRootDisk.succeed("touch /scratch/ens10-down")
279 initiatorRootDisk.succeed("sync /scratch")
280 initiatorRootDisk.succeed("ip link set ens10 up")
281 initiatorRootDisk.succeed("touch /scratch/ens10-down")
282 initiatorRootDisk.succeed("sync /scratch")
283
284 initiatorRootDisk.succeed("ip link set ens9 up")
285 initiatorRootDisk.succeed("ip link set ens10 up")
286 initiatorRootDisk.shutdown()
287
288 # Verify we can boot with the target's eth1 down, forcing
289 # it to multipath via the second link
290 target.succeed("ip link set eth1 down")
291 initiatorRootDisk.start()
292 initiatorRootDisk.wait_for_unit("multi-user.target")
293 initiatorRootDisk.wait_for_unit("iscsid")
294 initiatorRootDisk.succeed("test -e /scratch/both-up")
295 '';
296}