pyrox.dev
1{ ... }:
2
3let
4 node =
5 { pkgs, ... }:
6 {
7 networking = {
8 firewall = {
9 allowedUDPPorts = [ 4791 ]; # open RoCE port
10 allowedTCPPorts = [ 4800 ]; # port for test utils
11 };
12 rxe = {
13 enable = true;
14 interfaces = [ "eth1" ];
15 };
16 };
17
18 environment.systemPackages = with pkgs; [
19 rdma-core
20 screen
21 ];
22 };
23
24in
25{
26 name = "rxe";
27
28 nodes = {
29 server = node;
30 client = node;
31 };
32
33 testScript = ''
34 # Test if rxe interface comes up
35 server.wait_for_unit("default.target")
36 server.succeed("systemctl status rxe.service")
37 server.succeed("ibv_devices | grep rxe_eth1")
38
39 client.wait_for_unit("default.target")
40
41 # ping pong tests
42 for proto in "rc", "uc", "ud", "srq":
43 server.succeed(
44 "screen -dmS {0}_pingpong ibv_{0}_pingpong -p 4800 -s 1024 -g0".format(proto)
45 )
46 client.succeed("sleep 2; ibv_{}_pingpong -p 4800 -s 1024 -g0 server".format(proto))
47
48 server.succeed("screen -dmS rping rping -s -a server -C 10")
49 client.succeed("sleep 2; rping -c -a server -C 10")
50 '';
51}