pyrox.dev
1{
2 config,
3 lib,
4 pkgs,
5 ...
6}:
7let
8
9 cfg = config.hardware.infiniband;
10 opensm-services =
11 {
12 "opensm@" = {
13 enable = true;
14 description = "Starts OpenSM Infiniband fabric Subnet Managers";
15 before = [ "network.target" ];
16 unitConfig = {
17 ConditionPathExists = "/sys/class/infiniband_mad/abi_version";
18 };
19 serviceConfig = {
20 Type = "simple";
21 ExecStart = "${pkgs.opensm}/bin/opensm --guid %I --log_file /var/log/opensm.%I.log";
22 };
23 };
24 }
25 // (builtins.listToAttrs (
26 map (guid: {
27 name = "opensm@${guid}";
28 value = {
29 enable = true;
30 wantedBy = [ "machines.target" ];
31 overrideStrategy = "asDropin";
32 };
33 }) cfg.guids
34 ));
35
36in
37
38{
39 options.hardware.infiniband = {
40 enable = lib.mkEnableOption "Infiniband support";
41 guids = lib.mkOption {
42 type = with lib.types; listOf str;
43 default = [ ];
44 example = [ "0xe8ebd30000eee2e1" ];
45 description = ''
46 A list of infiniband port guids on the system. This is discoverable using `ibstat -p`
47 '';
48 };
49 };
50
51 config = lib.mkIf cfg.enable {
52 boot.initrd.kernelModules = [
53 "mlx5_core"
54 "mlx5_ib"
55 "ib_cm"
56 "rdma_cm"
57 "rdma_ucm"
58 "rpcrdma"
59 "ib_ipoib"
60 "ib_isert"
61 "ib_umad"
62 "ib_uverbs"
63 ];
64 # rdma-core exposes ibstat, mstflint exposes mstconfig (which can be needed for
65 # setting link configurations), qperf needed to affirm link speeds
66 environment.systemPackages = with pkgs; [
67 rdma-core
68 mstflint
69 qperf
70 ];
71 systemd.services = opensm-services;
72 };
73}