pyrox.dev
1{
2 config,
3 lib,
4 pkgs,
5 ...
6}:
7let
8
9 cfg = config.hardware.infiniband;
10 opensm-services = {
11 "opensm@" = {
12 enable = true;
13 description = "Starts OpenSM Infiniband fabric Subnet Managers";
14 before = [ "network.target" ];
15 unitConfig = {
16 ConditionPathExists = "/sys/class/infiniband_mad/abi_version";
17 };
18 serviceConfig = {
19 Type = "simple";
20 ExecStart = "${pkgs.opensm}/bin/opensm --guid %I --log_file /var/log/opensm.%I.log";
21 };
22 };
23 }
24 // (builtins.listToAttrs (
25 map (guid: {
26 name = "opensm@${guid}";
27 value = {
28 enable = true;
29 wantedBy = [ "machines.target" ];
30 overrideStrategy = "asDropin";
31 };
32 }) cfg.guids
33 ));
34
35in
36
37{
38 options.hardware.infiniband = {
39 enable = lib.mkEnableOption "Infiniband support";
40 guids = lib.mkOption {
41 type = with lib.types; listOf str;
42 default = [ ];
43 example = [ "0xe8ebd30000eee2e1" ];
44 description = ''
45 A list of infiniband port guids on the system. This is discoverable using `ibstat -p`
46 '';
47 };
48 };
49
50 config = lib.mkIf cfg.enable {
51 boot.initrd.kernelModules = [
52 "mlx5_core"
53 "mlx5_ib"
54 "ib_cm"
55 "rdma_cm"
56 "rdma_ucm"
57 "rpcrdma"
58 "ib_ipoib"
59 "ib_isert"
60 "ib_umad"
61 "ib_uverbs"
62 ];
63 # rdma-core exposes ibstat, mstflint exposes mstconfig (which can be needed for
64 # setting link configurations), qperf needed to affirm link speeds
65 environment.systemPackages = with pkgs; [
66 rdma-core
67 mstflint
68 qperf
69 ];
70 systemd.services = opensm-services;
71 };
72}