pyrox.dev
1{ config, lib, pkgs, ... }:
2
3with lib;
4
5let
6
7 cfg = config.hardware.infiniband;
8 opensm-services = {
9 "opensm@" = {
10 enable = true;
11 description = "Starts OpenSM Infiniband fabric Subnet Managers";
12 before = [ "network.target"];
13 unitConfig = {
14 ConditionPathExists = "/sys/class/infiniband_mad/abi_version";
15 };
16 serviceConfig = {
17 Type = "simple";
18 ExecStart = "${pkgs.opensm}/bin/opensm --guid %I --log_file /var/log/opensm.%I.log";
19 };
20 };
21 } // (builtins.listToAttrs (map (guid: {
22 name = "opensm@${guid}";
23 value = {
24 enable = true;
25 wantedBy = [ "machines.target" ];
26 overrideStrategy = "asDropin";
27 };
28 } ) cfg.guids));
29
30in
31
32{
33 options.hardware.infiniband = {
34 enable = mkEnableOption "Infiniband support";
35 guids = mkOption {
36 type = with types; listOf str;
37 default = [];
38 example = [ "0xe8ebd30000eee2e1" ];
39 description = ''
40 A list of infiniband port guids on the system. This is discoverable using `ibstat -p`
41 '';
42 };
43 };
44
45 config = mkIf cfg.enable {
46 boot.initrd.kernelModules = [
47 "mlx5_core" "mlx5_ib" "ib_cm"
48 "rdma_cm" "rdma_ucm" "rpcrdma"
49 "ib_ipoib" "ib_isert" "ib_umad" "ib_uverbs"
50 ];
51 # rdma-core exposes ibstat, mstflint exposes mstconfig (which can be needed for
52 # setting link configurations), qperf needed to affirm link speeds
53 environment.systemPackages = with pkgs; [
54 rdma-core mstflint qperf
55 ];
56 systemd.services = opensm-services;
57 };
58}