pyrox.dev
1{ config, lib, pkgs, ... }:
2
3with lib;
4
5let
6
7 cfg = config.services.slurm;
8 # configuration file can be generated by http://slurm.schedmd.com/configurator.html
9 configFile = pkgs.writeTextDir "slurm.conf"
10 ''
11 ${optionalString (cfg.controlMachine != null) ''controlMachine=${cfg.controlMachine}''}
12 ${optionalString (cfg.controlAddr != null) ''controlAddr=${cfg.controlAddr}''}
13 ${optionalString (cfg.nodeName != null) ''nodeName=${cfg.nodeName}''}
14 ${optionalString (cfg.partitionName != null) ''partitionName=${cfg.partitionName}''}
15 PlugStackConfig=${plugStackConfig}
16 ProctrackType=${cfg.procTrackType}
17 ${cfg.extraConfig}
18 '';
19
20 plugStackConfig = pkgs.writeTextDir "plugstack.conf"
21 ''
22 ${optionalString cfg.enableSrunX11 ''optional ${pkgs.slurm-spank-x11}/lib/x11.so''}
23 ${cfg.extraPlugstackConfig}
24 '';
25
26
27 cgroupConfig = pkgs.writeTextDir "cgroup.conf"
28 ''
29 ${cfg.extraCgroupConfig}
30 '';
31
32 # slurm expects some additional config files to be
33 # in the same directory as slurm.conf
34 etcSlurm = pkgs.symlinkJoin {
35 name = "etc-slurm";
36 paths = [ configFile cgroupConfig plugStackConfig ];
37 };
38
39in
40
41{
42
43 ###### interface
44
45 options = {
46
47 services.slurm = {
48
49 server = {
50 enable = mkOption {
51 type = types.bool;
52 default = false;
53 description = ''
54 Wether to enable the slurm control daemon.
55 Note that the standard authentication method is "munge".
56 The "munge" service needs to be provided with a password file in order for
57 slurm to work properly (see <literal>services.munge.password</literal>).
58 '';
59 };
60 };
61
62 client = {
63 enable = mkEnableOption "slurm client daemon";
64 };
65
66 enableStools = mkOption {
67 type = types.bool;
68 default = false;
69 description = ''
70 Wether to provide a slurm.conf file.
71 Enable this option if you do not run a slurm daemon on this host
72 (i.e. <literal>server.enable</literal> and <literal>client.enable</literal> are <literal>false</literal>)
73 but you still want to run slurm commands from this host.
74 '';
75 };
76
77 package = mkOption {
78 type = types.package;
79 default = pkgs.slurm;
80 defaultText = "pkgs.slurm";
81 example = literalExample "pkgs.slurm-full";
82 description = ''
83 The package to use for slurm binaries.
84 '';
85 };
86
87 controlMachine = mkOption {
88 type = types.nullOr types.str;
89 default = null;
90 example = null;
91 description = ''
92 The short hostname of the machine where SLURM control functions are
93 executed (i.e. the name returned by the command "hostname -s", use "tux001"
94 rather than "tux001.my.com").
95 '';
96 };
97
98 controlAddr = mkOption {
99 type = types.nullOr types.str;
100 default = cfg.controlMachine;
101 example = null;
102 description = ''
103 Name that ControlMachine should be referred to in establishing a
104 communications path.
105 '';
106 };
107
108 nodeName = mkOption {
109 type = types.nullOr types.str;
110 default = null;
111 example = "linux[1-32] CPUs=1 State=UNKNOWN";
112 description = ''
113 Name that SLURM uses to refer to a node (or base partition for BlueGene
114 systems). Typically this would be the string that "/bin/hostname -s"
115 returns. Note that now you have to write node's parameters after the name.
116 '';
117 };
118
119 partitionName = mkOption {
120 type = types.nullOr types.str;
121 default = null;
122 example = "debug Nodes=linux[1-32] Default=YES MaxTime=INFINITE State=UP";
123 description = ''
124 Name by which the partition may be referenced. Note that now you have
125 to write the partition's parameters after the name.
126 '';
127 };
128
129 enableSrunX11 = mkOption {
130 default = false;
131 type = types.bool;
132 description = ''
133 If enabled srun will accept the option "--x11" to allow for X11 forwarding
134 from within an interactive session or a batch job. This activates the
135 slurm-spank-x11 module. Note that this option also enables
136 'services.openssh.forwardX11' on the client.
137
138 This option requires slurm to be compiled without native X11 support.
139 '';
140 };
141
142 procTrackType = mkOption {
143 type = types.string;
144 default = "proctrack/linuxproc";
145 description = ''
146 Plugin to be used for process tracking on a job step basis.
147 The slurmd daemon uses this mechanism to identify all processes
148 which are children of processes it spawns for a user job step.
149 '';
150 };
151
152 extraConfig = mkOption {
153 default = "";
154 type = types.lines;
155 description = ''
156 Extra configuration options that will be added verbatim at
157 the end of the slurm configuration file.
158 '';
159 };
160
161 extraPlugstackConfig = mkOption {
162 default = "";
163 type = types.lines;
164 description = ''
165 Extra configuration that will be added to the end of <literal>plugstack.conf</literal>.
166 '';
167 };
168
169 extraCgroupConfig = mkOption {
170 default = "";
171 type = types.lines;
172 description = ''
173 Extra configuration for <literal>cgroup.conf</literal>. This file is
174 used when <literal>procTrackType=proctrack/cgroup</literal>.
175 '';
176 };
177 };
178
179 };
180
181
182 ###### implementation
183
184 config =
185 let
186 wrappedSlurm = pkgs.stdenv.mkDerivation {
187 name = "wrappedSlurm";
188
189 builder = pkgs.writeText "builder.sh" ''
190 source $stdenv/setup
191 mkdir -p $out/bin
192 find ${getBin cfg.package}/bin -type f -executable | while read EXE
193 do
194 exename="$(basename $EXE)"
195 wrappername="$out/bin/$exename"
196 cat > "$wrappername" <<EOT
197 #!/bin/sh
198 if [ -z "$SLURM_CONF" ]
199 then
200 SLURM_CONF="${etcSlurm}/slurm.conf" "$EXE" "\$@"
201 else
202 "$EXE" "\$0"
203 fi
204 EOT
205 chmod +x "$wrappername"
206 done
207
208 mkdir -p $out/share
209 ln -s ${getBin cfg.package}/share/man $out/share/man
210 '';
211 };
212
213 in mkIf (cfg.enableStools || cfg.client.enable || cfg.server.enable) {
214
215 environment.systemPackages = [ wrappedSlurm ];
216
217 services.munge.enable = mkDefault true;
218
219 systemd.services.slurmd = mkIf (cfg.client.enable) {
220 path = with pkgs; [ wrappedSlurm coreutils ]
221 ++ lib.optional cfg.enableSrunX11 slurm-spank-x11;
222
223 wantedBy = [ "multi-user.target" ];
224 after = [ "systemd-tmpfiles-clean.service" ];
225
226 serviceConfig = {
227 Type = "forking";
228 ExecStart = "${wrappedSlurm}/bin/slurmd";
229 PIDFile = "/run/slurmd.pid";
230 ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
231 };
232
233 preStart = ''
234 mkdir -p /var/spool
235 '';
236 };
237
238 services.openssh.forwardX11 = mkIf cfg.client.enable (mkDefault true);
239
240 systemd.services.slurmctld = mkIf (cfg.server.enable) {
241 path = with pkgs; [ wrappedSlurm munge coreutils ]
242 ++ lib.optional cfg.enableSrunX11 slurm-spank-x11;
243
244 wantedBy = [ "multi-user.target" ];
245 after = [ "network.target" "munged.service" ];
246 requires = [ "munged.service" ];
247
248 serviceConfig = {
249 Type = "forking";
250 ExecStart = "${wrappedSlurm}/bin/slurmctld";
251 PIDFile = "/run/slurmctld.pid";
252 ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
253 };
254 };
255
256 };
257
258}