pyrox.dev
1{
2 config,
3 lib,
4 pkgs,
5 ...
6}:
7let
8 nvidiaEnabled = lib.elem "nvidia" config.services.xserver.videoDrivers;
9 nvidia_x11 = if nvidiaEnabled || cfg.datacenter.enable then cfg.package else null;
10
11 cfg = config.hardware.nvidia;
12
13 useOpenModules = cfg.open == true;
14
15 pCfg = cfg.prime;
16 syncCfg = pCfg.sync;
17 offloadCfg = pCfg.offload;
18 reverseSyncCfg = pCfg.reverseSync;
19 primeEnabled = syncCfg.enable || reverseSyncCfg.enable || offloadCfg.enable;
20 busIDType = lib.types.strMatching "([[:print:]]+:[0-9]{1,3}(@[0-9]{1,10})?:[0-9]{1,2}:[0-9])?";
21 ibtSupport = useOpenModules || (nvidia_x11.ibtSupport or false);
22 settingsFormat = pkgs.formats.keyValue { };
23in
24{
25 options = {
26 hardware.nvidia = {
27 enabled = lib.mkOption {
28 readOnly = true;
29 type = lib.types.bool;
30 default = nvidia_x11 != null;
31 defaultText = lib.literalMD "`true` if NVIDIA support is enabled";
32 description = "True if NVIDIA support is enabled";
33 };
34 datacenter.enable = lib.mkEnableOption ''
35 Data Center drivers for NVIDIA cards on a NVLink topology
36 '';
37 datacenter.settings = lib.mkOption {
38 type = settingsFormat.type;
39 default = {
40 LOG_LEVEL = 4;
41 LOG_FILE_NAME = "/var/log/fabricmanager.log";
42 LOG_APPEND_TO_LOG = 1;
43 LOG_FILE_MAX_SIZE = 1024;
44 LOG_USE_SYSLOG = 0;
45 DAEMONIZE = 1;
46 BIND_INTERFACE_IP = "127.0.0.1";
47 STARTING_TCP_PORT = 16000;
48 FABRIC_MODE = 0;
49 FABRIC_MODE_RESTART = 0;
50 STATE_FILE_NAME = "/var/tmp/fabricmanager.state";
51 FM_CMD_BIND_INTERFACE = "127.0.0.1";
52 FM_CMD_PORT_NUMBER = 6666;
53 FM_STAY_RESIDENT_ON_FAILURES = 0;
54 ACCESS_LINK_FAILURE_MODE = 0;
55 TRUNK_LINK_FAILURE_MODE = 0;
56 NVSWITCH_FAILURE_MODE = 0;
57 ABORT_CUDA_JOBS_ON_FM_EXIT = 1;
58 };
59 defaultText = lib.literalExpression ''
60 {
61 LOG_LEVEL=4;
62 LOG_FILE_NAME="/var/log/fabricmanager.log";
63 LOG_APPEND_TO_LOG=1;
64 LOG_FILE_MAX_SIZE=1024;
65 LOG_USE_SYSLOG=0;
66 DAEMONIZE=1;
67 BIND_INTERFACE_IP="127.0.0.1";
68 STARTING_TCP_PORT=16000;
69 FABRIC_MODE=0;
70 FABRIC_MODE_RESTART=0;
71 STATE_FILE_NAME="/var/tmp/fabricmanager.state";
72 FM_CMD_BIND_INTERFACE="127.0.0.1";
73 FM_CMD_PORT_NUMBER=6666;
74 FM_STAY_RESIDENT_ON_FAILURES=0;
75 ACCESS_LINK_FAILURE_MODE=0;
76 TRUNK_LINK_FAILURE_MODE=0;
77 NVSWITCH_FAILURE_MODE=0;
78 ABORT_CUDA_JOBS_ON_FM_EXIT=1;
79 }
80 '';
81 description = ''
82 Additional configuration options for fabricmanager.
83 '';
84 };
85
86 powerManagement.enable = lib.mkEnableOption ''
87 experimental power management through systemd. For more information, see
88 the NVIDIA docs, on Chapter 21. Configuring Power Management Support
89 '';
90
91 powerManagement.finegrained = lib.mkEnableOption ''
92 experimental power management of PRIME offload. For more information, see
93 the NVIDIA docs, on Chapter 22. PCI-Express Runtime D3 (RTD3) Power Management
94 '';
95
96 dynamicBoost.enable = lib.mkEnableOption ''
97 dynamic Boost balances power between the CPU and the GPU for improved
98 performance on supported laptops using the nvidia-powerd daemon. For more
99 information, see the NVIDIA docs, on Chapter 23. Dynamic Boost on Linux
100 '';
101
102 modesetting.enable =
103 lib.mkEnableOption ''
104 kernel modesetting when using the NVIDIA proprietary driver.
105
106 Enabling this fixes screen tearing when using Optimus via PRIME (see
107 {option}`hardware.nvidia.prime.sync.enable`. This is not enabled
108 by default because it is not officially supported by NVIDIA and would not
109 work with SLI.
110
111 Enabling this and using version 545 or newer of the proprietary NVIDIA
112 driver causes it to provide its own framebuffer device, which can cause
113 Wayland compositors to work when they otherwise wouldn't.
114 ''
115 // {
116 default = lib.versionAtLeast cfg.package.version "535";
117 defaultText = lib.literalExpression "lib.versionAtLeast cfg.package.version \"535\"";
118 };
119
120 prime.nvidiaBusId = lib.mkOption {
121 type = busIDType;
122 default = "";
123 example = "PCI:1@0:0:0";
124 description = ''
125 Bus ID of the NVIDIA GPU. You can find it using lspci; for example if lspci
126 shows the NVIDIA GPU at "0001:02:03.4", set this option to "PCI:2@1:3:4".
127
128 lspci might omit the PCI domain (0001 in above example) if it is zero.
129 In which case, use "@0" instead.
130
131 Please be aware that this option takes decimal address while lspci reports
132 hexadecimal address. So for device at domain "10000", use "@65536".
133 '';
134 };
135
136 prime.intelBusId = lib.mkOption {
137 type = busIDType;
138 default = "";
139 example = "PCI:0@0:2:0";
140 description = ''
141 Bus ID of the Intel GPU. You can find it using lspci; for example if lspci
142 shows the Intel GPU at "0001:02:03.4", set this option to "PCI:2@1:3:4".
143
144 lspci might omit the PCI domain (0001 in above example) if it is zero.
145 In which case, use "@0" instead.
146
147 Please be aware that this option takes decimal address while lspci reports
148 hexadecimal address. So for device at domain "10000", use "@65536".
149 '';
150 };
151
152 prime.amdgpuBusId = lib.mkOption {
153 type = busIDType;
154 default = "";
155 example = "PCI:4@0:0:0";
156 description = ''
157 Bus ID of the AMD APU. You can find it using lspci; for example if lspci
158 shows the AMD APU at "0001:02:03.4", set this option to "PCI:2@1:3:4".
159
160 lspci might omit the PCI domain (0001 in above example) if it is zero.
161 In which case, use "@0" instead.
162
163 Please be aware that this option takes decimal address while lspci reports
164 hexadecimal address. So for device at domain "10000", use "@65536".
165 '';
166 };
167
168 prime.sync.enable = lib.mkEnableOption ''
169 NVIDIA Optimus support using the NVIDIA proprietary driver via PRIME.
170 If enabled, the NVIDIA GPU will be always on and used for all rendering,
171 while enabling output to displays attached only to the integrated Intel/AMD
172 GPU without a multiplexer.
173
174 Note that this option only has any effect if the "nvidia" driver is specified
175 in {option}`services.xserver.videoDrivers`, and it should preferably
176 be the only driver there.
177
178 If this is enabled, then the bus IDs of the NVIDIA and Intel/AMD GPUs have to
179 be specified ({option}`hardware.nvidia.prime.nvidiaBusId` and
180 {option}`hardware.nvidia.prime.intelBusId` or
181 {option}`hardware.nvidia.prime.amdgpuBusId`).
182
183 If you enable this, you may want to also enable kernel modesetting for the
184 NVIDIA driver ({option}`hardware.nvidia.modesetting.enable`) in order
185 to prevent tearing.
186
187 Note that this configuration will only be successful when a display manager
188 for which the {option}`services.xserver.displayManager.setupCommands`
189 option is supported is used
190 '';
191
192 prime.allowExternalGpu = lib.mkEnableOption ''
193 configuring X to allow external NVIDIA GPUs when using Prime [Reverse] sync optimus
194 '';
195
196 prime.offload.enable = lib.mkEnableOption ''
197 render offload support using the NVIDIA proprietary driver via PRIME.
198
199 If this is enabled, then the bus IDs of the NVIDIA and Intel/AMD GPUs have to
200 be specified ({option}`hardware.nvidia.prime.nvidiaBusId` and
201 {option}`hardware.nvidia.prime.intelBusId` or
202 {option}`hardware.nvidia.prime.amdgpuBusId`)
203 '';
204
205 prime.offload.enableOffloadCmd = lib.mkEnableOption ''
206 adding a `nvidia-offload` convenience script to {option}`environment.systemPackages`
207 for offloading programs to an nvidia device. To work, should have also enabled
208 {option}`hardware.nvidia.prime.offload.enable` or {option}`hardware.nvidia.prime.reverseSync.enable`.
209
210 Example usage `nvidia-offload sauerbraten_client`
211 '';
212
213 prime.reverseSync.enable = lib.mkEnableOption ''
214 NVIDIA Optimus support using the NVIDIA proprietary driver via reverse
215 PRIME. If enabled, the Intel/AMD GPU will be used for all rendering, while
216 enabling output to displays attached only to the NVIDIA GPU without a
217 multiplexer.
218
219 Warning: This feature is relatively new, depending on your system this might
220 work poorly. AMD support, especially so.
221 See: https://forums.developer.nvidia.com/t/the-all-new-outputsink-feature-aka-reverse-prime/129828
222
223 Note that this option only has any effect if the "nvidia" driver is specified
224 in {option}`services.xserver.videoDrivers`, and it should preferably
225 be the only driver there.
226
227 If this is enabled, then the bus IDs of the NVIDIA and Intel/AMD GPUs have to
228 be specified ({option}`hardware.nvidia.prime.nvidiaBusId` and
229 {option}`hardware.nvidia.prime.intelBusId` or
230 {option}`hardware.nvidia.prime.amdgpuBusId`).
231
232 If you enable this, you may want to also enable kernel modesetting for the
233 NVIDIA driver ({option}`hardware.nvidia.modesetting.enable`) in order
234 to prevent tearing.
235
236 Note that this configuration will only be successful when a display manager
237 for which the {option}`services.xserver.displayManager.setupCommands`
238 option is supported is used
239 '';
240
241 prime.reverseSync.setupCommands.enable =
242 (lib.mkEnableOption ''
243 configure the display manager to be able to use the outputs
244 attached to the NVIDIA GPU.
245 Disable in order to configure the NVIDIA GPU outputs manually using xrandr.
246 Note that this configuration will only be successful when a display manager
247 for which the {option}`services.xserver.displayManager.setupCommands`
248 option is supported is used
249 '')
250 // {
251 default = true;
252 };
253
254 nvidiaSettings =
255 (lib.mkEnableOption ''
256 nvidia-settings, NVIDIA's GUI configuration tool
257 '')
258 // {
259 default = true;
260 };
261
262 nvidiaPersistenced = lib.mkEnableOption ''
263 nvidia-persistenced a update for NVIDIA GPU headless mode, i.e.
264 It ensures all GPUs stay awake even during headless mode
265 '';
266
267 forceFullCompositionPipeline = lib.mkEnableOption ''
268 forcefully the full composition pipeline.
269 This sometimes fixes screen tearing issues.
270 This has been reported to reduce the performance of some OpenGL applications and may produce issues in WebGL.
271 It also drastically increases the time the driver needs to clock down after load
272 '';
273
274 package = lib.mkOption {
275 default =
276 config.boot.kernelPackages.nvidiaPackages."${if cfg.datacenter.enable then "dc" else "stable"}";
277 defaultText = lib.literalExpression ''
278 config.boot.kernelPackages.nvidiaPackages."\$\{if cfg.datacenter.enable then "dc" else "stable"}"
279 '';
280 example = "config.boot.kernelPackages.nvidiaPackages.legacy_470";
281 description = ''
282 The NVIDIA driver package to use.
283 '';
284 };
285
286 open = lib.mkOption {
287 example = true;
288 description = "Whether to enable the open source NVIDIA kernel module.";
289 type = lib.types.nullOr lib.types.bool;
290 default = if lib.versionOlder nvidia_x11.version "560" then false else null;
291 defaultText = lib.literalExpression ''
292 if lib.versionOlder config.hardware.nvidia.package.version "560" then false else null
293 '';
294 };
295
296 gsp.enable =
297 lib.mkEnableOption ''
298 the GPU System Processor (GSP) on the video card
299 ''
300 // {
301 default = useOpenModules || lib.versionAtLeast nvidia_x11.version "555";
302 defaultText = lib.literalExpression ''
303 config.hardware.nvidia.open == true || lib.versionAtLeast config.hardware.nvidia.package.version "555"
304 '';
305 };
306
307 videoAcceleration =
308 (lib.mkEnableOption ''
309 Whether video acceleration (VA-API) should be enabled.
310 '')
311 // {
312 default = true;
313 };
314 };
315 };
316
317 config =
318 let
319 igpuDriver = if pCfg.intelBusId != "" then "modesetting" else "amdgpu";
320 igpuBusId = if pCfg.intelBusId != "" then pCfg.intelBusId else pCfg.amdgpuBusId;
321 in
322 lib.mkIf cfg.enabled (
323 lib.mkMerge [
324 # Common
325 ({
326 assertions = [
327 {
328 assertion = !(nvidiaEnabled && cfg.datacenter.enable);
329 message = "You cannot configure both X11 and Data Center drivers at the same time.";
330 }
331 {
332 assertion = cfg.open != null || cfg.datacenter.enable;
333 message = ''
334 You must configure `hardware.nvidia.open` on NVIDIA driver versions >= 560.
335 It is suggested to use the open source kernel modules on Turing or later GPUs (RTX series, GTX 16xx), and the closed source modules otherwise.
336 '';
337 }
338 ];
339 boot = {
340 blacklistedKernelModules = [
341 "nouveau"
342 "nvidiafb"
343 ];
344
345 # Don't add `nvidia-uvm` to `kernelModules`, because we want
346 # `nvidia-uvm` be loaded only after the GPU device is available, i.e. after `udev` rules
347 # for `nvidia` kernel module are applied.
348 # This matters on Azure GPU instances: https://github.com/NixOS/nixpkgs/pull/267335
349 #
350 # Instead, we use `softdep` to lazily load `nvidia-uvm` kernel module
351 # after `nvidia` kernel module is loaded and `udev` rules are applied.
352 extraModprobeConfig = ''
353 softdep nvidia post: nvidia-uvm
354 '';
355
356 # Exception is the open-source kernel module failing to load nvidia-uvm using softdep
357 # for unknown reasons.
358 # It affects CUDA: https://github.com/NixOS/nixpkgs/issues/334180
359 # Previously nvidia-uvm was explicitly loaded only when xserver was enabled:
360 # https://github.com/NixOS/nixpkgs/pull/334340/commits/4548c392862115359e50860bcf658cfa8715bde9
361 # We are now loading the module eagerly for all users of the open driver (including headless).
362 kernelModules = lib.optionals useOpenModules [ "nvidia_uvm" ];
363 };
364 systemd.tmpfiles.rules = lib.mkIf config.virtualisation.docker.enableNvidia [
365 "L+ /run/nvidia-docker/bin - - - - ${nvidia_x11.bin}/origBin"
366 ];
367 services.udev.extraRules = ''
368 # Create /dev/nvidia-uvm when the nvidia-uvm module is loaded.
369 KERNEL=="nvidia", RUN+="${pkgs.runtimeShell} -c 'mknod -m 666 /dev/nvidiactl c 195 255'"
370 KERNEL=="nvidia", RUN+="${pkgs.runtimeShell} -c 'for i in $$(cat /proc/driver/nvidia/gpus/*/information | grep Minor | cut -d \ -f 4); do mknod -m 666 /dev/nvidia$${i} c 195 $${i}; done'"
371 KERNEL=="nvidia_modeset", RUN+="${pkgs.runtimeShell} -c 'mknod -m 666 /dev/nvidia-modeset c 195 254'"
372 KERNEL=="nvidia_uvm", RUN+="${pkgs.runtimeShell} -c 'mknod -m 666 /dev/nvidia-uvm c $$(grep nvidia-uvm /proc/devices | cut -d \ -f 1) 0'"
373 KERNEL=="nvidia_uvm", RUN+="${pkgs.runtimeShell} -c 'mknod -m 666 /dev/nvidia-uvm-tools c $$(grep nvidia-uvm /proc/devices | cut -d \ -f 1) 1'"
374 '';
375 hardware.graphics = {
376 extraPackages = [ nvidia_x11.out ];
377 extraPackages32 = [ nvidia_x11.lib32 ];
378 };
379 environment.systemPackages = [ nvidia_x11.bin ];
380 })
381
382 # X11
383 (lib.mkIf nvidiaEnabled {
384 assertions = [
385 {
386 assertion = primeEnabled -> pCfg.intelBusId == "" || pCfg.amdgpuBusId == "";
387 message = "You cannot configure both an Intel iGPU and an AMD APU. Pick the one corresponding to your processor.";
388 }
389
390 {
391 assertion = offloadCfg.enableOffloadCmd -> offloadCfg.enable || reverseSyncCfg.enable;
392 message = "Offload command requires offloading or reverse prime sync to be enabled.";
393 }
394
395 {
396 assertion =
397 primeEnabled -> pCfg.nvidiaBusId != "" && (pCfg.intelBusId != "" || pCfg.amdgpuBusId != "");
398 message = "When NVIDIA PRIME is enabled, the GPU bus IDs must be configured.";
399 }
400
401 {
402 assertion = offloadCfg.enable -> lib.versionAtLeast nvidia_x11.version "435.21";
403 message = "NVIDIA PRIME render offload is currently only supported on versions >= 435.21.";
404 }
405
406 {
407 assertion =
408 (reverseSyncCfg.enable && pCfg.amdgpuBusId != "") -> lib.versionAtLeast nvidia_x11.version "470.0";
409 message = "NVIDIA PRIME render offload for AMD APUs is currently only supported on versions >= 470 beta.";
410 }
411
412 {
413 assertion = !(syncCfg.enable && offloadCfg.enable);
414 message = "PRIME Sync and Offload cannot be both enabled";
415 }
416
417 {
418 assertion = !(syncCfg.enable && reverseSyncCfg.enable);
419 message = "PRIME Sync and PRIME Reverse Sync cannot be both enabled";
420 }
421
422 {
423 assertion = !(syncCfg.enable && cfg.powerManagement.finegrained);
424 message = "Sync precludes powering down the NVIDIA GPU.";
425 }
426
427 {
428 assertion = cfg.powerManagement.finegrained -> offloadCfg.enable;
429 message = "Fine-grained power management requires offload to be enabled.";
430 }
431
432 {
433 assertion = cfg.powerManagement.enable -> lib.versionAtLeast nvidia_x11.version "430.09";
434 message = "Required files for driver based power management only exist on versions >= 430.09.";
435 }
436
437 {
438 assertion = cfg.gsp.enable -> (cfg.package ? firmware);
439 message = "This version of NVIDIA driver does not provide a GSP firmware.";
440 }
441
442 {
443 assertion = useOpenModules -> (cfg.package ? open);
444 message = "This version of NVIDIA driver does not provide a corresponding opensource kernel driver.";
445 }
446
447 {
448 assertion = useOpenModules -> cfg.gsp.enable;
449 message = "The GSP cannot be disabled when using the opensource kernel driver.";
450 }
451
452 {
453 assertion = cfg.dynamicBoost.enable -> lib.versionAtLeast nvidia_x11.version "510.39.01";
454 message = "NVIDIA's Dynamic Boost feature only exists on versions >= 510.39.01";
455 }
456 ];
457
458 # If Optimus/PRIME is enabled, we:
459 # - Specify the configured NVIDIA GPU bus ID in the Device section for the
460 # "nvidia" driver.
461 # - Add the AllowEmptyInitialConfiguration option to the Screen section for the
462 # "nvidia" driver, in order to allow the X server to start without any outputs.
463 # - Add a separate Device section for the Intel GPU, using the "modesetting"
464 # driver and with the configured BusID.
465 # - OR add a separate Device section for the AMD APU, using the "amdgpu"
466 # driver and with the configures BusID.
467 # - Reference that Device section from the ServerLayout section as an inactive
468 # device.
469 # - Configure the display manager to run specific `xrandr` commands which will
470 # configure/enable displays connected to the Intel iGPU / AMD APU.
471
472 # reverse sync implies offloading
473 hardware.nvidia.prime.offload.enable = lib.mkDefault reverseSyncCfg.enable;
474
475 services.xserver.drivers =
476 lib.optional primeEnabled {
477 name = igpuDriver;
478 display = offloadCfg.enable;
479 modules = lib.optional (igpuDriver == "amdgpu") pkgs.xorg.xf86videoamdgpu;
480 deviceSection =
481 ''
482 BusID "${igpuBusId}"
483 ''
484 + lib.optionalString (syncCfg.enable && igpuDriver != "amdgpu") ''
485 Option "AccelMethod" "none"
486 '';
487 }
488 ++ lib.singleton {
489 name = "nvidia";
490 modules = [ nvidia_x11.bin ];
491 display = !offloadCfg.enable;
492 deviceSection =
493 ''
494 Option "SidebandSocketPath" "/run/nvidia-xdriver/"
495 ''
496 + lib.optionalString primeEnabled ''
497 BusID "${pCfg.nvidiaBusId}"
498 ''
499 + lib.optionalString pCfg.allowExternalGpu ''
500 Option "AllowExternalGpus"
501 '';
502 screenSection =
503 ''
504 Option "RandRRotation" "on"
505 ''
506 + lib.optionalString syncCfg.enable ''
507 Option "AllowEmptyInitialConfiguration"
508 ''
509 + lib.optionalString cfg.forceFullCompositionPipeline ''
510 Option "metamodes" "nvidia-auto-select +0+0 {ForceFullCompositionPipeline=On}"
511 Option "AllowIndirectGLXProtocol" "off"
512 Option "TripleBuffer" "on"
513 '';
514 };
515
516 services.xserver.serverLayoutSection =
517 lib.optionalString syncCfg.enable ''
518 Inactive "Device-${igpuDriver}[0]"
519 ''
520 + lib.optionalString reverseSyncCfg.enable ''
521 Inactive "Device-nvidia[0]"
522 ''
523 + lib.optionalString offloadCfg.enable ''
524 Option "AllowNVIDIAGPUScreens"
525 '';
526
527 services.xserver.displayManager.setupCommands =
528 let
529 gpuProviderName =
530 if igpuDriver == "amdgpu" then
531 # find the name of the provider if amdgpu
532 "`${lib.getExe pkgs.xorg.xrandr} --listproviders | ${lib.getExe pkgs.gnugrep} -i AMD | ${lib.getExe pkgs.gnused} -n 's/^.*name://p'`"
533 else
534 igpuDriver;
535 providerCmdParams =
536 if syncCfg.enable then "\"${gpuProviderName}\" NVIDIA-0" else "NVIDIA-G0 \"${gpuProviderName}\"";
537 in
538 lib.optionalString
539 (syncCfg.enable || (reverseSyncCfg.enable && reverseSyncCfg.setupCommands.enable))
540 ''
541 # Added by nvidia configuration module for Optimus/PRIME.
542 ${lib.getExe pkgs.xorg.xrandr} --setprovideroutputsource ${providerCmdParams}
543 ${lib.getExe pkgs.xorg.xrandr} --auto
544 '';
545
546 environment.etc = {
547 "nvidia/nvidia-application-profiles-rc" = lib.mkIf nvidia_x11.useProfiles {
548 source = "${nvidia_x11.bin}/share/nvidia/nvidia-application-profiles-rc";
549 };
550
551 # 'nvidia_x11' installs it's files to /run/opengl-driver/...
552 "egl/egl_external_platform.d".source = "/run/opengl-driver/share/egl/egl_external_platform.d/";
553 };
554
555 hardware.graphics.extraPackages = lib.optional cfg.videoAcceleration pkgs.nvidia-vaapi-driver;
556
557 environment.systemPackages =
558 lib.optional cfg.nvidiaSettings nvidia_x11.settings
559 ++ lib.optional cfg.nvidiaPersistenced nvidia_x11.persistenced
560 ++ lib.optional offloadCfg.enableOffloadCmd (
561 pkgs.writeShellScriptBin "nvidia-offload" ''
562 export __NV_PRIME_RENDER_OFFLOAD=1
563 export __NV_PRIME_RENDER_OFFLOAD_PROVIDER=NVIDIA-G0
564 export __GLX_VENDOR_LIBRARY_NAME=nvidia
565 export __VK_LAYER_NV_optimus=NVIDIA_only
566 exec "$@"
567 ''
568 );
569
570 systemd.packages = lib.optional cfg.powerManagement.enable nvidia_x11.out;
571
572 systemd.services =
573 let
574 nvidiaService = state: {
575 description = "NVIDIA system ${state} actions";
576 path = [ pkgs.kbd ];
577 serviceConfig = {
578 Type = "oneshot";
579 ExecStart = "${nvidia_x11.out}/bin/nvidia-sleep.sh '${state}'";
580 };
581 before = [ "systemd-${state}.service" ];
582 requiredBy = [ "systemd-${state}.service" ];
583 };
584 in
585 lib.mkMerge [
586 (lib.mkIf cfg.powerManagement.enable {
587 nvidia-suspend = nvidiaService "suspend";
588 nvidia-hibernate = nvidiaService "hibernate";
589 nvidia-resume = (nvidiaService "resume") // {
590 before = [ ];
591 after = [
592 "systemd-suspend.service"
593 "systemd-hibernate.service"
594 ];
595 requiredBy = [
596 "systemd-suspend.service"
597 "systemd-hibernate.service"
598 ];
599 };
600 })
601 (lib.mkIf cfg.nvidiaPersistenced {
602 "nvidia-persistenced" = {
603 description = "NVIDIA Persistence Daemon";
604 wantedBy = [ "multi-user.target" ];
605 serviceConfig = {
606 Type = "forking";
607 Restart = "always";
608 PIDFile = "/var/run/nvidia-persistenced/nvidia-persistenced.pid";
609 ExecStart = "${lib.getExe nvidia_x11.persistenced} --verbose";
610 ExecStopPost = "${pkgs.coreutils}/bin/rm -rf /var/run/nvidia-persistenced";
611 };
612 };
613 })
614 (lib.mkIf cfg.dynamicBoost.enable {
615 "nvidia-powerd" = {
616 description = "nvidia-powerd service";
617 path = [
618 pkgs.util-linux # nvidia-powerd wants lscpu
619 ];
620 wantedBy = [ "multi-user.target" ];
621 serviceConfig = {
622 Type = "dbus";
623 BusName = "nvidia.powerd.server";
624 ExecStart = "${nvidia_x11.bin}/bin/nvidia-powerd";
625 };
626 };
627 })
628 ];
629
630 services.acpid.enable = true;
631
632 services.dbus.packages = lib.optional cfg.dynamicBoost.enable nvidia_x11.bin;
633
634 hardware.firmware = lib.optional cfg.gsp.enable nvidia_x11.firmware;
635
636 systemd.tmpfiles.rules =
637 [
638 # Remove the following log message:
639 # (WW) NVIDIA: Failed to bind sideband socket to
640 # (WW) NVIDIA: '/var/run/nvidia-xdriver-b4f69129' Permission denied
641 #
642 # https://bbs.archlinux.org/viewtopic.php?pid=1909115#p1909115
643 "d /run/nvidia-xdriver 0770 root users"
644 ]
645 ++ lib.optional (nvidia_x11.persistenced != null && config.virtualisation.docker.enableNvidia)
646 "L+ /run/nvidia-docker/extras/bin/nvidia-persistenced - - - - ${nvidia_x11.persistenced}/origBin/nvidia-persistenced";
647
648 boot = {
649 extraModulePackages = if useOpenModules then [ nvidia_x11.open ] else [ nvidia_x11.bin ];
650 # nvidia-uvm is required by CUDA applications.
651 kernelModules = lib.optionals config.services.xserver.enable [
652 "nvidia"
653 "nvidia_modeset"
654 "nvidia_drm"
655 ];
656
657 # If requested enable modesetting via kernel parameters.
658 kernelParams =
659 lib.optional (offloadCfg.enable || cfg.modesetting.enable) "nvidia-drm.modeset=1"
660 ++ lib.optional (
661 (offloadCfg.enable || cfg.modesetting.enable) && lib.versionAtLeast nvidia_x11.version "545"
662 ) "nvidia-drm.fbdev=1"
663 ++ lib.optional cfg.powerManagement.enable "nvidia.NVreg_PreserveVideoMemoryAllocations=1"
664 ++ lib.optional useOpenModules "nvidia.NVreg_OpenRmEnableUnsupportedGpus=1"
665 ++ lib.optional (config.boot.kernelPackages.kernel.kernelAtLeast "6.2" && !ibtSupport) "ibt=off";
666
667 # enable finegrained power management
668 extraModprobeConfig = lib.optionalString cfg.powerManagement.finegrained ''
669 options nvidia "NVreg_DynamicPowerManagement=0x02"
670 '';
671 };
672 services.udev.extraRules = lib.optionalString cfg.powerManagement.finegrained (
673 lib.optionalString (lib.versionOlder config.boot.kernelPackages.kernel.version "5.5") ''
674 # Remove NVIDIA USB xHCI Host Controller devices, if present
675 ACTION=="add", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x0c0330", ATTR{remove}="1"
676
677 # Remove NVIDIA USB Type-C UCSI devices, if present
678 ACTION=="add", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x0c8000", ATTR{remove}="1"
679
680 # Remove NVIDIA Audio devices, if present
681 ACTION=="add", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x040300", ATTR{remove}="1"
682 ''
683 + ''
684 # Enable runtime PM for NVIDIA VGA/3D controller devices on driver bind
685 ACTION=="bind", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x030000", TEST=="power/control", ATTR{power/control}="auto"
686 ACTION=="bind", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x030200", TEST=="power/control", ATTR{power/control}="auto"
687
688 # Disable runtime PM for NVIDIA VGA/3D controller devices on driver unbind
689 ACTION=="unbind", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x030000", TEST=="power/control", ATTR{power/control}="on"
690 ACTION=="unbind", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x030200", TEST=="power/control", ATTR{power/control}="on"
691 ''
692 );
693 })
694 # Data Center
695 (lib.mkIf (cfg.datacenter.enable) {
696 boot.extraModulePackages = [ nvidia_x11.bin ];
697
698 systemd = {
699 tmpfiles.rules =
700 lib.optional (nvidia_x11.persistenced != null && config.virtualisation.docker.enableNvidia)
701 "L+ /run/nvidia-docker/extras/bin/nvidia-persistenced - - - - ${nvidia_x11.persistenced}/origBin/nvidia-persistenced";
702
703 services = lib.mkMerge [
704 ({
705 nvidia-fabricmanager = {
706 enable = true;
707 description = "Start NVIDIA NVLink Management";
708 wantedBy = [ "multi-user.target" ];
709 unitConfig.After = [ "network-online.target" ];
710 unitConfig.Requires = [ "network-online.target" ];
711 serviceConfig = {
712 Type = "forking";
713 TimeoutStartSec = 240;
714 ExecStart =
715 let
716 # Since these rely on the `nvidia_x11.fabricmanager` derivation, they're
717 # unsuitable to be mentioned in the configuration defaults, but they _can_
718 # be overridden in `cfg.datacenter.settings` if needed.
719 fabricManagerConfDefaults = {
720 TOPOLOGY_FILE_PATH = "${nvidia_x11.fabricmanager}/share/nvidia-fabricmanager/nvidia/nvswitch";
721 DATABASE_PATH = "${nvidia_x11.fabricmanager}/share/nvidia-fabricmanager/nvidia/nvswitch";
722 };
723 nv-fab-conf = settingsFormat.generate "fabricmanager.conf" (
724 fabricManagerConfDefaults // cfg.datacenter.settings
725 );
726 in
727 "${lib.getExe nvidia_x11.fabricmanager} -c ${nv-fab-conf}";
728 LimitCORE = "infinity";
729 };
730 };
731 })
732 (lib.mkIf cfg.nvidiaPersistenced {
733 "nvidia-persistenced" = {
734 description = "NVIDIA Persistence Daemon";
735 wantedBy = [ "multi-user.target" ];
736 serviceConfig = {
737 Type = "forking";
738 Restart = "always";
739 PIDFile = "/var/run/nvidia-persistenced/nvidia-persistenced.pid";
740 ExecStart = "${lib.getExe nvidia_x11.persistenced} --verbose";
741 ExecStopPost = "${pkgs.coreutils}/bin/rm -rf /var/run/nvidia-persistenced";
742 };
743 };
744 })
745 ];
746 };
747
748 environment.systemPackages =
749 lib.optional cfg.datacenter.enable nvidia_x11.fabricmanager
750 ++ lib.optional cfg.nvidiaPersistenced nvidia_x11.persistenced;
751 })
752 ]
753 );
754}