pyrox.dev
1{
2 config,
3 lib,
4 pkgs,
5 ...
6}:
7let
8 nvidiaEnabled = lib.elem "nvidia" config.services.xserver.videoDrivers;
9 nvidia_x11 = if nvidiaEnabled || cfg.datacenter.enable then cfg.package else null;
10
11 cfg = config.hardware.nvidia;
12
13 useOpenModules = cfg.open == true;
14
15 pCfg = cfg.prime;
16 syncCfg = pCfg.sync;
17 offloadCfg = pCfg.offload;
18 reverseSyncCfg = pCfg.reverseSync;
19 primeEnabled = syncCfg.enable || reverseSyncCfg.enable || offloadCfg.enable;
20 busIDType = lib.types.strMatching "([[:print:]]+:[0-9]{1,3}(@[0-9]{1,10})?:[0-9]{1,2}:[0-9])?";
21 ibtSupport = useOpenModules || (nvidia_x11.ibtSupport or false);
22 settingsFormat = pkgs.formats.keyValue { };
23in
24{
25 options = {
26 hardware.nvidia = {
27 enabled = lib.mkOption {
28 readOnly = true;
29 type = lib.types.bool;
30 default = nvidia_x11 != null;
31 defaultText = lib.literalMD "`true` if NVIDIA support is enabled";
32 description = "True if NVIDIA support is enabled";
33 };
34 datacenter.enable = lib.mkEnableOption ''
35 Data Center drivers for NVIDIA cards on a NVLink topology
36 '';
37 datacenter.settings = lib.mkOption {
38 type = settingsFormat.type;
39 default = {
40 LOG_LEVEL = 4;
41 LOG_FILE_NAME = "/var/log/fabricmanager.log";
42 LOG_APPEND_TO_LOG = 1;
43 LOG_FILE_MAX_SIZE = 1024;
44 LOG_USE_SYSLOG = 0;
45 DAEMONIZE = 1;
46 BIND_INTERFACE_IP = "127.0.0.1";
47 STARTING_TCP_PORT = 16000;
48 FABRIC_MODE = 0;
49 FABRIC_MODE_RESTART = 0;
50 STATE_FILE_NAME = "/var/tmp/fabricmanager.state";
51 FM_CMD_BIND_INTERFACE = "127.0.0.1";
52 FM_CMD_PORT_NUMBER = 6666;
53 FM_STAY_RESIDENT_ON_FAILURES = 0;
54 ACCESS_LINK_FAILURE_MODE = 0;
55 TRUNK_LINK_FAILURE_MODE = 0;
56 NVSWITCH_FAILURE_MODE = 0;
57 ABORT_CUDA_JOBS_ON_FM_EXIT = 1;
58 };
59 defaultText = lib.literalExpression ''
60 {
61 LOG_LEVEL=4;
62 LOG_FILE_NAME="/var/log/fabricmanager.log";
63 LOG_APPEND_TO_LOG=1;
64 LOG_FILE_MAX_SIZE=1024;
65 LOG_USE_SYSLOG=0;
66 DAEMONIZE=1;
67 BIND_INTERFACE_IP="127.0.0.1";
68 STARTING_TCP_PORT=16000;
69 FABRIC_MODE=0;
70 FABRIC_MODE_RESTART=0;
71 STATE_FILE_NAME="/var/tmp/fabricmanager.state";
72 FM_CMD_BIND_INTERFACE="127.0.0.1";
73 FM_CMD_PORT_NUMBER=6666;
74 FM_STAY_RESIDENT_ON_FAILURES=0;
75 ACCESS_LINK_FAILURE_MODE=0;
76 TRUNK_LINK_FAILURE_MODE=0;
77 NVSWITCH_FAILURE_MODE=0;
78 ABORT_CUDA_JOBS_ON_FM_EXIT=1;
79 }
80 '';
81 description = ''
82 Additional configuration options for fabricmanager.
83 '';
84 };
85
86 powerManagement.enable = lib.mkEnableOption ''
87 experimental power management through systemd. For more information, see
88 the NVIDIA docs, on Chapter 21. Configuring Power Management Support
89 '';
90
91 powerManagement.finegrained = lib.mkEnableOption ''
92 experimental power management of PRIME offload. For more information, see
93 the NVIDIA docs, on Chapter 22. PCI-Express Runtime D3 (RTD3) Power Management
94 '';
95
96 dynamicBoost.enable = lib.mkEnableOption ''
97 dynamic Boost balances power between the CPU and the GPU for improved
98 performance on supported laptops using the nvidia-powerd daemon. For more
99 information, see the NVIDIA docs, on Chapter 23. Dynamic Boost on Linux
100 '';
101
102 modesetting.enable =
103 lib.mkEnableOption ''
104 kernel modesetting when using the NVIDIA proprietary driver.
105
106 Enabling this fixes screen tearing when using Optimus via PRIME (see
107 {option}`hardware.nvidia.prime.sync.enable`. This is not enabled
108 by default because it is not officially supported by NVIDIA and would not
109 work with SLI.
110
111 Enabling this and using version 545 or newer of the proprietary NVIDIA
112 driver causes it to provide its own framebuffer device, which can cause
113 Wayland compositors to work when they otherwise wouldn't.
114 ''
115 // {
116 default = lib.versionAtLeast cfg.package.version "535";
117 defaultText = lib.literalExpression "lib.versionAtLeast cfg.package.version \"535\"";
118 };
119
120 prime.nvidiaBusId = lib.mkOption {
121 type = busIDType;
122 default = "";
123 example = "PCI:1@0:0:0";
124 description = ''
125 Bus ID of the NVIDIA GPU. You can find it using lspci; for example if lspci
126 shows the NVIDIA GPU at "0001:02:03.4", set this option to "PCI:2@1:3:4".
127
128 lspci might omit the PCI domain (0001 in above example) if it is zero.
129 In which case, use "@0" instead.
130
131 Please be aware that this option takes decimal address while lspci reports
132 hexadecimal address. So for device at domain "10000", use "@65536".
133 '';
134 };
135
136 prime.intelBusId = lib.mkOption {
137 type = busIDType;
138 default = "";
139 example = "PCI:0@0:2:0";
140 description = ''
141 Bus ID of the Intel GPU. You can find it using lspci; for example if lspci
142 shows the Intel GPU at "0001:02:03.4", set this option to "PCI:2@1:3:4".
143
144 lspci might omit the PCI domain (0001 in above example) if it is zero.
145 In which case, use "@0" instead.
146
147 Please be aware that this option takes decimal address while lspci reports
148 hexadecimal address. So for device at domain "10000", use "@65536".
149 '';
150 };
151
152 prime.amdgpuBusId = lib.mkOption {
153 type = busIDType;
154 default = "";
155 example = "PCI:4@0:0:0";
156 description = ''
157 Bus ID of the AMD APU. You can find it using lspci; for example if lspci
158 shows the AMD APU at "0001:02:03.4", set this option to "PCI:2@1:3:4".
159
160 lspci might omit the PCI domain (0001 in above example) if it is zero.
161 In which case, use "@0" instead.
162
163 Please be aware that this option takes decimal address while lspci reports
164 hexadecimal address. So for device at domain "10000", use "@65536".
165 '';
166 };
167
168 prime.sync.enable = lib.mkEnableOption ''
169 NVIDIA Optimus support using the NVIDIA proprietary driver via PRIME.
170 If enabled, the NVIDIA GPU will be always on and used for all rendering,
171 while enabling output to displays attached only to the integrated Intel/AMD
172 GPU without a multiplexer.
173
174 Note that this option only has any effect if the "nvidia" driver is specified
175 in {option}`services.xserver.videoDrivers`, and it should preferably
176 be the only driver there.
177
178 If this is enabled, then the bus IDs of the NVIDIA and Intel/AMD GPUs have to
179 be specified ({option}`hardware.nvidia.prime.nvidiaBusId` and
180 {option}`hardware.nvidia.prime.intelBusId` or
181 {option}`hardware.nvidia.prime.amdgpuBusId`).
182
183 If you enable this, you may want to also enable kernel modesetting for the
184 NVIDIA driver ({option}`hardware.nvidia.modesetting.enable`) in order
185 to prevent tearing.
186
187 Note that this configuration will only be successful when a display manager
188 for which the {option}`services.xserver.displayManager.setupCommands`
189 option is supported is used
190 '';
191
192 prime.allowExternalGpu = lib.mkEnableOption ''
193 configuring X to allow external NVIDIA GPUs when using Prime [Reverse] sync optimus
194 '';
195
196 prime.offload.enable = lib.mkEnableOption ''
197 render offload support using the NVIDIA proprietary driver via PRIME.
198
199 If this is enabled, then the bus IDs of the NVIDIA and Intel/AMD GPUs have to
200 be specified ({option}`hardware.nvidia.prime.nvidiaBusId` and
201 {option}`hardware.nvidia.prime.intelBusId` or
202 {option}`hardware.nvidia.prime.amdgpuBusId`)
203 '';
204
205 prime.offload.enableOffloadCmd = lib.mkEnableOption ''
206 adding a `nvidia-offload` convenience script to {option}`environment.systemPackages`
207 for offloading programs to an nvidia device. To work, you must also enable
208 {option}`hardware.nvidia.prime.offload.enable` or {option}`hardware.nvidia.prime.reverseSync.enable`.
209
210 Example usage: `nvidia-offload sauerbraten_client`
211
212 This script can be renamed with {option}`hardware.nvidia.prime.offload.enableOffloadCmd`.
213 '';
214 prime.offload.offloadCmdMainProgram = lib.mkOption {
215 type = lib.types.str;
216 description = ''
217 Specifies the CLI name of the {option}`hardware.nvidia.prime.offload.enableOffloadCmd`
218 convenience script for offloading programs to an nvidia device.
219 '';
220 default = "nvidia-offload";
221 example = "prime-run";
222 };
223
224 prime.reverseSync.enable = lib.mkEnableOption ''
225 NVIDIA Optimus support using the NVIDIA proprietary driver via reverse
226 PRIME. If enabled, the Intel/AMD GPU will be used for all rendering, while
227 enabling output to displays attached only to the NVIDIA GPU without a
228 multiplexer.
229
230 Warning: This feature is relatively new, depending on your system this might
231 work poorly. AMD support, especially so.
232 See: <https://forums.developer.nvidia.com/t/the-all-new-outputsink-feature-aka-reverse-prime/129828>
233
234 Note that this option only has any effect if the "nvidia" driver is specified
235 in {option}`services.xserver.videoDrivers`, and it should preferably
236 be the only driver there.
237
238 If this is enabled, then the bus IDs of the NVIDIA and Intel/AMD GPUs have to
239 be specified ({option}`hardware.nvidia.prime.nvidiaBusId` and
240 {option}`hardware.nvidia.prime.intelBusId` or
241 {option}`hardware.nvidia.prime.amdgpuBusId`).
242
243 If you enable this, you may want to also enable kernel modesetting for the
244 NVIDIA driver ({option}`hardware.nvidia.modesetting.enable`) in order
245 to prevent tearing.
246
247 Note that this configuration will only be successful when a display manager
248 for which the {option}`services.xserver.displayManager.setupCommands`
249 option is supported is used
250 '';
251
252 prime.reverseSync.setupCommands.enable =
253 (lib.mkEnableOption ''
254 configure the display manager to be able to use the outputs
255 attached to the NVIDIA GPU.
256 Disable in order to configure the NVIDIA GPU outputs manually using xrandr.
257 Note that this configuration will only be successful when a display manager
258 for which the {option}`services.xserver.displayManager.setupCommands`
259 option is supported is used
260 '')
261 // {
262 default = true;
263 };
264
265 nvidiaSettings =
266 (lib.mkEnableOption ''
267 nvidia-settings, NVIDIA's GUI configuration tool
268 '')
269 // {
270 default = true;
271 };
272
273 nvidiaPersistenced = lib.mkEnableOption ''
274 nvidia-persistenced a update for NVIDIA GPU headless mode, i.e.
275 It ensures all GPUs stay awake even during headless mode
276 '';
277
278 forceFullCompositionPipeline = lib.mkEnableOption ''
279 forcefully the full composition pipeline.
280 This sometimes fixes screen tearing issues.
281 This has been reported to reduce the performance of some OpenGL applications and may produce issues in WebGL.
282 It also drastically increases the time the driver needs to clock down after load
283 '';
284
285 package = lib.mkOption {
286 default =
287 config.boot.kernelPackages.nvidiaPackages."${if cfg.datacenter.enable then "dc" else "stable"}";
288 defaultText = lib.literalExpression ''
289 config.boot.kernelPackages.nvidiaPackages."\$\{if cfg.datacenter.enable then "dc" else "stable"}"
290 '';
291 example = "config.boot.kernelPackages.nvidiaPackages.legacy_470";
292 description = ''
293 The NVIDIA driver package to use.
294 '';
295 };
296
297 open = lib.mkOption {
298 example = true;
299 description = "Whether to enable the open source NVIDIA kernel module.";
300 type = lib.types.nullOr lib.types.bool;
301 default = if lib.versionOlder nvidia_x11.version "560" then false else null;
302 defaultText = lib.literalExpression ''
303 if lib.versionOlder config.hardware.nvidia.package.version "560" then false else null
304 '';
305 };
306
307 gsp.enable =
308 lib.mkEnableOption ''
309 the GPU System Processor (GSP) on the video card
310 ''
311 // {
312 default = useOpenModules || lib.versionAtLeast nvidia_x11.version "555";
313 defaultText = lib.literalExpression ''
314 config.hardware.nvidia.open == true || lib.versionAtLeast config.hardware.nvidia.package.version "555"
315 '';
316 };
317
318 videoAcceleration =
319 (lib.mkEnableOption ''
320 Whether video acceleration (VA-API) should be enabled.
321 '')
322 // {
323 default = true;
324 };
325 };
326 };
327
328 config =
329 let
330 igpuDriver = if pCfg.intelBusId != "" then "modesetting" else "amdgpu";
331 igpuBusId = if pCfg.intelBusId != "" then pCfg.intelBusId else pCfg.amdgpuBusId;
332 in
333 lib.mkIf cfg.enabled (
334 lib.mkMerge [
335 # Common
336 ({
337 assertions = [
338 {
339 assertion = !(nvidiaEnabled && cfg.datacenter.enable);
340 message = "You cannot configure both X11 and Data Center drivers at the same time.";
341 }
342 {
343 assertion = cfg.open != null || cfg.datacenter.enable;
344 message = ''
345 You must configure `hardware.nvidia.open` on NVIDIA driver versions >= 560.
346 It is suggested to use the open source kernel modules on Turing or later GPUs (RTX series, GTX 16xx), and the closed source modules otherwise.
347 '';
348 }
349 ];
350 boot = {
351 blacklistedKernelModules = [
352 "nouveau"
353 "nvidiafb"
354 ];
355
356 # Don't add `nvidia-uvm` to `kernelModules`, because we want
357 # `nvidia-uvm` be loaded only after the GPU device is available, i.e. after `udev` rules
358 # for `nvidia` kernel module are applied.
359 # This matters on Azure GPU instances: https://github.com/NixOS/nixpkgs/pull/267335
360 #
361 # Instead, we use `softdep` to lazily load `nvidia-uvm` kernel module
362 # after `nvidia` kernel module is loaded and `udev` rules are applied.
363 extraModprobeConfig = ''
364 softdep nvidia post: nvidia-uvm
365 '';
366
367 # Exception is the open-source kernel module failing to load nvidia-uvm using softdep
368 # for unknown reasons.
369 # It affects CUDA: https://github.com/NixOS/nixpkgs/issues/334180
370 # Previously nvidia-uvm was explicitly loaded only when xserver was enabled:
371 # https://github.com/NixOS/nixpkgs/pull/334340/commits/4548c392862115359e50860bcf658cfa8715bde9
372 # We are now loading the module eagerly for all users of the open driver (including headless).
373 kernelModules = lib.optionals useOpenModules [ "nvidia_uvm" ];
374 };
375 systemd.tmpfiles.rules = lib.mkIf config.virtualisation.docker.enableNvidia [
376 "L+ /run/nvidia-docker/bin - - - - ${nvidia_x11.bin}/origBin"
377 ];
378 services.udev.extraRules = ''
379 # Create /dev/nvidia-uvm when the nvidia-uvm module is loaded.
380 KERNEL=="nvidia", RUN+="${pkgs.runtimeShell} -c 'mknod -m 666 /dev/nvidiactl c 195 255'"
381 KERNEL=="nvidia", RUN+="${pkgs.runtimeShell} -c 'for i in $$(cat /proc/driver/nvidia/gpus/*/information | grep Minor | cut -d \ -f 4); do mknod -m 666 /dev/nvidia$${i} c 195 $${i}; done'"
382 KERNEL=="nvidia_modeset", RUN+="${pkgs.runtimeShell} -c 'mknod -m 666 /dev/nvidia-modeset c 195 254'"
383 KERNEL=="nvidia_uvm", RUN+="${pkgs.runtimeShell} -c 'mknod -m 666 /dev/nvidia-uvm c $$(grep nvidia-uvm /proc/devices | cut -d \ -f 1) 0'"
384 KERNEL=="nvidia_uvm", RUN+="${pkgs.runtimeShell} -c 'mknod -m 666 /dev/nvidia-uvm-tools c $$(grep nvidia-uvm /proc/devices | cut -d \ -f 1) 1'"
385 '';
386 hardware.graphics = {
387 extraPackages = [ nvidia_x11.out ];
388 extraPackages32 = [ nvidia_x11.lib32 ];
389 };
390 environment.systemPackages = [ nvidia_x11.bin ];
391 })
392
393 # X11
394 (lib.mkIf nvidiaEnabled {
395 assertions = [
396 {
397 assertion = primeEnabled -> pCfg.intelBusId == "" || pCfg.amdgpuBusId == "";
398 message = "You cannot configure both an Intel iGPU and an AMD APU. Pick the one corresponding to your processor.";
399 }
400
401 {
402 assertion = offloadCfg.enableOffloadCmd -> offloadCfg.enable || reverseSyncCfg.enable;
403 message = "Offload command requires offloading or reverse prime sync to be enabled.";
404 }
405
406 {
407 assertion =
408 primeEnabled -> pCfg.nvidiaBusId != "" && (pCfg.intelBusId != "" || pCfg.amdgpuBusId != "");
409 message = "When NVIDIA PRIME is enabled, the GPU bus IDs must be configured.";
410 }
411
412 {
413 assertion = offloadCfg.enable -> lib.versionAtLeast nvidia_x11.version "435.21";
414 message = "NVIDIA PRIME render offload is currently only supported on versions >= 435.21.";
415 }
416
417 {
418 assertion =
419 (reverseSyncCfg.enable && pCfg.amdgpuBusId != "") -> lib.versionAtLeast nvidia_x11.version "470.0";
420 message = "NVIDIA PRIME render offload for AMD APUs is currently only supported on versions >= 470 beta.";
421 }
422
423 {
424 assertion = !(syncCfg.enable && offloadCfg.enable);
425 message = "PRIME Sync and Offload cannot be both enabled";
426 }
427
428 {
429 assertion = !(syncCfg.enable && reverseSyncCfg.enable);
430 message = "PRIME Sync and PRIME Reverse Sync cannot be both enabled";
431 }
432
433 {
434 assertion = !(syncCfg.enable && cfg.powerManagement.finegrained);
435 message = "Sync precludes powering down the NVIDIA GPU.";
436 }
437
438 {
439 assertion = cfg.powerManagement.finegrained -> offloadCfg.enable;
440 message = "Fine-grained power management requires offload to be enabled.";
441 }
442
443 {
444 assertion = cfg.powerManagement.enable -> lib.versionAtLeast nvidia_x11.version "430.09";
445 message = "Required files for driver based power management only exist on versions >= 430.09.";
446 }
447
448 {
449 assertion = cfg.gsp.enable -> (cfg.package ? firmware);
450 message = "This version of NVIDIA driver does not provide a GSP firmware.";
451 }
452
453 {
454 assertion = useOpenModules -> (cfg.package ? open);
455 message = "This version of NVIDIA driver does not provide a corresponding opensource kernel driver.";
456 }
457
458 {
459 assertion = useOpenModules -> cfg.gsp.enable;
460 message = "The GSP cannot be disabled when using the opensource kernel driver.";
461 }
462
463 {
464 assertion = cfg.dynamicBoost.enable -> lib.versionAtLeast nvidia_x11.version "510.39.01";
465 message = "NVIDIA's Dynamic Boost feature only exists on versions >= 510.39.01";
466 }
467 ];
468
469 # If Optimus/PRIME is enabled, we:
470 # - Specify the configured NVIDIA GPU bus ID in the Device section for the
471 # "nvidia" driver.
472 # - Add the AllowEmptyInitialConfiguration option to the Screen section for the
473 # "nvidia" driver, in order to allow the X server to start without any outputs.
474 # - Add a separate Device section for the Intel GPU, using the "modesetting"
475 # driver and with the configured BusID.
476 # - OR add a separate Device section for the AMD APU, using the "amdgpu"
477 # driver and with the configures BusID.
478 # - Reference that Device section from the ServerLayout section as an inactive
479 # device.
480 # - Configure the display manager to run specific `xrandr` commands which will
481 # configure/enable displays connected to the Intel iGPU / AMD APU.
482
483 # reverse sync implies offloading
484 hardware.nvidia.prime.offload.enable = lib.mkDefault reverseSyncCfg.enable;
485
486 services.xserver.drivers =
487 lib.optional primeEnabled {
488 name = igpuDriver;
489 display = offloadCfg.enable;
490 modules = lib.optional (igpuDriver == "amdgpu") pkgs.xorg.xf86videoamdgpu;
491 deviceSection = ''
492 BusID "${igpuBusId}"
493 ''
494 + lib.optionalString (syncCfg.enable && igpuDriver != "amdgpu") ''
495 Option "AccelMethod" "none"
496 '';
497 }
498 ++ lib.singleton {
499 name = "nvidia";
500 modules = [ nvidia_x11.bin ];
501 display = !offloadCfg.enable;
502 deviceSection = ''
503 Option "SidebandSocketPath" "/run/nvidia-xdriver/"
504 ''
505 + lib.optionalString primeEnabled ''
506 BusID "${pCfg.nvidiaBusId}"
507 ''
508 + lib.optionalString pCfg.allowExternalGpu ''
509 Option "AllowExternalGpus"
510 '';
511 screenSection = ''
512 Option "RandRRotation" "on"
513 ''
514 + lib.optionalString syncCfg.enable ''
515 Option "AllowEmptyInitialConfiguration"
516 ''
517 + lib.optionalString cfg.forceFullCompositionPipeline ''
518 Option "metamodes" "nvidia-auto-select +0+0 {ForceFullCompositionPipeline=On}"
519 Option "AllowIndirectGLXProtocol" "off"
520 Option "TripleBuffer" "on"
521 '';
522 };
523
524 services.xserver.serverLayoutSection =
525 lib.optionalString syncCfg.enable ''
526 Inactive "Device-${igpuDriver}[0]"
527 ''
528 + lib.optionalString reverseSyncCfg.enable ''
529 Inactive "Device-nvidia[0]"
530 ''
531 + lib.optionalString offloadCfg.enable ''
532 Option "AllowNVIDIAGPUScreens"
533 '';
534
535 services.xserver.displayManager.setupCommands =
536 let
537 gpuProviderName =
538 if igpuDriver == "amdgpu" then
539 # find the name of the provider if amdgpu
540 "`${lib.getExe pkgs.xorg.xrandr} --listproviders | ${lib.getExe pkgs.gnugrep} -i AMD | ${lib.getExe pkgs.gnused} -n 's/^.*name://p'`"
541 else
542 igpuDriver;
543 providerCmdParams =
544 if syncCfg.enable then "\"${gpuProviderName}\" NVIDIA-0" else "NVIDIA-G0 \"${gpuProviderName}\"";
545 in
546 lib.optionalString
547 (syncCfg.enable || (reverseSyncCfg.enable && reverseSyncCfg.setupCommands.enable))
548 ''
549 # Added by nvidia configuration module for Optimus/PRIME.
550 ${lib.getExe pkgs.xorg.xrandr} --setprovideroutputsource ${providerCmdParams}
551 ${lib.getExe pkgs.xorg.xrandr} --auto
552 '';
553
554 environment.etc = {
555 "nvidia/nvidia-application-profiles-rc" = lib.mkIf nvidia_x11.useProfiles {
556 source = "${nvidia_x11.bin}/share/nvidia/nvidia-application-profiles-rc";
557 };
558
559 # 'nvidia_x11' installs it's files to /run/opengl-driver/...
560 "egl/egl_external_platform.d".source = "/run/opengl-driver/share/egl/egl_external_platform.d/";
561 };
562
563 hardware.graphics.extraPackages = lib.optional cfg.videoAcceleration pkgs.nvidia-vaapi-driver;
564
565 environment.systemPackages =
566 lib.optional cfg.nvidiaSettings nvidia_x11.settings
567 ++ lib.optional cfg.nvidiaPersistenced nvidia_x11.persistenced
568 ++ lib.optional offloadCfg.enableOffloadCmd (
569 pkgs.writeShellScriptBin cfg.prime.offload.offloadCmdMainProgram ''
570 export __NV_PRIME_RENDER_OFFLOAD=1
571 export __NV_PRIME_RENDER_OFFLOAD_PROVIDER=NVIDIA-G0
572 export __GLX_VENDOR_LIBRARY_NAME=nvidia
573 export __VK_LAYER_NV_optimus=NVIDIA_only
574 exec "$@"
575 ''
576 );
577
578 systemd.packages = lib.optional cfg.powerManagement.enable nvidia_x11.out;
579
580 systemd.services =
581 let
582 nvidiaService = state: {
583 description = "NVIDIA system ${state} actions";
584 path = [ pkgs.kbd ];
585 serviceConfig = {
586 Type = "oneshot";
587 ExecStart = "${nvidia_x11.out}/bin/nvidia-sleep.sh '${state}'";
588 };
589 before = [ "systemd-${state}.service" ];
590 requiredBy = [ "systemd-${state}.service" ];
591 };
592 in
593 lib.mkMerge [
594 (lib.mkIf cfg.powerManagement.enable {
595 nvidia-suspend = nvidiaService "suspend";
596 nvidia-hibernate = nvidiaService "hibernate";
597 nvidia-resume = (nvidiaService "resume") // {
598 before = [ ];
599 after = [
600 "systemd-suspend.service"
601 "systemd-hibernate.service"
602 ];
603 requiredBy = [
604 "systemd-suspend.service"
605 "systemd-hibernate.service"
606 ];
607 };
608 })
609 (lib.mkIf cfg.nvidiaPersistenced {
610 "nvidia-persistenced" = {
611 description = "NVIDIA Persistence Daemon";
612 wantedBy = [ "multi-user.target" ];
613 serviceConfig = {
614 Type = "forking";
615 Restart = "always";
616 PIDFile = "/var/run/nvidia-persistenced/nvidia-persistenced.pid";
617 ExecStart = "${lib.getExe nvidia_x11.persistenced} --verbose";
618 ExecStopPost = "${pkgs.coreutils}/bin/rm -rf /var/run/nvidia-persistenced";
619 };
620 };
621 })
622 (lib.mkIf cfg.dynamicBoost.enable {
623 "nvidia-powerd" = {
624 description = "nvidia-powerd service";
625 path = [
626 pkgs.util-linux # nvidia-powerd wants lscpu
627 ];
628 wantedBy = [ "multi-user.target" ];
629 serviceConfig = {
630 Type = "dbus";
631 BusName = "nvidia.powerd.server";
632 ExecStart = "${nvidia_x11.bin}/bin/nvidia-powerd";
633 };
634 };
635 })
636 ];
637
638 services.acpid.enable = true;
639
640 services.dbus.packages = lib.optional cfg.dynamicBoost.enable nvidia_x11.bin;
641
642 hardware.firmware = lib.optional cfg.gsp.enable nvidia_x11.firmware;
643
644 systemd.tmpfiles.rules = [
645 # Remove the following log message:
646 # (WW) NVIDIA: Failed to bind sideband socket to
647 # (WW) NVIDIA: '/var/run/nvidia-xdriver-b4f69129' Permission denied
648 #
649 # https://bbs.archlinux.org/viewtopic.php?pid=1909115#p1909115
650 "d /run/nvidia-xdriver 0770 root users"
651 ]
652 ++
653 lib.optional (nvidia_x11.persistenced != null && config.virtualisation.docker.enableNvidia)
654 "L+ /run/nvidia-docker/extras/bin/nvidia-persistenced - - - - ${nvidia_x11.persistenced}/origBin/nvidia-persistenced";
655
656 boot = {
657 extraModulePackages = if useOpenModules then [ nvidia_x11.open ] else [ nvidia_x11.bin ];
658 # nvidia-uvm is required by CUDA applications.
659 kernelModules = lib.optionals config.services.xserver.enable [
660 "nvidia"
661 "nvidia_modeset"
662 "nvidia_drm"
663 ];
664
665 # If requested enable modesetting via kernel parameters.
666 kernelParams =
667 lib.optional (offloadCfg.enable || cfg.modesetting.enable) "nvidia-drm.modeset=1"
668 ++ lib.optional (
669 (offloadCfg.enable || cfg.modesetting.enable) && lib.versionAtLeast nvidia_x11.version "545"
670 ) "nvidia-drm.fbdev=1"
671 ++ lib.optional cfg.powerManagement.enable "nvidia.NVreg_PreserveVideoMemoryAllocations=1"
672 ++ lib.optional useOpenModules "nvidia.NVreg_OpenRmEnableUnsupportedGpus=1"
673 ++ lib.optional (config.boot.kernelPackages.kernel.kernelAtLeast "6.2" && !ibtSupport) "ibt=off";
674
675 # enable finegrained power management
676 extraModprobeConfig = lib.optionalString cfg.powerManagement.finegrained ''
677 options nvidia "NVreg_DynamicPowerManagement=0x02"
678 '';
679 };
680 services.udev.extraRules = lib.optionalString cfg.powerManagement.finegrained (
681 lib.optionalString (lib.versionOlder config.boot.kernelPackages.kernel.version "5.5") ''
682 # Remove NVIDIA USB xHCI Host Controller devices, if present
683 ACTION=="add", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x0c0330", ATTR{remove}="1"
684
685 # Remove NVIDIA USB Type-C UCSI devices, if present
686 ACTION=="add", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x0c8000", ATTR{remove}="1"
687
688 # Remove NVIDIA Audio devices, if present
689 ACTION=="add", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x040300", ATTR{remove}="1"
690 ''
691 + ''
692 # Enable runtime PM for NVIDIA VGA/3D controller devices on driver bind
693 ACTION=="bind", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x030000", TEST=="power/control", ATTR{power/control}="auto"
694 ACTION=="bind", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x030200", TEST=="power/control", ATTR{power/control}="auto"
695
696 # Disable runtime PM for NVIDIA VGA/3D controller devices on driver unbind
697 ACTION=="unbind", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x030000", TEST=="power/control", ATTR{power/control}="on"
698 ACTION=="unbind", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x030200", TEST=="power/control", ATTR{power/control}="on"
699 ''
700 );
701 })
702 # Data Center
703 (lib.mkIf (cfg.datacenter.enable) {
704 boot.extraModulePackages = [ nvidia_x11.bin ];
705
706 systemd = {
707 tmpfiles.rules =
708 lib.optional (nvidia_x11.persistenced != null && config.virtualisation.docker.enableNvidia)
709 "L+ /run/nvidia-docker/extras/bin/nvidia-persistenced - - - - ${nvidia_x11.persistenced}/origBin/nvidia-persistenced";
710
711 services = lib.mkMerge [
712 ({
713 nvidia-fabricmanager = {
714 enable = true;
715 description = "Start NVIDIA NVLink Management";
716 wantedBy = [ "multi-user.target" ];
717 unitConfig.After = [ "network-online.target" ];
718 unitConfig.Requires = [ "network-online.target" ];
719 serviceConfig = {
720 Type = "forking";
721 TimeoutStartSec = 240;
722 ExecStart =
723 let
724 # Since these rely on the `nvidia_x11.fabricmanager` derivation, they're
725 # unsuitable to be mentioned in the configuration defaults, but they _can_
726 # be overridden in `cfg.datacenter.settings` if needed.
727 fabricManagerConfDefaults = {
728 TOPOLOGY_FILE_PATH = "${nvidia_x11.fabricmanager}/share/nvidia-fabricmanager/nvidia/nvswitch";
729 DATABASE_PATH = "${nvidia_x11.fabricmanager}/share/nvidia-fabricmanager/nvidia/nvswitch";
730 };
731 nv-fab-conf = settingsFormat.generate "fabricmanager.conf" (
732 fabricManagerConfDefaults // cfg.datacenter.settings
733 );
734 in
735 "${lib.getExe nvidia_x11.fabricmanager} -c ${nv-fab-conf}";
736 LimitCORE = "infinity";
737 };
738 };
739 })
740 (lib.mkIf cfg.nvidiaPersistenced {
741 "nvidia-persistenced" = {
742 description = "NVIDIA Persistence Daemon";
743 wantedBy = [ "multi-user.target" ];
744 serviceConfig = {
745 Type = "forking";
746 Restart = "always";
747 PIDFile = "/var/run/nvidia-persistenced/nvidia-persistenced.pid";
748 ExecStart = "${lib.getExe nvidia_x11.persistenced} --verbose";
749 ExecStopPost = "${pkgs.coreutils}/bin/rm -rf /var/run/nvidia-persistenced";
750 };
751 };
752 })
753 ];
754 };
755
756 environment.systemPackages =
757 lib.optional cfg.datacenter.enable nvidia_x11.fabricmanager
758 ++ lib.optional cfg.nvidiaPersistenced nvidia_x11.persistenced;
759 })
760 ]
761 );
762}