pyrox.dev
1{ pkgs, lib, ... }:
2let
3 testCDIScript = pkgs.writeShellScriptBin "test-cdi" ''
4 die() {
5 echo "$1"
6 exit 1
7 }
8
9 check_file_referential_integrity() {
10 echo "checking $1 referential integrity"
11 ( ${pkgs.glibc.bin}/bin/ldd "$1" | ${lib.getExe pkgs.gnugrep} "not found" &> /dev/null ) && return 1
12 return 0
13 }
14
15 check_directory_referential_integrity() {
16 ${lib.getExe pkgs.findutils} "$1" -type f -print0 | while read -d $'\0' file; do
17 if [[ $(${lib.getExe pkgs.file} "$file" | ${lib.getExe pkgs.gnugrep} ELF) ]]; then
18 check_file_referential_integrity "$file" || exit 1
19 else
20 echo "skipping $file: not an ELF file"
21 fi
22 done
23 }
24
25 check_directory_referential_integrity "/usr/bin" || exit 1
26 check_directory_referential_integrity "${pkgs.addDriverRunpath.driverLink}" || exit 1
27 check_directory_referential_integrity "/usr/local/nvidia" || exit 1
28 '';
29 testContainerImage = pkgs.dockerTools.buildImage {
30 name = "cdi-test";
31 tag = "latest";
32 config = {
33 Cmd = [ (lib.getExe testCDIScript) ];
34 };
35 copyToRoot = with pkgs.dockerTools; [
36 usrBinEnv
37 binSh
38 ];
39 };
40 emptyCDISpec = ''
41 {
42 "cdiVersion": "0.5.0",
43 "kind": "nvidia.com/gpu",
44 "devices": [
45 {
46 "name": "all",
47 "containerEdits": {
48 "deviceNodes": [
49 {
50 "path": "/dev/urandom"
51 }
52 ],
53 "hooks": [],
54 "mounts": []
55 }
56 }
57 ],
58 "containerEdits": {
59 "deviceNodes": [],
60 "hooks": [],
61 "mounts": []
62 }
63 }
64 '';
65 nvidia-container-toolkit = {
66 enable = true;
67 package = pkgs.stdenv.mkDerivation {
68 pname = "nvidia-ctk-dummy";
69 version = "1.0.0";
70 dontUnpack = true;
71 dontBuild = true;
72
73 inherit emptyCDISpec;
74 passAsFile = [ "emptyCDISpec" ];
75
76 installPhase = ''
77 mkdir -p $out/bin $out/share/nvidia-container-toolkit
78 cp "$emptyCDISpecPath" "$out/share/nvidia-container-toolkit/spec.json"
79 echo -n "$emptyCDISpec" > "$out/bin/nvidia-ctk";
80 cat << EOF > "$out/bin/nvidia-ctk"
81 #!${pkgs.runtimeShell}
82 cat "$out/share/nvidia-container-toolkit/spec.json"
83 EOF
84 chmod +x $out/bin/nvidia-ctk
85 '';
86 meta.mainProgram = "nvidia-ctk";
87 };
88 suppressNvidiaDriverAssertion = true;
89 };
90in
91{
92 name = "nvidia-container-toolkit";
93 meta = with lib.maintainers; {
94 maintainers = [
95 ereslibre
96 christoph-heiss
97 ];
98 };
99 defaults =
100 { config, ... }:
101 {
102 environment.systemPackages = with pkgs; [ jq ];
103 virtualisation.diskSize = lib.mkDefault 10240;
104 virtualisation.containers = {
105 containersConf.settings.engine.cdi_spec_dirs = [ "/var/run/cdi" ];
106 enable = lib.mkDefault true;
107 };
108 hardware = {
109 inherit nvidia-container-toolkit;
110 nvidia = {
111 open = true;
112 package = config.boot.kernelPackages.nvidiaPackages.stable.open;
113 };
114 graphics.enable = lib.mkDefault true;
115 };
116 };
117 nodes = {
118 no-gpus = {
119 virtualisation.containers.enable = false;
120 };
121
122 one-gpu =
123 { pkgs, ... }:
124 {
125 environment.systemPackages = with pkgs; [ podman ];
126 hardware.graphics.enable = true;
127 };
128
129 one-gpu-invalid-host-paths = {
130 hardware.nvidia-container-toolkit.mounts = [
131 {
132 hostPath = "/non-existant-path";
133 containerPath = "/some/path";
134 }
135 ];
136 };
137 };
138 testScript = ''
139 start_all()
140
141 with subtest("Generate an empty CDI spec for a machine with no Nvidia GPUs"):
142 no_gpus.wait_for_unit("nvidia-container-toolkit-cdi-generator.service")
143 no_gpus.succeed("cat /var/run/cdi/nvidia-container-toolkit.json | jq")
144
145 with subtest("Podman loads the generated CDI spec for a machine with an Nvidia GPU"):
146 one_gpu.wait_for_unit("nvidia-container-toolkit-cdi-generator.service")
147 one_gpu.succeed("cat /var/run/cdi/nvidia-container-toolkit.json | jq")
148 one_gpu.succeed("podman load < ${testContainerImage}")
149 one_gpu.succeed("podman run --pull=never --device=nvidia.com/gpu=all -v /run/opengl-driver:/run/opengl-driver:ro cdi-test:latest")
150
151 # Issue: https://github.com/NixOS/nixpkgs/issues/319201
152 with subtest("The generated CDI spec skips specified non-existant paths in the host"):
153 one_gpu_invalid_host_paths.wait_for_unit("nvidia-container-toolkit-cdi-generator.service")
154 one_gpu_invalid_host_paths.fail("grep 'non-existant-path' /var/run/cdi/nvidia-container-toolkit.json")
155 '';
156}