pyrox.dev
1{
2 callPackage,
3 fetchgit,
4 fontconfig,
5 git,
6 lib,
7 makeWrapper,
8 python3,
9 runCommand,
10 writeTextFile,
11
12 # Artifacts dependencies
13 fetchurl,
14 gcc,
15 glibc,
16 pkgs,
17 stdenv,
18
19 julia,
20
21 # Special registry which is equal to JuliaRegistries/General, but every Versions.toml
22 # entry is augmented with a Nix sha256 hash
23 augmentedRegistry ? callPackage ./registry.nix { },
24
25 # Other overridable arguments
26 extraLibs ? [ ],
27 juliaCpuTarget ? null,
28 makeTransitiveDependenciesImportable ? false, # Used to support symbol indexing
29 makeWrapperArgs ? "",
30 packageOverrides ? { },
31 precompile ? true,
32 setDefaultDepot ? true,
33}:
34
35packageNames:
36
37let
38 util = callPackage ./util.nix { };
39
40 # Some Julia packages require access to Python. Provide a Nixpkgs version so it
41 # doesn't try to install its own.
42 pythonToUse =
43 let
44 extraPythonPackages = (
45 (callPackage ./extra-python-packages.nix { inherit python3; }).getExtraPythonPackages packageNames
46 );
47 in
48 (
49 if extraPythonPackages == [ ] then
50 python3
51 else
52 util.addPackagesToPython python3 (map (pkg: lib.getAttr pkg python3.pkgs) extraPythonPackages)
53 );
54
55 # Start by wrapping Julia so it has access to Python and any other extra libs.
56 # Also, prevent various packages (CondaPkg.jl, PythonCall.jl) from trying to do network calls.
57 juliaWrapped =
58 runCommand "julia-${julia.version}-wrapped"
59 {
60 nativeBuildInputs = [ makeWrapper ];
61 inherit makeWrapperArgs;
62 }
63 ''
64 mkdir -p $out/bin
65 makeWrapper ${julia}/bin/julia $out/bin/julia \
66 --suffix LD_LIBRARY_PATH : "${lib.makeLibraryPath extraLibs}" \
67 --set FONTCONFIG_FILE ${fontconfig.out}/etc/fonts/fonts.conf \
68 --set PYTHONHOME "${pythonToUse}" \
69 --prefix PYTHONPATH : "${pythonToUse}/${pythonToUse.sitePackages}" \
70 --set PYTHON ${pythonToUse}/bin/python $makeWrapperArgs \
71 --set JULIA_CONDAPKG_OFFLINE yes \
72 --set JULIA_CONDAPKG_BACKEND Null \
73 --set JULIA_PYTHONCALL_EXE "@PyCall"
74 '';
75
76 # If our closure ends up with certain packages, add others.
77 packageImplications = {
78 # Because we want to put PythonCall in PyCall mode so it doesn't try to download
79 # Python packages
80 PythonCall = [ "PyCall" ];
81 };
82
83 # Invoke Julia resolution logic to determine the full dependency closure. Also
84 # gather information on the Julia standard libraries, which we'll need to
85 # generate a Manifest.toml.
86 packageOverridesRepoified = lib.mapAttrs util.repoifySimple packageOverrides;
87 closureYaml = callPackage ./package-closure.nix {
88 inherit
89 augmentedRegistry
90 julia
91 packageNames
92 packageImplications
93 ;
94 packageOverrides = packageOverridesRepoified;
95 };
96 stdlibInfos = callPackage ./stdlib-infos.nix {
97 inherit julia;
98 };
99
100 # Generate a Nix file consisting of a map from dependency UUID --> package info with fetchgit call:
101 # {
102 # "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" = {
103 # src = fetchgit {...};
104 # name = "...";
105 # version = "...";
106 # treehash = "...";
107 # };
108 # ...
109 # }
110 dependencies =
111 runCommand "julia-sources.nix"
112 {
113 buildInputs = [
114 (python3.withPackages (
115 ps: with ps; [
116 toml
117 pyyaml
118 ]
119 ))
120 git
121 ];
122 }
123 ''
124 python ${./python}/sources_nix.py \
125 "${augmentedRegistry}" \
126 '${lib.generators.toJSON { } packageOverridesRepoified}' \
127 "${closureYaml}" \
128 "$out"
129 '';
130
131 # Import the Nix file from the previous step (IFD) and turn each dependency repo into
132 # a dummy Git repository, as Julia expects. Format the results as a YAML map from
133 # dependency UUID -> Nix store location:
134 # {
135 # "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3":"/nix/store/...-NaNMath.jl-0877504",
136 # ...
137 # }
138 # This is also the point where we apply the packageOverrides.
139 dependencyUuidToInfo = import dependencies { inherit fetchgit; };
140 fillInOverrideSrc =
141 uuid: info:
142 if lib.hasAttr info.name packageOverrides then
143 (info // { src = lib.getAttr info.name packageOverrides; })
144 else
145 info;
146 dependencyUuidToRepo = lib.mapAttrs util.repoifyInfo (
147 lib.mapAttrs fillInOverrideSrc dependencyUuidToInfo
148 );
149 dependencyUuidToRepoYaml = writeTextFile {
150 name = "dependency-uuid-to-repo.yml";
151 text = lib.generators.toYAML { } dependencyUuidToRepo;
152 };
153
154 # Given the augmented registry, closure info yaml, and dependency path yaml, construct a complete
155 # Julia registry containing all the necessary packages
156 dependencyUuidToInfoYaml = writeTextFile {
157 name = "dependency-uuid-to-info.yml";
158 text = lib.generators.toYAML { } dependencyUuidToInfo;
159 };
160 fillInOverrideSrc' =
161 uuid: info:
162 if lib.hasAttr info.name packageOverridesRepoified then
163 (info // { src = lib.getAttr info.name packageOverridesRepoified; })
164 else
165 info;
166 overridesOnly = lib.mapAttrs fillInOverrideSrc' (
167 lib.filterAttrs (uuid: info: info.src == null) dependencyUuidToInfo
168 );
169 minimalRegistry =
170 runCommand "minimal-julia-registry"
171 {
172 buildInputs = [
173 (python3.withPackages (
174 ps: with ps; [
175 toml
176 pyyaml
177 ]
178 ))
179 git
180 ];
181 }
182 ''
183 python ${./python}/minimal_registry.py \
184 "${augmentedRegistry}" \
185 "${closureYaml}" \
186 '${lib.generators.toJSON { } overridesOnly}' \
187 "${dependencyUuidToRepoYaml}" \
188 "$out"
189 '';
190 project =
191 runCommand "julia-project"
192 {
193 buildInputs = [
194 (python3.withPackages (
195 ps: with ps; [
196 toml
197 pyyaml
198 ]
199 ))
200 git
201 ];
202 }
203 ''
204 python ${./python}/project.py \
205 "${closureYaml}" \
206 "${stdlibInfos}" \
207 '${lib.generators.toJSON { } overridesOnly}' \
208 "${dependencyUuidToRepoYaml}" \
209 "$out"
210 '';
211
212 # Next, deal with artifacts. Scan each artifacts file individually and generate a Nix file that
213 # produces the desired Overrides.toml.
214 artifactsNix =
215 runCommand "julia-artifacts.nix"
216 {
217 buildInputs = [
218 (python3.withPackages (
219 ps: with ps; [
220 toml
221 pyyaml
222 ]
223 ))
224 ];
225 }
226 ''
227 python ${./python}/extract_artifacts.py \
228 "${dependencyUuidToRepoYaml}" \
229 "${closureYaml}" \
230 "${juliaWrapped}/bin/julia" \
231 "${
232 if lib.versionAtLeast julia.version "1.7" then ./extract_artifacts.jl else ./extract_artifacts_16.jl
233 }" \
234 '${lib.generators.toJSON { } (import ./extra-libs.nix)}' \
235 '${lib.generators.toJSON { } (stdenv.hostPlatform.isDarwin)}' \
236 "$out"
237 '';
238
239 # Import the artifacts Nix to build Overrides.toml (IFD)
240 artifacts = import artifactsNix (
241 {
242 inherit
243 lib
244 fetchurl
245 pkgs
246 stdenv
247 ;
248 }
249 // lib.optionalAttrs (!stdenv.targetPlatform.isDarwin) {
250 inherit gcc glibc;
251 }
252 );
253 overridesJson = writeTextFile {
254 name = "Overrides.json";
255 text = lib.generators.toJSON { } artifacts;
256 };
257 overridesToml =
258 runCommand "Overrides.toml" { buildInputs = [ (python3.withPackages (ps: with ps; [ toml ])) ]; }
259 ''
260 python ${./python}/format_overrides.py \
261 "${overridesJson}" \
262 "$out"
263 '';
264
265 # Build a Julia project and depot under $out/project and $out/depot respectively
266 projectAndDepot = callPackage ./depot.nix {
267 inherit
268 closureYaml
269 extraLibs
270 juliaCpuTarget
271 overridesToml
272 packageImplications
273 precompile
274 ;
275 julia = juliaWrapped;
276 inherit project;
277 registry = minimalRegistry;
278 };
279
280in
281
282runCommand "julia-${julia.version}-env"
283 {
284 nativeBuildInputs = [ makeWrapper ];
285
286 passthru = {
287 inherit julia;
288 inherit juliaWrapped;
289 inherit (julia) pname version meta;
290
291 # Expose the steps we used along the way in case the user wants to use them, for example to build
292 # expressions and build them separately to avoid IFD.
293 inherit dependencies;
294 inherit closureYaml;
295 inherit dependencyUuidToInfoYaml;
296 inherit dependencyUuidToRepoYaml;
297 inherit minimalRegistry;
298 inherit artifactsNix;
299 inherit overridesJson;
300 inherit overridesToml;
301 inherit project;
302 inherit projectAndDepot;
303 inherit stdlibInfos;
304 };
305 }
306 (
307 ''
308 mkdir -p $out/bin
309 makeWrapper ${juliaWrapped}/bin/julia $out/bin/julia \
310 --suffix JULIA_DEPOT_PATH : "${projectAndDepot}/depot" \
311 --set-default JULIA_PROJECT "${projectAndDepot}/project" \
312 --set-default JULIA_LOAD_PATH '@:${projectAndDepot}/project/Project.toml:@v#.#:@stdlib'
313 ''
314 + lib.optionalString setDefaultDepot ''
315 sed -i '2 i\JULIA_DEPOT_PATH=''${JULIA_DEPOT_PATH-"$HOME/.julia"}' $out/bin/julia
316 ''
317 )