pyrox.dev
1# Generic builder.
2
3{
4 lib,
5 config,
6 python,
7 wrapPython,
8 unzip,
9 ensureNewerSourcesForZipFilesHook,
10 # Whether the derivation provides a Python module or not.
11 toPythonModule,
12 namePrefix,
13 nix-update-script,
14 setuptools,
15 pypaBuildHook,
16 pypaInstallHook,
17 pythonCatchConflictsHook,
18 pythonImportsCheckHook,
19 pythonNamespacesHook,
20 pythonOutputDistHook,
21 pythonRelaxDepsHook,
22 pythonRemoveBinBytecodeHook,
23 pythonRemoveTestsDirHook,
24 pythonRuntimeDepsCheckHook,
25 setuptoolsBuildHook,
26 wheelUnpackHook,
27 eggUnpackHook,
28 eggBuildHook,
29 eggInstallHook,
30}:
31
32let
33 inherit (builtins) unsafeGetAttrPos;
34 inherit (lib)
35 elem
36 extendDerivation
37 fixedWidthString
38 flip
39 getName
40 hasSuffix
41 head
42 isBool
43 max
44 optional
45 optionalAttrs
46 optionals
47 optionalString
48 removePrefix
49 splitString
50 stringLength
51 ;
52
53 getOptionalAttrs =
54 names: attrs: lib.getAttrs (lib.intersectLists names (lib.attrNames attrs)) attrs;
55
56 leftPadName =
57 name: against:
58 let
59 len = max (stringLength name) (stringLength against);
60 in
61 fixedWidthString len " " name;
62
63 isPythonModule =
64 drv:
65 # all pythonModules have the pythonModule attribute
66 (drv ? "pythonModule")
67 # Some pythonModules are turned in to a pythonApplication by setting the field to false
68 && (!isBool drv.pythonModule);
69
70 isMismatchedPython = drv: drv.pythonModule != python;
71
72 withDistOutput' = flip elem [
73 "pyproject"
74 "setuptools"
75 "wheel"
76 ];
77
78 isBootstrapInstallPackage' = flip elem [
79 "flit-core"
80 "installer"
81 ];
82
83 isBootstrapPackage' = flip elem (
84 [
85 "build"
86 "packaging"
87 "pyproject-hooks"
88 "wheel"
89 ]
90 ++ optionals (python.pythonOlder "3.11") [
91 "tomli"
92 ]
93 );
94
95 isSetuptoolsDependency' = flip elem [
96 "setuptools"
97 "wheel"
98 ];
99
100 cleanAttrs = flip removeAttrs [
101 "disabled"
102 "checkPhase"
103 "checkInputs"
104 "nativeCheckInputs"
105 "doCheck"
106 "doInstallCheck"
107 "pyproject"
108 "format"
109 "outputs"
110 "stdenv"
111 "dependencies"
112 "optional-dependencies"
113 "build-system"
114 ];
115
116in
117
118{
119 # Build-time dependencies for the package
120 nativeBuildInputs ? [ ],
121
122 # Run-time dependencies for the package
123 buildInputs ? [ ],
124
125 # Dependencies needed for running the checkPhase.
126 # These are added to buildInputs when doCheck = true.
127 checkInputs ? [ ],
128 nativeCheckInputs ? [ ],
129
130 # propagate build dependencies so in case we have A -> B -> C,
131 # C can import package A propagated by B
132 propagatedBuildInputs ? [ ],
133
134 # Python module dependencies.
135 # These are named after PEP-621.
136 dependencies ? [ ],
137 optional-dependencies ? { },
138
139 # Python PEP-517 build systems.
140 build-system ? [ ],
141
142 # DEPRECATED: use propagatedBuildInputs
143 pythonPath ? [ ],
144
145 # Enabled to detect some (native)BuildInputs mistakes
146 strictDeps ? true,
147
148 outputs ? [ "out" ],
149
150 # used to disable derivation, useful for specific python versions
151 disabled ? false,
152
153 # Raise an error if two packages are installed with the same name
154 # TODO: For cross we probably need a different PYTHONPATH, or not
155 # add the runtime deps until after buildPhase.
156 catchConflicts ? (python.stdenv.hostPlatform == python.stdenv.buildPlatform),
157
158 # Additional arguments to pass to the makeWrapper function, which wraps
159 # generated binaries.
160 makeWrapperArgs ? [ ],
161
162 # Skip wrapping of python programs altogether
163 dontWrapPythonPrograms ? false,
164
165 # Don't use Pip to install a wheel
166 # Note this is actually a variable for the pipInstallPhase in pip's setupHook.
167 # It's included here to prevent an infinite recursion.
168 dontUsePipInstall ? false,
169
170 # Skip setting the PYTHONNOUSERSITE environment variable in wrapped programs
171 permitUserSite ? false,
172
173 # Remove bytecode from bin folder.
174 # When a Python script has the extension `.py`, bytecode is generated
175 # Typically, executables in bin have no extension, so no bytecode is generated.
176 # However, some packages do provide executables with extensions, and thus bytecode is generated.
177 removeBinBytecode ? true,
178
179 # pyproject = true <-> format = "pyproject"
180 # pyproject = false <-> format = "other"
181 # https://github.com/NixOS/nixpkgs/issues/253154
182 pyproject ? null,
183
184 # Several package formats are supported.
185 # "setuptools" : Install a common setuptools/distutils based package. This builds a wheel.
186 # "wheel" : Install from a pre-compiled wheel.
187 # "pyproject": Install a package using a ``pyproject.toml`` file (PEP517). This builds a wheel.
188 # "egg": Install a package from an egg.
189 # "other" : Provide your own buildPhase and installPhase.
190 format ? null,
191
192 meta ? { },
193
194 doCheck ? true,
195
196 # Allow passing in a custom stdenv to buildPython*
197 stdenv ? python.stdenv,
198
199 ...
200}@attrs:
201
202let
203 # Keep extra attributes from `attrs`, e.g., `patchPhase', etc.
204 self = stdenv.mkDerivation (
205 finalAttrs:
206 let
207 format' =
208 assert (pyproject != null) -> (format == null);
209 if pyproject != null then
210 if pyproject then "pyproject" else "other"
211 else if format != null then
212 format
213 else
214 throw "${name} does not configure a `format`. To build with setuptools as before, set `pyproject = true` and `build-system = [ setuptools ]`.`";
215
216 withDistOutput = withDistOutput' format';
217
218 validatePythonMatches =
219 let
220 throwMismatch =
221 attrName: drv:
222 let
223 myName = "'${finalAttrs.name}'";
224 theirName = "'${drv.name}'";
225 optionalLocation =
226 let
227 pos = unsafeGetAttrPos (if attrs ? "pname" then "pname" else "name") attrs;
228 in
229 optionalString (pos != null) " at ${pos.file}:${toString pos.line}:${toString pos.column}";
230 in
231 throw ''
232 Python version mismatch in ${myName}:
233
234 The Python derivation ${myName} depends on a Python derivation
235 named ${theirName}, but the two derivations use different versions
236 of Python:
237
238 ${leftPadName myName theirName} uses ${python}
239 ${leftPadName theirName myName} uses ${toString drv.pythonModule}
240
241 Possible solutions:
242
243 * If ${theirName} is a Python library, change the reference to ${theirName}
244 in the ${attrName} of ${myName} to use a ${theirName} built from the same
245 version of Python
246
247 * If ${theirName} is used as a tool during the build, move the reference to
248 ${theirName} in ${myName} from ${attrName} to nativeBuildInputs
249
250 * If ${theirName} provides executables that are called at run time, pass its
251 bin path to makeWrapperArgs:
252
253 makeWrapperArgs = [ "--prefix PATH : ''${lib.makeBinPath [ ${getName drv} ] }" ];
254
255 ${optionalLocation}
256 '';
257
258 checkDrv =
259 attrName: drv:
260 if (isPythonModule drv) && (isMismatchedPython drv) then throwMismatch attrName drv else drv;
261
262 in
263 attrName: inputs: map (checkDrv attrName) inputs;
264
265 isBootstrapInstallPackage = isBootstrapInstallPackage' (attrs.pname or null);
266
267 isBootstrapPackage = isBootstrapInstallPackage || isBootstrapPackage' (attrs.pname or null);
268
269 isSetuptoolsDependency = isSetuptoolsDependency' (attrs.pname or null);
270
271 name = namePrefix + attrs.name or "${finalAttrs.pname}-${finalAttrs.version}";
272
273 in
274 (cleanAttrs attrs)
275 // {
276 inherit name;
277
278 inherit catchConflicts;
279
280 nativeBuildInputs = [
281 python
282 wrapPython
283 ensureNewerSourcesForZipFilesHook # move to wheel installer (pip) or builder (setuptools, flit, ...)?
284 pythonRemoveTestsDirHook
285 ]
286 ++ optionals (finalAttrs.catchConflicts && !isBootstrapPackage && !isSetuptoolsDependency) [
287 #
288 # 1. When building a package that is also part of the bootstrap chain, we
289 # must ignore conflicts after installation, because there will be one with
290 # the package in the bootstrap.
291 #
292 # 2. When a package is a dependency of setuptools, we must ignore conflicts
293 # because the hook that checks for conflicts uses setuptools.
294 #
295 pythonCatchConflictsHook
296 ]
297 ++ optionals (attrs ? pythonRelaxDeps || attrs ? pythonRemoveDeps) [
298 pythonRelaxDepsHook
299 ]
300 ++ optionals removeBinBytecode [
301 pythonRemoveBinBytecodeHook
302 ]
303 ++ optionals (hasSuffix "zip" (attrs.src.name or "")) [
304 unzip
305 ]
306 ++ optionals (format' == "setuptools") [
307 setuptoolsBuildHook
308 ]
309 ++ optionals (format' == "pyproject") [
310 (
311 if isBootstrapPackage then
312 pypaBuildHook.override {
313 inherit (python.pythonOnBuildForHost.pkgs.bootstrap) build;
314 wheel = null;
315 }
316 else
317 pypaBuildHook
318 )
319 (
320 if isBootstrapPackage then
321 pythonRuntimeDepsCheckHook.override {
322 inherit (python.pythonOnBuildForHost.pkgs.bootstrap) packaging;
323 }
324 else
325 pythonRuntimeDepsCheckHook
326 )
327 ]
328 ++ optionals (format' == "wheel") [
329 wheelUnpackHook
330 ]
331 ++ optionals (format' == "egg") [
332 eggUnpackHook
333 eggBuildHook
334 eggInstallHook
335 ]
336 ++ optionals (format' != "other") [
337 (
338 if isBootstrapInstallPackage then
339 pypaInstallHook.override {
340 inherit (python.pythonOnBuildForHost.pkgs.bootstrap) installer;
341 }
342 else
343 pypaInstallHook
344 )
345 ]
346 ++ optionals (stdenv.buildPlatform == stdenv.hostPlatform) [
347 # This is a test, however, it should be ran independent of the checkPhase and checkInputs
348 pythonImportsCheckHook
349 ]
350 ++ optionals (python.pythonAtLeast "3.3") [
351 # Optionally enforce PEP420 for python3
352 pythonNamespacesHook
353 ]
354 ++ optionals withDistOutput [
355 pythonOutputDistHook
356 ]
357 ++ nativeBuildInputs
358 ++ build-system;
359
360 buildInputs = validatePythonMatches "buildInputs" (buildInputs ++ pythonPath);
361
362 propagatedBuildInputs = validatePythonMatches "propagatedBuildInputs" (
363 propagatedBuildInputs
364 ++ dependencies
365 ++ [
366 # we propagate python even for packages transformed with 'toPythonApplication'
367 # this pollutes the PATH but avoids rebuilds
368 # see https://github.com/NixOS/nixpkgs/issues/170887 for more context
369 python
370 ]
371 );
372
373 inherit strictDeps;
374
375 LANG = "${if python.stdenv.hostPlatform.isDarwin then "en_US" else "C"}.UTF-8";
376
377 # Python packages don't have a checkPhase, only an installCheckPhase
378 doCheck = false;
379 doInstallCheck = attrs.doCheck or true;
380 nativeInstallCheckInputs = nativeCheckInputs ++ attrs.nativeInstallCheckInputs or [ ];
381 installCheckInputs = checkInputs ++ attrs.installCheckInputs or [ ];
382
383 inherit dontWrapPythonPrograms;
384
385 postFixup =
386 optionalString (!finalAttrs.dontWrapPythonPrograms) ''
387 wrapPythonPrograms
388 ''
389 + attrs.postFixup or "";
390
391 # Python packages built through cross-compilation are always for the host platform.
392 disallowedReferences = optionals (python.stdenv.hostPlatform != python.stdenv.buildPlatform) [
393 python.pythonOnBuildForHost
394 ];
395
396 outputs = outputs ++ optional withDistOutput "dist";
397
398 passthru = {
399 inherit disabled;
400 }
401 // {
402 updateScript = nix-update-script { };
403 }
404 // optionalAttrs (dependencies != [ ]) {
405 inherit dependencies;
406 }
407 // optionalAttrs (optional-dependencies != { }) {
408 inherit optional-dependencies;
409 }
410 // optionalAttrs (build-system != [ ]) {
411 inherit build-system;
412 }
413 // attrs.passthru or { };
414
415 meta = {
416 # default to python's platforms
417 platforms = python.meta.platforms;
418 isBuildPythonPackage = python.meta.platforms;
419 }
420 // meta;
421 }
422 // optionalAttrs (attrs ? checkPhase) {
423 # If given use the specified checkPhase, otherwise use the setup hook.
424 # Longer-term we should get rid of `checkPhase` and use `installCheckPhase`.
425 installCheckPhase = attrs.checkPhase;
426 }
427 //
428 lib.mapAttrs
429 (
430 name: value:
431 lib.throwIf (
432 attrs.${name} == [ ]
433 ) "${lib.getName finalAttrs}: ${name} must be unspecified, null or a non-empty list." attrs.${name}
434 )
435 (
436 getOptionalAttrs [
437 "enabledTestMarks"
438 "enabledTestPaths"
439 "enabledTests"
440 ] attrs
441 )
442 );
443
444 # This derivation transformation function must be independent to `attrs`
445 # for fixed-point arguments support in the future.
446 transformDrv =
447 let
448 # Workaround to make the `lib.extendDerivation`-based disabled functionality
449 # respect `<pkg>.overrideAttrs`
450 # It doesn't cover `<pkg>.<output>.overrideAttrs`.
451 disablePythonPackage =
452 drv:
453 extendDerivation (
454 drv.disabled
455 -> throw "${removePrefix namePrefix drv.name} not supported for interpreter ${python.executable}"
456 ) { } drv
457 // {
458 overrideAttrs = fdrv: disablePythonPackage (drv.overrideAttrs fdrv);
459 };
460 in
461 drv: disablePythonPackage (toPythonModule drv);
462
463in
464transformDrv self