pyrox.dev
1# Functions for copying sources to the Nix store.
2{ lib }:
3
4# Tested in lib/tests/sources.sh
5let
6 inherit (lib.strings)
7 match
8 split
9 storeDir
10 ;
11 inherit (lib)
12 boolToString
13 filter
14 isString
15 readFile
16 ;
17 inherit (lib.filesystem)
18 pathIsRegularFile
19 ;
20
21 /**
22 A basic filter for `cleanSourceWith` that removes
23 directories of version control system, backup files (*~)
24 and some generated files.
25
26 # Inputs
27
28 `name`
29
30 : 1\. Function argument
31
32 `type`
33
34 : 2\. Function argument
35 */
36 cleanSourceFilter =
37 name: type:
38 let
39 baseName = baseNameOf (toString name);
40 in
41 !(
42 # Filter out version control software files/directories
43 (
44 baseName == ".git"
45 ||
46 type == "directory"
47 && (baseName == ".svn" || baseName == "CVS" || baseName == ".hg" || baseName == ".jj")
48 )
49 ||
50 # Filter out editor backup / swap files.
51 lib.hasSuffix "~" baseName
52 || match "^\\.sw[a-z]$" baseName != null
53 || match "^\\..*\\.sw[a-z]$" baseName != null
54 ||
55
56 # Filter out generates files.
57 lib.hasSuffix ".o" baseName
58 || lib.hasSuffix ".so" baseName
59 ||
60 # Filter out nix-build result symlinks
61 (type == "symlink" && lib.hasPrefix "result" baseName)
62 ||
63 # Filter out sockets and other types of files we can't have in the store.
64 (type == "unknown")
65 );
66
67 /**
68 Filters a source tree removing version control files and directories using cleanSourceFilter.
69
70 # Inputs
71
72 `src`
73
74 : 1\. Function argument
75
76 # Examples
77 :::{.example}
78 ## `cleanSource` usage example
79
80 ```nix
81 cleanSource ./.
82 ```
83
84 :::
85 */
86 cleanSource =
87 src:
88 cleanSourceWith {
89 filter = cleanSourceFilter;
90 inherit src;
91 };
92
93 /**
94 Like `builtins.filterSource`, except it will compose with itself,
95 allowing you to chain multiple calls together without any
96 intermediate copies being put in the nix store.
97
98 # Examples
99 :::{.example}
100 ## `cleanSourceWith` usage example
101
102 ```nix
103 lib.cleanSourceWith {
104 filter = f;
105 src = lib.cleanSourceWith {
106 filter = g;
107 src = ./.;
108 };
109 }
110 # Succeeds!
111
112 builtins.filterSource f (builtins.filterSource g ./.)
113 # Fails!
114 ```
115
116 :::
117 */
118 cleanSourceWith =
119 {
120 # A path or cleanSourceWith result to filter and/or rename.
121 src,
122 # Optional with default value: constant true (include everything)
123 # The function will be combined with the && operator such
124 # that src.filter is called lazily.
125 # For implementing a filter, see
126 # https://nixos.org/nix/manual/#builtin-filterSource
127 # Type: A function (path -> type -> bool)
128 filter ? _path: _type: true,
129 # Optional name to use as part of the store path.
130 # This defaults to `src.name` or otherwise `"source"`.
131 name ? null,
132 }:
133 let
134 orig = toSourceAttributes src;
135 in
136 fromSourceAttributes {
137 inherit (orig) origSrc;
138 filter = path: type: filter path type && orig.filter path type;
139 name = if name != null then name else orig.name;
140 };
141
142 /**
143 Add logging to a source, for troubleshooting the filtering behavior.
144
145 # Inputs
146
147 `src`
148
149 : Source to debug. The returned source will behave like this source, but also log its filter invocations.
150
151 # Type
152
153 ```
154 sources.trace :: sourceLike -> Source
155 ```
156 */
157 trace =
158 # Source to debug. The returned source will behave like this source, but also log its filter invocations.
159 src:
160 let
161 attrs = toSourceAttributes src;
162 in
163 fromSourceAttributes (
164 attrs
165 // {
166 filter =
167 path: type:
168 let
169 r = attrs.filter path type;
170 in
171 builtins.trace "${attrs.name}.filter ${path} = ${boolToString r}" r;
172 }
173 )
174 // {
175 satisfiesSubpathInvariant = src ? satisfiesSubpathInvariant && src.satisfiesSubpathInvariant;
176 };
177
178 /**
179 Filter sources by a list of regular expressions.
180
181 # Inputs
182
183 `src`
184
185 : 1\. Function argument
186
187 `regexes`
188
189 : 2\. Function argument
190
191 # Examples
192 :::{.example}
193 ## `sourceByRegex` usage example
194
195 ```nix
196 src = sourceByRegex ./my-subproject [".*\.py$" "^database.sql$"]
197 ```
198
199 :::
200 */
201 sourceByRegex =
202 src: regexes:
203 let
204 isFiltered = src ? _isLibCleanSourceWith;
205 origSrc = if isFiltered then src.origSrc else src;
206 in
207 lib.cleanSourceWith {
208 filter = (
209 path: type:
210 let
211 relPath = lib.removePrefix (toString origSrc + "/") (toString path);
212 in
213 lib.any (re: match re relPath != null) regexes
214 );
215 inherit src;
216 };
217
218 /**
219 Get all files ending with the specified suffices from the given
220 source directory or its descendants, omitting files that do not match
221 any suffix. The result of the example below will include files like
222 `./dir/module.c` and `./dir/subdir/doc.xml` if present.
223
224 # Inputs
225
226 `src`
227
228 : Path or source containing the files to be returned
229
230 `exts`
231
232 : A list of file suffix strings
233
234 # Type
235
236 ```
237 sourceLike -> [String] -> Source
238 ```
239
240 # Examples
241 :::{.example}
242 ## `sourceFilesBySuffices` usage example
243
244 ```nix
245 sourceFilesBySuffices ./. [ ".xml" ".c" ]
246 ```
247
248 :::
249 */
250 sourceFilesBySuffices =
251 # Path or source containing the files to be returned
252 src:
253 # A list of file suffix strings
254 exts:
255 let
256 filter =
257 name: type:
258 let
259 base = baseNameOf (toString name);
260 in
261 type == "directory" || lib.any (ext: lib.hasSuffix ext base) exts;
262 in
263 cleanSourceWith { inherit filter src; };
264
265 pathIsGitRepo = path: (_commitIdFromGitRepoOrError path) ? value;
266
267 /**
268 Get the commit id of a git repo.
269
270 # Inputs
271
272 `path`
273
274 : 1\. Function argument
275
276 # Examples
277 :::{.example}
278 ## `commitIdFromGitRepo` usage example
279
280 ```nix
281 commitIdFromGitRepo <nixpkgs/.git>
282 ```
283
284 :::
285 */
286 commitIdFromGitRepo =
287 path:
288 let
289 commitIdOrError = _commitIdFromGitRepoOrError path;
290 in
291 commitIdOrError.value or (throw commitIdOrError.error);
292
293 # Get the commit id of a git repo.
294
295 # Returns `{ value = commitHash }` or `{ error = "... message ..." }`.
296
297 # Example: commitIdFromGitRepo <nixpkgs/.git>
298 # not exported, used for commitIdFromGitRepo
299 _commitIdFromGitRepoOrError =
300 let
301 readCommitFromFile =
302 file: path:
303 let
304 fileName = path + "/${file}";
305 packedRefsName = path + "/packed-refs";
306 absolutePath =
307 base: path: if lib.hasPrefix "/" path then path else toString (/. + "${base}/${path}");
308 in
309 if
310 pathIsRegularFile path
311 # Resolve git worktrees. See gitrepository-layout(5)
312 then
313 let
314 m = match "^gitdir: (.*)$" (lib.fileContents path);
315 in
316 if m == null then
317 { error = "File contains no gitdir reference: " + path; }
318 else
319 let
320 gitDir = absolutePath (dirOf path) (lib.head m);
321 commonDir'' =
322 if pathIsRegularFile "${gitDir}/commondir" then lib.fileContents "${gitDir}/commondir" else gitDir;
323 commonDir' = lib.removeSuffix "/" commonDir'';
324 commonDir = absolutePath gitDir commonDir';
325 refFile = lib.removePrefix "${commonDir}/" "${gitDir}/${file}";
326 in
327 readCommitFromFile refFile commonDir
328
329 else if
330 pathIsRegularFile fileName
331 # Sometimes git stores the commitId directly in the file but
332 # sometimes it stores something like: «ref: refs/heads/branch-name»
333 then
334 let
335 fileContent = lib.fileContents fileName;
336 matchRef = match "^ref: (.*)$" fileContent;
337 in
338 if matchRef == null then { value = fileContent; } else readCommitFromFile (lib.head matchRef) path
339
340 else if
341 pathIsRegularFile packedRefsName
342 # Sometimes, the file isn't there at all and has been packed away in the
343 # packed-refs file, so we have to grep through it:
344 then
345 let
346 fileContent = readFile packedRefsName;
347 matchRef = match "([a-z0-9]+) ${file}";
348 isRef = s: isString s && (matchRef s) != null;
349 # there is a bug in libstdc++ leading to stackoverflow for long strings:
350 # https://github.com/NixOS/nix/issues/2147#issuecomment-659868795
351 refs = filter isRef (split "\n" fileContent);
352 in
353 if refs == [ ] then
354 { error = "Could not find " + file + " in " + packedRefsName; }
355 else
356 { value = lib.head (matchRef (lib.head refs)); }
357
358 else
359 { error = "Not a .git directory: " + toString path; };
360 in
361 readCommitFromFile "HEAD";
362
363 pathHasContext = builtins.hasContext or (lib.hasPrefix storeDir);
364
365 canCleanSource = src: src ? _isLibCleanSourceWith || !(pathHasContext (toString src));
366
367 # -------------------------------------------------------------------------- #
368 # Internal functions
369 #
370
371 # toSourceAttributes : sourceLike -> SourceAttrs
372 #
373 # Convert any source-like object into a simple, singular representation.
374 # We don't expose this representation in order to avoid having a fifth path-
375 # like class of objects in the wild.
376 # (Existing ones being: paths, strings, sources and x//{outPath})
377 # So instead of exposing internals, we build a library of combinator functions.
378 toSourceAttributes =
379 src:
380 let
381 isFiltered = src ? _isLibCleanSourceWith;
382 in
383 {
384 # The original path
385 origSrc = if isFiltered then src.origSrc else src;
386 filter = if isFiltered then src.filter else _: _: true;
387 name = if isFiltered then src.name else "source";
388 };
389
390 # fromSourceAttributes : SourceAttrs -> Source
391 #
392 # Inverse of toSourceAttributes for Source objects.
393 fromSourceAttributes =
394 {
395 origSrc,
396 filter,
397 name,
398 }:
399 {
400 _isLibCleanSourceWith = true;
401 inherit origSrc filter name;
402 outPath = builtins.path {
403 inherit filter name;
404 path = origSrc;
405 };
406 };
407
408 # urlToName : (URL | Path | String) -> String
409 #
410 # Transform a URL (or path, or string) into a clean package name.
411 urlToName =
412 url:
413 let
414 inherit (lib.strings) stringLength;
415 base = baseNameOf (lib.removeSuffix "/" (lib.last (lib.splitString ":" (toString url))));
416 # chop away one git or archive-related extension
417 removeExt =
418 name:
419 let
420 matchExt = match "(.*)\\.(git|tar|zip|gz|tgz|bz|tbz|bz2|tbz2|lzma|txz|xz|zstd)$" name;
421 in
422 if matchExt != null then lib.head matchExt else name;
423 # apply function f to string x while the result shrinks
424 shrink =
425 f: x:
426 let
427 v = f x;
428 in
429 if stringLength v < stringLength x then shrink f v else x;
430 in
431 shrink removeExt base;
432
433 # shortRev : (String | Integer) -> String
434 #
435 # Given a package revision (like "refs/tags/v12.0"), produce a short revision ("12.0").
436 shortRev =
437 rev:
438 let
439 baseRev = baseNameOf (toString rev);
440 matchHash = match "[a-f0-9]+" baseRev;
441 matchVer = match "([A-Za-z]+[-_. ]?)*(v)?([0-9.]+.*)" baseRev;
442 in
443 if matchHash != null then
444 builtins.substring 0 7 baseRev
445 else if matchVer != null then
446 lib.last matchVer
447 else
448 baseRev;
449
450 # revOrTag : String -> String -> String
451 #
452 # Turn git `rev` and `tag` pair into a revision usable in `repoRevToName*`.
453 revOrTag =
454 rev: tag:
455 if tag != null then
456 tag
457 else if rev != null then
458 rev
459 else
460 "HEAD";
461
462 # repoRevToNameFull : (URL | Path | String) -> (String | Integer | null) -> (String | null) -> String
463 #
464 # See `repoRevToName` below.
465 repoRevToNameFull =
466 repo_: rev_: suffix_:
467 let
468 repo = urlToName repo_;
469 rev = if rev_ != null then "-${shortRev rev_}" else "";
470 suffix = if suffix_ != null then "-${suffix_}" else "";
471 in
472 "${repo}${rev}${suffix}-source";
473
474 # repoRevToName : String -> (URL | Path | String) -> (String | Integer | null) -> String -> String
475 #
476 # Produce derivation.name attribute for a given repository URL/path/name and (optionally) its revision/version tag.
477 #
478 # This is used by fetch(zip|git|FromGitHub|hg|svn|etc) to generate discoverable
479 # /nix/store paths.
480 #
481 # This uses a different implementation depending on the `pretty` argument:
482 # "source" -> name everything as "source"
483 # "versioned" -> name everything as "${repo}-${rev}-source"
484 # "full" -> name everything as "${repo}-${rev}-${fetcher}-source"
485 repoRevToName =
486 kind:
487 # match on `kind` first to minimize the thunk
488 if kind == "source" then
489 (
490 repo: rev: suffix:
491 "source"
492 )
493 else if kind == "versioned" then
494 (
495 repo: rev: suffix:
496 repoRevToNameFull repo rev null
497 )
498 else if kind == "full" then
499 repoRevToNameFull
500 else
501 throw "repoRevToName: invalid kind";
502
503in
504{
505
506 pathType =
507 lib.warnIf (lib.oldestSupportedReleaseIsAtLeast 2305)
508 "lib.sources.pathType has been moved to lib.filesystem.pathType."
509 lib.filesystem.pathType;
510
511 pathIsDirectory =
512 lib.warnIf (lib.oldestSupportedReleaseIsAtLeast 2305)
513 "lib.sources.pathIsDirectory has been moved to lib.filesystem.pathIsDirectory."
514 lib.filesystem.pathIsDirectory;
515
516 pathIsRegularFile =
517 lib.warnIf (lib.oldestSupportedReleaseIsAtLeast 2305)
518 "lib.sources.pathIsRegularFile has been moved to lib.filesystem.pathIsRegularFile."
519 lib.filesystem.pathIsRegularFile;
520
521 inherit
522 pathIsGitRepo
523 commitIdFromGitRepo
524
525 cleanSource
526 cleanSourceWith
527 cleanSourceFilter
528 pathHasContext
529 canCleanSource
530
531 urlToName
532 shortRev
533 revOrTag
534 repoRevToName
535
536 sourceByRegex
537 sourceFilesBySuffices
538
539 trace
540 ;
541}