pyrox.dev
1# Functions for copying sources to the Nix store.
2{ lib }:
3
4# Tested in lib/tests/sources.sh
5let
6 inherit (builtins)
7 match
8 split
9 storeDir
10 ;
11 inherit (lib)
12 boolToString
13 filter
14 isString
15 readFile
16 ;
17 inherit (lib.filesystem)
18 pathIsRegularFile
19 ;
20
21 /**
22 A basic filter for `cleanSourceWith` that removes
23 directories of version control system, backup files (*~)
24 and some generated files.
25
26 # Inputs
27
28 `name`
29
30 : 1\. Function argument
31
32 `type`
33
34 : 2\. Function argument
35 */
36 cleanSourceFilter =
37 name: type:
38 let
39 baseName = baseNameOf (toString name);
40 in
41 !(
42 # Filter out version control software files/directories
43 (
44 baseName == ".git"
45 || type == "directory" && (baseName == ".svn" || baseName == "CVS" || baseName == ".hg")
46 )
47 ||
48 # Filter out editor backup / swap files.
49 lib.hasSuffix "~" baseName
50 || match "^\\.sw[a-z]$" baseName != null
51 || match "^\\..*\\.sw[a-z]$" baseName != null
52 ||
53
54 # Filter out generates files.
55 lib.hasSuffix ".o" baseName
56 || lib.hasSuffix ".so" baseName
57 ||
58 # Filter out nix-build result symlinks
59 (type == "symlink" && lib.hasPrefix "result" baseName)
60 ||
61 # Filter out sockets and other types of files we can't have in the store.
62 (type == "unknown")
63 );
64
65 /**
66 Filters a source tree removing version control files and directories using cleanSourceFilter.
67
68 # Inputs
69
70 `src`
71
72 : 1\. Function argument
73
74 # Examples
75 :::{.example}
76 ## `cleanSource` usage example
77
78 ```nix
79 cleanSource ./.
80 ```
81
82 :::
83 */
84 cleanSource =
85 src:
86 cleanSourceWith {
87 filter = cleanSourceFilter;
88 inherit src;
89 };
90
91 /**
92 Like `builtins.filterSource`, except it will compose with itself,
93 allowing you to chain multiple calls together without any
94 intermediate copies being put in the nix store.
95
96 # Examples
97 :::{.example}
98 ## `cleanSourceWith` usage example
99
100 ```nix
101 lib.cleanSourceWith {
102 filter = f;
103 src = lib.cleanSourceWith {
104 filter = g;
105 src = ./.;
106 };
107 }
108 # Succeeds!
109
110 builtins.filterSource f (builtins.filterSource g ./.)
111 # Fails!
112 ```
113
114 :::
115 */
116 cleanSourceWith =
117 {
118 # A path or cleanSourceWith result to filter and/or rename.
119 src,
120 # Optional with default value: constant true (include everything)
121 # The function will be combined with the && operator such
122 # that src.filter is called lazily.
123 # For implementing a filter, see
124 # https://nixos.org/nix/manual/#builtin-filterSource
125 # Type: A function (path -> type -> bool)
126 filter ? _path: _type: true,
127 # Optional name to use as part of the store path.
128 # This defaults to `src.name` or otherwise `"source"`.
129 name ? null,
130 }:
131 let
132 orig = toSourceAttributes src;
133 in
134 fromSourceAttributes {
135 inherit (orig) origSrc;
136 filter = path: type: filter path type && orig.filter path type;
137 name = if name != null then name else orig.name;
138 };
139
140 /**
141 Add logging to a source, for troubleshooting the filtering behavior.
142
143 # Inputs
144
145 `src`
146
147 : Source to debug. The returned source will behave like this source, but also log its filter invocations.
148
149 # Type
150
151 ```
152 sources.trace :: sourceLike -> Source
153 ```
154 */
155 trace =
156 # Source to debug. The returned source will behave like this source, but also log its filter invocations.
157 src:
158 let
159 attrs = toSourceAttributes src;
160 in
161 fromSourceAttributes (
162 attrs
163 // {
164 filter =
165 path: type:
166 let
167 r = attrs.filter path type;
168 in
169 builtins.trace "${attrs.name}.filter ${path} = ${boolToString r}" r;
170 }
171 )
172 // {
173 satisfiesSubpathInvariant = src ? satisfiesSubpathInvariant && src.satisfiesSubpathInvariant;
174 };
175
176 /**
177 Filter sources by a list of regular expressions.
178
179 # Inputs
180
181 `src`
182
183 : 1\. Function argument
184
185 `regexes`
186
187 : 2\. Function argument
188
189 # Examples
190 :::{.example}
191 ## `sourceByRegex` usage example
192
193 ```nix
194 src = sourceByRegex ./my-subproject [".*\.py$" "^database.sql$"]
195 ```
196
197 :::
198 */
199 sourceByRegex =
200 src: regexes:
201 let
202 isFiltered = src ? _isLibCleanSourceWith;
203 origSrc = if isFiltered then src.origSrc else src;
204 in
205 lib.cleanSourceWith {
206 filter = (
207 path: type:
208 let
209 relPath = lib.removePrefix (toString origSrc + "/") (toString path);
210 in
211 lib.any (re: match re relPath != null) regexes
212 );
213 inherit src;
214 };
215
216 /**
217 Get all files ending with the specified suffices from the given
218 source directory or its descendants, omitting files that do not match
219 any suffix. The result of the example below will include files like
220 `./dir/module.c` and `./dir/subdir/doc.xml` if present.
221
222 # Inputs
223
224 `src`
225
226 : Path or source containing the files to be returned
227
228 `exts`
229
230 : A list of file suffix strings
231
232 # Type
233
234 ```
235 sourceLike -> [String] -> Source
236 ```
237
238 # Examples
239 :::{.example}
240 ## `sourceFilesBySuffices` usage example
241
242 ```nix
243 sourceFilesBySuffices ./. [ ".xml" ".c" ]
244 ```
245
246 :::
247 */
248 sourceFilesBySuffices =
249 # Path or source containing the files to be returned
250 src:
251 # A list of file suffix strings
252 exts:
253 let
254 filter =
255 name: type:
256 let
257 base = baseNameOf (toString name);
258 in
259 type == "directory" || lib.any (ext: lib.hasSuffix ext base) exts;
260 in
261 cleanSourceWith { inherit filter src; };
262
263 pathIsGitRepo = path: (_commitIdFromGitRepoOrError path) ? value;
264
265 /**
266 Get the commit id of a git repo.
267
268 # Inputs
269
270 `path`
271
272 : 1\. Function argument
273
274 # Examples
275 :::{.example}
276 ## `commitIdFromGitRepo` usage example
277
278 ```nix
279 commitIdFromGitRepo <nixpkgs/.git>
280 ```
281
282 :::
283 */
284 commitIdFromGitRepo =
285 path:
286 let
287 commitIdOrError = _commitIdFromGitRepoOrError path;
288 in
289 commitIdOrError.value or (throw commitIdOrError.error);
290
291 # Get the commit id of a git repo.
292
293 # Returns `{ value = commitHash }` or `{ error = "... message ..." }`.
294
295 # Example: commitIdFromGitRepo <nixpkgs/.git>
296 # not exported, used for commitIdFromGitRepo
297 _commitIdFromGitRepoOrError =
298 let
299 readCommitFromFile =
300 file: path:
301 let
302 fileName = path + "/${file}";
303 packedRefsName = path + "/packed-refs";
304 absolutePath =
305 base: path: if lib.hasPrefix "/" path then path else toString (/. + "${base}/${path}");
306 in
307 if
308 pathIsRegularFile path
309 # Resolve git worktrees. See gitrepository-layout(5)
310 then
311 let
312 m = match "^gitdir: (.*)$" (lib.fileContents path);
313 in
314 if m == null then
315 { error = "File contains no gitdir reference: " + path; }
316 else
317 let
318 gitDir = absolutePath (dirOf path) (lib.head m);
319 commonDir'' =
320 if pathIsRegularFile "${gitDir}/commondir" then lib.fileContents "${gitDir}/commondir" else gitDir;
321 commonDir' = lib.removeSuffix "/" commonDir'';
322 commonDir = absolutePath gitDir commonDir';
323 refFile = lib.removePrefix "${commonDir}/" "${gitDir}/${file}";
324 in
325 readCommitFromFile refFile commonDir
326
327 else if
328 pathIsRegularFile fileName
329 # Sometimes git stores the commitId directly in the file but
330 # sometimes it stores something like: «ref: refs/heads/branch-name»
331 then
332 let
333 fileContent = lib.fileContents fileName;
334 matchRef = match "^ref: (.*)$" fileContent;
335 in
336 if matchRef == null then { value = fileContent; } else readCommitFromFile (lib.head matchRef) path
337
338 else if
339 pathIsRegularFile packedRefsName
340 # Sometimes, the file isn't there at all and has been packed away in the
341 # packed-refs file, so we have to grep through it:
342 then
343 let
344 fileContent = readFile packedRefsName;
345 matchRef = match "([a-z0-9]+) ${file}";
346 isRef = s: isString s && (matchRef s) != null;
347 # there is a bug in libstdc++ leading to stackoverflow for long strings:
348 # https://github.com/NixOS/nix/issues/2147#issuecomment-659868795
349 refs = filter isRef (split "\n" fileContent);
350 in
351 if refs == [ ] then
352 { error = "Could not find " + file + " in " + packedRefsName; }
353 else
354 { value = lib.head (matchRef (lib.head refs)); }
355
356 else
357 { error = "Not a .git directory: " + toString path; };
358 in
359 readCommitFromFile "HEAD";
360
361 pathHasContext = builtins.hasContext or (lib.hasPrefix storeDir);
362
363 canCleanSource = src: src ? _isLibCleanSourceWith || !(pathHasContext (toString src));
364
365 # -------------------------------------------------------------------------- #
366 # Internal functions
367 #
368
369 # toSourceAttributes : sourceLike -> SourceAttrs
370 #
371 # Convert any source-like object into a simple, singular representation.
372 # We don't expose this representation in order to avoid having a fifth path-
373 # like class of objects in the wild.
374 # (Existing ones being: paths, strings, sources and x//{outPath})
375 # So instead of exposing internals, we build a library of combinator functions.
376 toSourceAttributes =
377 src:
378 let
379 isFiltered = src ? _isLibCleanSourceWith;
380 in
381 {
382 # The original path
383 origSrc = if isFiltered then src.origSrc else src;
384 filter = if isFiltered then src.filter else _: _: true;
385 name = if isFiltered then src.name else "source";
386 };
387
388 # fromSourceAttributes : SourceAttrs -> Source
389 #
390 # Inverse of toSourceAttributes for Source objects.
391 fromSourceAttributes =
392 {
393 origSrc,
394 filter,
395 name,
396 }:
397 {
398 _isLibCleanSourceWith = true;
399 inherit origSrc filter name;
400 outPath = builtins.path {
401 inherit filter name;
402 path = origSrc;
403 };
404 };
405
406in
407{
408
409 pathType =
410 lib.warnIf (lib.oldestSupportedReleaseIsAtLeast 2305)
411 "lib.sources.pathType has been moved to lib.filesystem.pathType."
412 lib.filesystem.pathType;
413
414 pathIsDirectory =
415 lib.warnIf (lib.oldestSupportedReleaseIsAtLeast 2305)
416 "lib.sources.pathIsDirectory has been moved to lib.filesystem.pathIsDirectory."
417 lib.filesystem.pathIsDirectory;
418
419 pathIsRegularFile =
420 lib.warnIf (lib.oldestSupportedReleaseIsAtLeast 2305)
421 "lib.sources.pathIsRegularFile has been moved to lib.filesystem.pathIsRegularFile."
422 lib.filesystem.pathIsRegularFile;
423
424 inherit
425 pathIsGitRepo
426 commitIdFromGitRepo
427
428 cleanSource
429 cleanSourceWith
430 cleanSourceFilter
431 pathHasContext
432 canCleanSource
433
434 sourceByRegex
435 sourceFilesBySuffices
436
437 trace
438 ;
439}