pyrox.dev
1# Functions for copying sources to the Nix store.
2{ lib }:
3
4# Tested in lib/tests/sources.sh
5let
6 inherit (builtins)
7 match
8 readDir
9 split
10 storeDir
11 tryEval
12 ;
13 inherit (lib)
14 boolToString
15 filter
16 getAttr
17 isString
18 pathExists
19 readFile
20 ;
21
22 /*
23 Returns the type of a path: regular (for file), symlink, or directory.
24 */
25 pathType = path: getAttr (baseNameOf path) (readDir (dirOf path));
26
27 /*
28 Returns true if the path exists and is a directory, false otherwise.
29 */
30 pathIsDirectory = path: if pathExists path then (pathType path) == "directory" else false;
31
32 /*
33 Returns true if the path exists and is a regular file, false otherwise.
34 */
35 pathIsRegularFile = path: if pathExists path then (pathType path) == "regular" else false;
36
37 /*
38 A basic filter for `cleanSourceWith` that removes
39 directories of version control system, backup files (*~)
40 and some generated files.
41 */
42 cleanSourceFilter = name: type: let baseName = baseNameOf (toString name); in ! (
43 # Filter out version control software files/directories
44 (baseName == ".git" || type == "directory" && (baseName == ".svn" || baseName == "CVS" || baseName == ".hg")) ||
45 # Filter out editor backup / swap files.
46 lib.hasSuffix "~" baseName ||
47 match "^\\.sw[a-z]$" baseName != null ||
48 match "^\\..*\\.sw[a-z]$" baseName != null ||
49
50 # Filter out generates files.
51 lib.hasSuffix ".o" baseName ||
52 lib.hasSuffix ".so" baseName ||
53 # Filter out nix-build result symlinks
54 (type == "symlink" && lib.hasPrefix "result" baseName) ||
55 # Filter out sockets and other types of files we can't have in the store.
56 (type == "unknown")
57 );
58
59 /*
60 Filters a source tree removing version control files and directories using cleanSourceFilter.
61
62 Example:
63 cleanSource ./.
64 */
65 cleanSource = src: cleanSourceWith { filter = cleanSourceFilter; inherit src; };
66
67 /*
68 Like `builtins.filterSource`, except it will compose with itself,
69 allowing you to chain multiple calls together without any
70 intermediate copies being put in the nix store.
71
72 Example:
73 lib.cleanSourceWith {
74 filter = f;
75 src = lib.cleanSourceWith {
76 filter = g;
77 src = ./.;
78 };
79 }
80 # Succeeds!
81
82 builtins.filterSource f (builtins.filterSource g ./.)
83 # Fails!
84
85 */
86 cleanSourceWith =
87 {
88 # A path or cleanSourceWith result to filter and/or rename.
89 src,
90 # Optional with default value: constant true (include everything)
91 # The function will be combined with the && operator such
92 # that src.filter is called lazily.
93 # For implementing a filter, see
94 # https://nixos.org/nix/manual/#builtin-filterSource
95 # Type: A function (path -> type -> bool)
96 filter ? _path: _type: true,
97 # Optional name to use as part of the store path.
98 # This defaults to `src.name` or otherwise `"source"`.
99 name ? null
100 }:
101 let
102 orig = toSourceAttributes src;
103 in fromSourceAttributes {
104 inherit (orig) origSrc;
105 filter = path: type: filter path type && orig.filter path type;
106 name = if name != null then name else orig.name;
107 };
108
109 /*
110 Add logging to a source, for troubleshooting the filtering behavior.
111 Type:
112 sources.trace :: sourceLike -> Source
113 */
114 trace =
115 # Source to debug. The returned source will behave like this source, but also log its filter invocations.
116 src:
117 let
118 attrs = toSourceAttributes src;
119 in
120 fromSourceAttributes (
121 attrs // {
122 filter = path: type:
123 let
124 r = attrs.filter path type;
125 in
126 builtins.trace "${attrs.name}.filter ${path} = ${boolToString r}" r;
127 }
128 ) // {
129 satisfiesSubpathInvariant = src ? satisfiesSubpathInvariant && src.satisfiesSubpathInvariant;
130 };
131
132 /*
133 Filter sources by a list of regular expressions.
134
135 Example: src = sourceByRegex ./my-subproject [".*\.py$" "^database.sql$"]
136 */
137 sourceByRegex = src: regexes:
138 let
139 isFiltered = src ? _isLibCleanSourceWith;
140 origSrc = if isFiltered then src.origSrc else src;
141 in lib.cleanSourceWith {
142 filter = (path: type:
143 let relPath = lib.removePrefix (toString origSrc + "/") (toString path);
144 in lib.any (re: match re relPath != null) regexes);
145 inherit src;
146 };
147
148 /*
149 Get all files ending with the specified suffices from the given
150 source directory or its descendants, omitting files that do not match
151 any suffix. The result of the example below will include files like
152 `./dir/module.c` and `./dir/subdir/doc.xml` if present.
153
154 Type: sourceLike -> [String] -> Source
155
156 Example:
157 sourceFilesBySuffices ./. [ ".xml" ".c" ]
158 */
159 sourceFilesBySuffices =
160 # Path or source containing the files to be returned
161 src:
162 # A list of file suffix strings
163 exts:
164 let filter = name: type:
165 let base = baseNameOf (toString name);
166 in type == "directory" || lib.any (ext: lib.hasSuffix ext base) exts;
167 in cleanSourceWith { inherit filter src; };
168
169 pathIsGitRepo = path: (_commitIdFromGitRepoOrError path)?value;
170
171 /*
172 Get the commit id of a git repo.
173
174 Example: commitIdFromGitRepo <nixpkgs/.git>
175 */
176 commitIdFromGitRepo = path:
177 let commitIdOrError = _commitIdFromGitRepoOrError path;
178 in commitIdOrError.value or (throw commitIdOrError.error);
179
180 # Get the commit id of a git repo.
181
182 # Returns `{ value = commitHash }` or `{ error = "... message ..." }`.
183
184 # Example: commitIdFromGitRepo <nixpkgs/.git>
185 # not exported, used for commitIdFromGitRepo
186 _commitIdFromGitRepoOrError =
187 let readCommitFromFile = file: path:
188 let fileName = path + "/${file}";
189 packedRefsName = path + "/packed-refs";
190 absolutePath = base: path:
191 if lib.hasPrefix "/" path
192 then path
193 else toString (/. + "${base}/${path}");
194 in if pathIsRegularFile path
195 # Resolve git worktrees. See gitrepository-layout(5)
196 then
197 let m = match "^gitdir: (.*)$" (lib.fileContents path);
198 in if m == null
199 then { error = "File contains no gitdir reference: " + path; }
200 else
201 let gitDir = absolutePath (dirOf path) (lib.head m);
202 commonDir'' = if pathIsRegularFile "${gitDir}/commondir"
203 then lib.fileContents "${gitDir}/commondir"
204 else gitDir;
205 commonDir' = lib.removeSuffix "/" commonDir'';
206 commonDir = absolutePath gitDir commonDir';
207 refFile = lib.removePrefix "${commonDir}/" "${gitDir}/${file}";
208 in readCommitFromFile refFile commonDir
209
210 else if pathIsRegularFile fileName
211 # Sometimes git stores the commitId directly in the file but
212 # sometimes it stores something like: «ref: refs/heads/branch-name»
213 then
214 let fileContent = lib.fileContents fileName;
215 matchRef = match "^ref: (.*)$" fileContent;
216 in if matchRef == null
217 then { value = fileContent; }
218 else readCommitFromFile (lib.head matchRef) path
219
220 else if pathIsRegularFile packedRefsName
221 # Sometimes, the file isn't there at all and has been packed away in the
222 # packed-refs file, so we have to grep through it:
223 then
224 let fileContent = readFile packedRefsName;
225 matchRef = match "([a-z0-9]+) ${file}";
226 isRef = s: isString s && (matchRef s) != null;
227 # there is a bug in libstdc++ leading to stackoverflow for long strings:
228 # https://github.com/NixOS/nix/issues/2147#issuecomment-659868795
229 refs = filter isRef (split "\n" fileContent);
230 in if refs == []
231 then { error = "Could not find " + file + " in " + packedRefsName; }
232 else { value = lib.head (matchRef (lib.head refs)); }
233
234 else { error = "Not a .git directory: " + toString path; };
235 in readCommitFromFile "HEAD";
236
237 pathHasContext = builtins.hasContext or (lib.hasPrefix storeDir);
238
239 canCleanSource = src: src ? _isLibCleanSourceWith || !(pathHasContext (toString src));
240
241 # -------------------------------------------------------------------------- #
242 # Internal functions
243 #
244
245 # toSourceAttributes : sourceLike -> SourceAttrs
246 #
247 # Convert any source-like object into a simple, singular representation.
248 # We don't expose this representation in order to avoid having a fifth path-
249 # like class of objects in the wild.
250 # (Existing ones being: paths, strings, sources and x//{outPath})
251 # So instead of exposing internals, we build a library of combinator functions.
252 toSourceAttributes = src:
253 let
254 isFiltered = src ? _isLibCleanSourceWith;
255 in
256 {
257 # The original path
258 origSrc = if isFiltered then src.origSrc else src;
259 filter = if isFiltered then src.filter else _: _: true;
260 name = if isFiltered then src.name else "source";
261 };
262
263 # fromSourceAttributes : SourceAttrs -> Source
264 #
265 # Inverse of toSourceAttributes for Source objects.
266 fromSourceAttributes = { origSrc, filter, name }:
267 {
268 _isLibCleanSourceWith = true;
269 inherit origSrc filter name;
270 outPath = builtins.path { inherit filter name; path = origSrc; };
271 };
272
273in {
274 inherit
275 pathType
276 pathIsDirectory
277 pathIsRegularFile
278
279 pathIsGitRepo
280 commitIdFromGitRepo
281
282 cleanSource
283 cleanSourceWith
284 cleanSourceFilter
285 pathHasContext
286 canCleanSource
287
288 sourceByRegex
289 sourceFilesBySuffices
290
291 trace
292 ;
293}