pyrox.dev
1/* Functions for copying sources to the Nix store. */
2{ lib }:
3
4# Tested in lib/tests/sources.sh
5let
6 inherit (builtins)
7 match
8 split
9 storeDir
10 ;
11 inherit (lib)
12 boolToString
13 filter
14 isString
15 readFile
16 ;
17 inherit (lib.filesystem)
18 pathIsRegularFile
19 ;
20
21 /*
22 A basic filter for `cleanSourceWith` that removes
23 directories of version control system, backup files (*~)
24 and some generated files.
25 */
26 cleanSourceFilter = name: type: let baseName = baseNameOf (toString name); in ! (
27 # Filter out version control software files/directories
28 (baseName == ".git" || type == "directory" && (baseName == ".svn" || baseName == "CVS" || baseName == ".hg")) ||
29 # Filter out editor backup / swap files.
30 lib.hasSuffix "~" baseName ||
31 match "^\\.sw[a-z]$" baseName != null ||
32 match "^\\..*\\.sw[a-z]$" baseName != null ||
33
34 # Filter out generates files.
35 lib.hasSuffix ".o" baseName ||
36 lib.hasSuffix ".so" baseName ||
37 # Filter out nix-build result symlinks
38 (type == "symlink" && lib.hasPrefix "result" baseName) ||
39 # Filter out sockets and other types of files we can't have in the store.
40 (type == "unknown")
41 );
42
43 /*
44 Filters a source tree removing version control files and directories using cleanSourceFilter.
45
46 Example:
47 cleanSource ./.
48 */
49 cleanSource = src: cleanSourceWith { filter = cleanSourceFilter; inherit src; };
50
51 /*
52 Like `builtins.filterSource`, except it will compose with itself,
53 allowing you to chain multiple calls together without any
54 intermediate copies being put in the nix store.
55
56 Example:
57 lib.cleanSourceWith {
58 filter = f;
59 src = lib.cleanSourceWith {
60 filter = g;
61 src = ./.;
62 };
63 }
64 # Succeeds!
65
66 builtins.filterSource f (builtins.filterSource g ./.)
67 # Fails!
68
69 */
70 cleanSourceWith =
71 {
72 # A path or cleanSourceWith result to filter and/or rename.
73 src,
74 # Optional with default value: constant true (include everything)
75 # The function will be combined with the && operator such
76 # that src.filter is called lazily.
77 # For implementing a filter, see
78 # https://nixos.org/nix/manual/#builtin-filterSource
79 # Type: A function (path -> type -> bool)
80 filter ? _path: _type: true,
81 # Optional name to use as part of the store path.
82 # This defaults to `src.name` or otherwise `"source"`.
83 name ? null
84 }:
85 let
86 orig = toSourceAttributes src;
87 in fromSourceAttributes {
88 inherit (orig) origSrc;
89 filter = path: type: filter path type && orig.filter path type;
90 name = if name != null then name else orig.name;
91 };
92
93 /*
94 Add logging to a source, for troubleshooting the filtering behavior.
95 Type:
96 sources.trace :: sourceLike -> Source
97 */
98 trace =
99 # Source to debug. The returned source will behave like this source, but also log its filter invocations.
100 src:
101 let
102 attrs = toSourceAttributes src;
103 in
104 fromSourceAttributes (
105 attrs // {
106 filter = path: type:
107 let
108 r = attrs.filter path type;
109 in
110 builtins.trace "${attrs.name}.filter ${path} = ${boolToString r}" r;
111 }
112 ) // {
113 satisfiesSubpathInvariant = src ? satisfiesSubpathInvariant && src.satisfiesSubpathInvariant;
114 };
115
116 /*
117 Filter sources by a list of regular expressions.
118
119 Example: src = sourceByRegex ./my-subproject [".*\.py$" "^database.sql$"]
120 */
121 sourceByRegex = src: regexes:
122 let
123 isFiltered = src ? _isLibCleanSourceWith;
124 origSrc = if isFiltered then src.origSrc else src;
125 in lib.cleanSourceWith {
126 filter = (path: type:
127 let relPath = lib.removePrefix (toString origSrc + "/") (toString path);
128 in lib.any (re: match re relPath != null) regexes);
129 inherit src;
130 };
131
132 /*
133 Get all files ending with the specified suffices from the given
134 source directory or its descendants, omitting files that do not match
135 any suffix. The result of the example below will include files like
136 `./dir/module.c` and `./dir/subdir/doc.xml` if present.
137
138 Type: sourceLike -> [String] -> Source
139
140 Example:
141 sourceFilesBySuffices ./. [ ".xml" ".c" ]
142 */
143 sourceFilesBySuffices =
144 # Path or source containing the files to be returned
145 src:
146 # A list of file suffix strings
147 exts:
148 let filter = name: type:
149 let base = baseNameOf (toString name);
150 in type == "directory" || lib.any (ext: lib.hasSuffix ext base) exts;
151 in cleanSourceWith { inherit filter src; };
152
153 pathIsGitRepo = path: (_commitIdFromGitRepoOrError path)?value;
154
155 /*
156 Get the commit id of a git repo.
157
158 Example: commitIdFromGitRepo <nixpkgs/.git>
159 */
160 commitIdFromGitRepo = path:
161 let commitIdOrError = _commitIdFromGitRepoOrError path;
162 in commitIdOrError.value or (throw commitIdOrError.error);
163
164 # Get the commit id of a git repo.
165
166 # Returns `{ value = commitHash }` or `{ error = "... message ..." }`.
167
168 # Example: commitIdFromGitRepo <nixpkgs/.git>
169 # not exported, used for commitIdFromGitRepo
170 _commitIdFromGitRepoOrError =
171 let readCommitFromFile = file: path:
172 let fileName = path + "/${file}";
173 packedRefsName = path + "/packed-refs";
174 absolutePath = base: path:
175 if lib.hasPrefix "/" path
176 then path
177 else toString (/. + "${base}/${path}");
178 in if pathIsRegularFile path
179 # Resolve git worktrees. See gitrepository-layout(5)
180 then
181 let m = match "^gitdir: (.*)$" (lib.fileContents path);
182 in if m == null
183 then { error = "File contains no gitdir reference: " + path; }
184 else
185 let gitDir = absolutePath (dirOf path) (lib.head m);
186 commonDir'' = if pathIsRegularFile "${gitDir}/commondir"
187 then lib.fileContents "${gitDir}/commondir"
188 else gitDir;
189 commonDir' = lib.removeSuffix "/" commonDir'';
190 commonDir = absolutePath gitDir commonDir';
191 refFile = lib.removePrefix "${commonDir}/" "${gitDir}/${file}";
192 in readCommitFromFile refFile commonDir
193
194 else if pathIsRegularFile fileName
195 # Sometimes git stores the commitId directly in the file but
196 # sometimes it stores something like: «ref: refs/heads/branch-name»
197 then
198 let fileContent = lib.fileContents fileName;
199 matchRef = match "^ref: (.*)$" fileContent;
200 in if matchRef == null
201 then { value = fileContent; }
202 else readCommitFromFile (lib.head matchRef) path
203
204 else if pathIsRegularFile packedRefsName
205 # Sometimes, the file isn't there at all and has been packed away in the
206 # packed-refs file, so we have to grep through it:
207 then
208 let fileContent = readFile packedRefsName;
209 matchRef = match "([a-z0-9]+) ${file}";
210 isRef = s: isString s && (matchRef s) != null;
211 # there is a bug in libstdc++ leading to stackoverflow for long strings:
212 # https://github.com/NixOS/nix/issues/2147#issuecomment-659868795
213 refs = filter isRef (split "\n" fileContent);
214 in if refs == []
215 then { error = "Could not find " + file + " in " + packedRefsName; }
216 else { value = lib.head (matchRef (lib.head refs)); }
217
218 else { error = "Not a .git directory: " + toString path; };
219 in readCommitFromFile "HEAD";
220
221 pathHasContext = builtins.hasContext or (lib.hasPrefix storeDir);
222
223 canCleanSource = src: src ? _isLibCleanSourceWith || !(pathHasContext (toString src));
224
225 # -------------------------------------------------------------------------- #
226 # Internal functions
227 #
228
229 # toSourceAttributes : sourceLike -> SourceAttrs
230 #
231 # Convert any source-like object into a simple, singular representation.
232 # We don't expose this representation in order to avoid having a fifth path-
233 # like class of objects in the wild.
234 # (Existing ones being: paths, strings, sources and x//{outPath})
235 # So instead of exposing internals, we build a library of combinator functions.
236 toSourceAttributes = src:
237 let
238 isFiltered = src ? _isLibCleanSourceWith;
239 in
240 {
241 # The original path
242 origSrc = if isFiltered then src.origSrc else src;
243 filter = if isFiltered then src.filter else _: _: true;
244 name = if isFiltered then src.name else "source";
245 };
246
247 # fromSourceAttributes : SourceAttrs -> Source
248 #
249 # Inverse of toSourceAttributes for Source objects.
250 fromSourceAttributes = { origSrc, filter, name }:
251 {
252 _isLibCleanSourceWith = true;
253 inherit origSrc filter name;
254 outPath = builtins.path { inherit filter name; path = origSrc; };
255 };
256
257in {
258
259 pathType = lib.warnIf (lib.isInOldestRelease 2305)
260 "lib.sources.pathType has been moved to lib.filesystem.pathType."
261 lib.filesystem.pathType;
262
263 pathIsDirectory = lib.warnIf (lib.isInOldestRelease 2305)
264 "lib.sources.pathIsDirectory has been moved to lib.filesystem.pathIsDirectory."
265 lib.filesystem.pathIsDirectory;
266
267 pathIsRegularFile = lib.warnIf (lib.isInOldestRelease 2305)
268 "lib.sources.pathIsRegularFile has been moved to lib.filesystem.pathIsRegularFile."
269 lib.filesystem.pathIsRegularFile;
270
271 inherit
272 pathIsGitRepo
273 commitIdFromGitRepo
274
275 cleanSource
276 cleanSourceWith
277 cleanSourceFilter
278 pathHasContext
279 canCleanSource
280
281 sourceByRegex
282 sourceFilesBySuffices
283
284 trace
285 ;
286}