pyrox.dev
1# Functions for copying sources to the Nix store.
2{ lib }:
3
4# Tested in lib/tests/sources.sh
5let
6 inherit (builtins)
7 hasContext
8 match
9 readDir
10 split
11 storeDir
12 tryEval
13 ;
14 inherit (lib)
15 boolToString
16 filter
17 getAttr
18 isString
19 pathExists
20 readFile
21 ;
22
23 # Returns the type of a path: regular (for file), symlink, or directory
24 pathType = p: getAttr (baseNameOf p) (readDir (dirOf p));
25
26 # Returns true if the path exists and is a directory, false otherwise
27 pathIsDirectory = p: if pathExists p then (pathType p) == "directory" else false;
28
29 # Returns true if the path exists and is a regular file, false otherwise
30 pathIsRegularFile = p: if pathExists p then (pathType p) == "regular" else false;
31
32 # Bring in a path as a source, filtering out all Subversion and CVS
33 # directories, as well as backup files (*~).
34 cleanSourceFilter = name: type: let baseName = baseNameOf (toString name); in ! (
35 # Filter out version control software files/directories
36 (baseName == ".git" || type == "directory" && (baseName == ".svn" || baseName == "CVS" || baseName == ".hg")) ||
37 # Filter out editor backup / swap files.
38 lib.hasSuffix "~" baseName ||
39 match "^\\.sw[a-z]$" baseName != null ||
40 match "^\\..*\\.sw[a-z]$" baseName != null ||
41
42 # Filter out generates files.
43 lib.hasSuffix ".o" baseName ||
44 lib.hasSuffix ".so" baseName ||
45 # Filter out nix-build result symlinks
46 (type == "symlink" && lib.hasPrefix "result" baseName) ||
47 # Filter out sockets and other types of files we can't have in the store.
48 (type == "unknown")
49 );
50
51 # Filters a source tree removing version control files and directories using cleanSourceWith
52 #
53 # Example:
54 # cleanSource ./.
55 cleanSource = src: cleanSourceWith { filter = cleanSourceFilter; inherit src; };
56
57 # Like `builtins.filterSource`, except it will compose with itself,
58 # allowing you to chain multiple calls together without any
59 # intermediate copies being put in the nix store.
60 #
61 # lib.cleanSourceWith {
62 # filter = f;
63 # src = lib.cleanSourceWith {
64 # filter = g;
65 # src = ./.;
66 # };
67 # }
68 # # Succeeds!
69 #
70 # builtins.filterSource f (builtins.filterSource g ./.)
71 # # Fails!
72 #
73 # Parameters:
74 #
75 # src: A path or cleanSourceWith result to filter and/or rename.
76 #
77 # filter: A function (path -> type -> bool)
78 # Optional with default value: constant true (include everything)
79 # The function will be combined with the && operator such
80 # that src.filter is called lazily.
81 # For implementing a filter, see
82 # https://nixos.org/nix/manual/#builtin-filterSource
83 #
84 # name: Optional name to use as part of the store path.
85 # This defaults to `src.name` or otherwise `"source"`.
86 #
87 cleanSourceWith = { filter ? _path: _type: true, src, name ? null }:
88 let
89 orig = toSourceAttributes src;
90 in fromSourceAttributes {
91 inherit (orig) origSrc;
92 filter = path: type: filter path type && orig.filter path type;
93 name = if name != null then name else orig.name;
94 };
95
96 /*
97 Add logging to a source, for troubleshooting the filtering behavior.
98 Type:
99 sources.trace :: sourceLike -> Source
100 */
101 trace =
102 # Source to debug. The returned source will behave like this source, but also log its filter invocations.
103 src:
104 let
105 attrs = toSourceAttributes src;
106 in
107 fromSourceAttributes (
108 attrs // {
109 filter = path: type:
110 let
111 r = attrs.filter path type;
112 in
113 builtins.trace "${attrs.name}.filter ${path} = ${boolToString r}" r;
114 }
115 ) // {
116 satisfiesSubpathInvariant = src ? satisfiesSubpathInvariant && src.satisfiesSubpathInvariant;
117 };
118
119 # Filter sources by a list of regular expressions.
120 #
121 # E.g. `src = sourceByRegex ./my-subproject [".*\.py$" "^database.sql$"]`
122 sourceByRegex = src: regexes:
123 let
124 isFiltered = src ? _isLibCleanSourceWith;
125 origSrc = if isFiltered then src.origSrc else src;
126 in lib.cleanSourceWith {
127 filter = (path: type:
128 let relPath = lib.removePrefix (toString origSrc + "/") (toString path);
129 in lib.any (re: match re relPath != null) regexes);
130 inherit src;
131 };
132
133 /*
134 Get all files ending with the specified suffices from the given
135 source directory or its descendants, omitting files that do not match
136 any suffix. The result of the example below will include files like
137 `./dir/module.c` and `./dir/subdir/doc.xml` if present.
138
139 Type: sourceLike -> [String] -> Source
140
141 Example:
142 sourceFilesBySuffices ./. [ ".xml" ".c" ]
143 */
144 sourceFilesBySuffices =
145 # Path or source containing the files to be returned
146 src:
147 # A list of file suffix strings
148 exts:
149 let filter = name: type:
150 let base = baseNameOf (toString name);
151 in type == "directory" || lib.any (ext: lib.hasSuffix ext base) exts;
152 in cleanSourceWith { inherit filter src; };
153
154 pathIsGitRepo = path: (tryEval (commitIdFromGitRepo path)).success;
155
156 # Get the commit id of a git repo
157 # Example: commitIdFromGitRepo <nixpkgs/.git>
158 commitIdFromGitRepo =
159 let readCommitFromFile = file: path:
160 let fileName = toString path + "/" + file;
161 packedRefsName = toString path + "/packed-refs";
162 absolutePath = base: path:
163 if lib.hasPrefix "/" path
164 then path
165 else toString (/. + "${base}/${path}");
166 in if pathIsRegularFile path
167 # Resolve git worktrees. See gitrepository-layout(5)
168 then
169 let m = match "^gitdir: (.*)$" (lib.fileContents path);
170 in if m == null
171 then throw ("File contains no gitdir reference: " + path)
172 else
173 let gitDir = absolutePath (dirOf path) (lib.head m);
174 commonDir'' = if pathIsRegularFile "${gitDir}/commondir"
175 then lib.fileContents "${gitDir}/commondir"
176 else gitDir;
177 commonDir' = lib.removeSuffix "/" commonDir'';
178 commonDir = absolutePath gitDir commonDir';
179 refFile = lib.removePrefix "${commonDir}/" "${gitDir}/${file}";
180 in readCommitFromFile refFile commonDir
181
182 else if pathIsRegularFile fileName
183 # Sometimes git stores the commitId directly in the file but
184 # sometimes it stores something like: «ref: refs/heads/branch-name»
185 then
186 let fileContent = lib.fileContents fileName;
187 matchRef = match "^ref: (.*)$" fileContent;
188 in if matchRef == null
189 then fileContent
190 else readCommitFromFile (lib.head matchRef) path
191
192 else if pathIsRegularFile packedRefsName
193 # Sometimes, the file isn't there at all and has been packed away in the
194 # packed-refs file, so we have to grep through it:
195 then
196 let fileContent = readFile packedRefsName;
197 matchRef = match "([a-z0-9]+) ${file}";
198 isRef = s: isString s && (matchRef s) != null;
199 # there is a bug in libstdc++ leading to stackoverflow for long strings:
200 # https://github.com/NixOS/nix/issues/2147#issuecomment-659868795
201 refs = filter isRef (split "\n" fileContent);
202 in if refs == []
203 then throw ("Could not find " + file + " in " + packedRefsName)
204 else lib.head (matchRef (lib.head refs))
205
206 else throw ("Not a .git directory: " + path);
207 in readCommitFromFile "HEAD";
208
209 pathHasContext = builtins.hasContext or (lib.hasPrefix storeDir);
210
211 canCleanSource = src: src ? _isLibCleanSourceWith || !(pathHasContext (toString src));
212
213 # -------------------------------------------------------------------------- #
214 # Internal functions
215 #
216
217 # toSourceAttributes : sourceLike -> SourceAttrs
218 #
219 # Convert any source-like object into a simple, singular representation.
220 # We don't expose this representation in order to avoid having a fifth path-
221 # like class of objects in the wild.
222 # (Existing ones being: paths, strings, sources and x//{outPath})
223 # So instead of exposing internals, we build a library of combinator functions.
224 toSourceAttributes = src:
225 let
226 isFiltered = src ? _isLibCleanSourceWith;
227 in
228 {
229 # The original path
230 origSrc = if isFiltered then src.origSrc else src;
231 filter = if isFiltered then src.filter else _: _: true;
232 name = if isFiltered then src.name else "source";
233 };
234
235 # fromSourceAttributes : SourceAttrs -> Source
236 #
237 # Inverse of toSourceAttributes for Source objects.
238 fromSourceAttributes = { origSrc, filter, name }:
239 {
240 _isLibCleanSourceWith = true;
241 inherit origSrc filter name;
242 outPath = builtins.path { inherit filter name; path = origSrc; };
243 };
244
245in {
246 inherit
247 pathType
248 pathIsDirectory
249 pathIsRegularFile
250
251 pathIsGitRepo
252 commitIdFromGitRepo
253
254 cleanSource
255 cleanSourceWith
256 cleanSourceFilter
257 pathHasContext
258 canCleanSource
259
260 sourceByRegex
261 sourceFilesBySuffices
262
263 trace
264 ;
265}