pyrox.dev
1{
2 lib,
3 fetchFromGitHub,
4 runCommand,
5 yallback,
6 yara,
7}:
8
9/*
10 TODO/CAUTION:
11
12 I don't want to discourage use, but I'm not sure how stable
13 the API is. Have fun, but be prepared to track changes! :)
14
15 For _now_, binlore is basically a thin wrapper around
16 `<invoke yara> | <postprocess with yallback>` with support
17 for running it on a derivation, saving the result in the
18 store, and aggregating results from a set of packages.
19
20 In the longer term, I suspect there are more uses for this
21 general pattern (i.e., run some analysis tool that produces
22 a deterministic output and cache the result per package...).
23
24 I'm not sure how that'll look and if it'll be the case that
25 binlore automatically collects all of them, or if you'll be
26 configuring which "kind(s)" of lore it generates. Nailing
27 that down will almost certainly mean reworking the API.
28*/
29
30let
31 src = fetchFromGitHub {
32 owner = "abathur";
33 repo = "binlore";
34 rev = "v0.3.0";
35 hash = "sha256-4Fs6HThfDhKRskuDJx2+hucl8crMRm10K6949JdIwPY=";
36 };
37 /*
38 binlore has one one more yallbacks responsible for
39 routing the appropriate lore to a named file in the
40 appropriate format. At some point I might try to do
41 something fancy with this, but for now the answer to
42 *all* questions about the lore are: the bare minimum
43 to get resholve over the next feature hump in time to
44 hopefully slip this feature in before the branch-off.
45 */
46 # TODO: feeling really uninspired on the API
47 loreDef = {
48 # YARA rule file
49 rules = (src + "/execers.yar");
50 # output filenames; "types" of lore
51 types = [
52 "execers"
53 "wrappers"
54 ];
55 # shell rule callbacks; see github.com/abathur/yallback
56 yallback = (src + "/execers.yall");
57 # TODO:
58 # - echo for debug, can be removed at some point
59 # - I really just wanted to put the bit after the pipe
60 # in here, but I'm erring on the side of flexibility
61 # since this form will make it easier to pilot other
62 # uses of binlore.
63 callback = lore: drv: ''
64 if [[ -d "${drv}/bin" ]] || [[ -d "${drv}/lib" ]] || [[ -d "${drv}/libexec" ]]; then
65 echo generating binlore for $drv by running:
66 echo "${yara}/bin/yara --scan-list --recursive ${lore.rules} <(printf '%s\n' ${drv}/{bin,lib,libexec}) | ${yallback}/bin/yallback ${lore.yallback}"
67 else
68 echo "failed to generate binlore for $drv (none of ${drv}/{bin,lib,libexec} exist)"
69 fi
70
71 if [[ -d "${drv}/bin" ]] || [[ -d "${drv}/lib" ]] || [[ -d "${drv}/libexec" ]]; then
72 ${yara}/bin/yara --scan-list --recursive ${lore.rules} <(printf '%s\n' ${drv}/{bin,lib,libexec}) | ${yallback}/bin/yallback ${lore.yallback}
73 fi
74 '';
75 };
76
77in
78rec {
79 /*
80 Output a directory containing lore for multiple drvs.
81
82 This will `make` lore for drv in drvs and then combine lore
83 of the same type across all packages into a single file.
84
85 When drvs are also specified in the strip argument, corresponding
86 lore is made relative by stripping the path of each drv from
87 matching entries. (This is mainly useful in a build process that
88 uses a chain of two or more derivations where the output of one
89 is the source for the next. See resholve for an example.)
90 */
91 collect =
92 {
93 lore ? loreDef,
94 drvs,
95 strip ? [ ],
96 }:
97 (runCommand "more-binlore" { } ''
98 mkdir $out
99 for lorefile in ${toString lore.types}; do
100 cat ${
101 lib.concatMapStrings (x: x + "/$lorefile ") (
102 map (make lore) (map lib.getBin (builtins.filter lib.isDerivation drvs))
103 )
104 } > $out/$lorefile
105 substituteInPlace $out/$lorefile ${lib.concatMapStrings (x: "--replace-quiet '${x}/' '' ") strip}
106 done
107 '');
108
109 /*
110 Output a directory containing lore for a single drv.
111
112 This produces lore for the derivation (via lore.callback) and
113 appends any lore that the derivation itself wrote to nix-support
114 or which was overridden in drv.binlore.<outputName> (passthru).
115
116 > *Note*: Since the passthru is attached to all outputs, binlore
117 > is an attrset namespaced by outputName to support packages with
118 > executables in more than one output.
119
120 Since the last entry wins, the effective priority is:
121 drv.binlore.<outputName> > $drv/nix-support > lore generated here by callback
122 */
123 make =
124 lore: drv:
125 runCommand "${drv.name}-binlore"
126 {
127 drv = drv;
128 }
129 (
130 ''
131 mkdir $out
132 touch $out/{${builtins.concatStringsSep "," lore.types}}
133
134 ${lore.callback lore drv}
135 ''
136 +
137 # append lore from package's $out and drv.binlore.${drv.outputName} (last entry wins)
138 ''
139 for lore_type in ${builtins.toString lore.types}; do
140 if [[ -f "${drv}/nix-support/$lore_type" ]]; then
141 cat "${drv}/nix-support/$lore_type" >> "$out/$lore_type"
142 fi
143 ''
144 +
145 lib.optionalString (builtins.hasAttr "binlore" drv && builtins.hasAttr drv.outputName drv.binlore)
146 ''
147 if [[ -f "${drv.binlore."${drv.outputName}"}/$lore_type" ]]; then
148 cat "${drv.binlore."${drv.outputName}"}/$lore_type" >> "$out/$lore_type"
149 fi
150 ''
151 + ''
152 done
153
154 echo binlore for $drv written to $out
155 ''
156 );
157
158 /*
159 Utility function for creating override lore for drv.
160
161 We normally attach this lore to `drv.passthru.binlore.<outputName>`.
162
163 > *Notes*:
164 > - Since the passthru is attached to all outputs, binlore is an
165 > attrset namespaced by outputName to support packages with
166 > executables in more than one output. You'll generally just use
167 > `out` or `bin`.
168 > - We can reconsider the passthru attr name if someone adds
169 > a new lore provider. We settled on `.binlore` for now to make it
170 > easier for people to figure out what this is for.
171
172 The lore argument should be a Shell script (string) that generates
173 the necessary lore. You can use arbitrary Shell, but this function
174 includes a shell DSL you can use to declare/generate lore in most
175 cases. It has the following functions:
176
177 - `execer <verdict> [<path>...]`
178 - `wrapper <wrapper_path> <original_path>`
179
180 Writing every override explicitly in a Nix list would be tedious
181 for large packages, but this small shell DSL enables us to express
182 many overrides efficiently via pathname expansion/globbing.
183
184 Here's a very general example of both functions:
185
186 passthru.binlore.out = binlore.synthesize finalAttrs.finalPackage ''
187 execer can bin/hello bin/{a,b,c}
188 wrapper bin/hello bin/.hello-wrapped
189 '';
190
191 And here's a specific example of how pathname expansion enables us
192 to express lore for the single-binary variant of coreutils while
193 being both explicit and (somewhat) efficient:
194
195 passthru = {} // optionalAttrs (singleBinary != false) {
196 binlore.out = binlore.synthesize coreutils ''
197 execer can bin/{chroot,env,install,nice,nohup,runcon,sort,split,stdbuf,timeout}
198 execer cannot bin/{[,b2sum,base32,base64,basename,basenc,cat,chcon,chgrp,chmod,chown,cksum,comm,cp,csplit,cut,date,dd,df,dir,dircolors,dirname,du,echo,expand,expr,factor,false,fmt,fold,groups,head,hostid,id,join,kill,link,ln,logname,ls,md5sum,mkdir,mkfifo,mknod,mktemp,mv,nl,nproc,numfmt,od,paste,pathchk,pinky,pr,printenv,printf,ptx,pwd,readlink,realpath,rm,rmdir,seq,sha1sum,sha224sum,sha256sum,sha384sum,sha512sum,shred,shuf,sleep,stat,stty,sum,sync,tac,tail,tee,test,touch,tr,true,truncate,tsort,tty,uname,unexpand,uniq,unlink,uptime,users,vdir,wc,who,whoami,yes}
199 '';
200 };
201
202 Caution: Be thoughtful about using a bare wildcard (*) glob here.
203 We should generally override lore only when a human understands if
204 the executable will exec arbitrary user-passed executables. A bare
205 glob can match new executables added in future package versions
206 before anyone can audit them.
207 */
208 synthesize =
209 drv: loreSynthesizingScript:
210 runCommand "${drv.name}-lore-override"
211 {
212 drv = drv;
213 }
214 (
215 ''
216 execer(){
217 local verdict="$1"
218
219 shift
220
221 for path in "$@"; do
222 if [[ -f "$PWD/$path" ]]; then
223 echo "$verdict:$PWD/$path"
224 else
225 echo "error: Tried to synthesize execer lore for missing file: $PWD/$path" >&2
226 exit 2
227 fi
228 done
229 } >> $out/execers
230
231 wrapper(){
232 local wrapper="$1"
233 local original="$2"
234
235 if [[ ! -f "$wrapper" ]]; then
236 echo "error: Tried to synthesize wrapper lore for missing wrapper: $PWD/$wrapper" >&2
237 exit 2
238 fi
239
240 if [[ ! -f "$original" ]]; then
241 echo "error: Tried to synthesize wrapper lore for missing original: $PWD/$original" >&2
242 exit 2
243 fi
244
245 echo "$PWD/$wrapper:$PWD/$original"
246
247 } >> $out/wrappers
248
249 mkdir $out
250
251 # lore override commands are relative to the drv root
252 cd $drv
253
254 ''
255 + loreSynthesizingScript
256 );
257}