pyrox.dev
1{
2 lib,
3 runCommand,
4 writeShellScript,
5 writeText,
6 linkFarm,
7 time,
8 procps,
9 nixVersions,
10 jq,
11 sta,
12 python3,
13}:
14
15let
16 nixpkgs =
17 with lib.fileset;
18 toSource {
19 root = ../..;
20 fileset = unions (
21 map (lib.path.append ../..) [
22 "default.nix"
23 "doc"
24 "lib"
25 "maintainers"
26 "nixos"
27 "pkgs"
28 ".version"
29 "ci/supportedSystems.json"
30 ]
31 );
32 };
33
34 nix = nixVersions.nix_2_24;
35
36 supportedSystems = builtins.fromJSON (builtins.readFile ../supportedSystems.json);
37
38 attrpathsSuperset =
39 runCommand "attrpaths-superset.json"
40 {
41 src = nixpkgs;
42 nativeBuildInputs = [
43 nix
44 time
45 ];
46 }
47 ''
48 export NIX_STATE_DIR=$(mktemp -d)
49 mkdir $out
50 export GC_INITIAL_HEAP_SIZE=4g
51 command time -f "Attribute eval done [%MKB max resident, %Es elapsed] %C" \
52 nix-instantiate --eval --strict --json --show-trace \
53 "$src/pkgs/top-level/release-attrpaths-superset.nix" \
54 -A paths \
55 -I "$src" \
56 --option restrict-eval true \
57 --option allow-import-from-derivation false \
58 --arg enableWarnings false > $out/paths.json
59 '';
60
61 singleSystem =
62 {
63 # The system to evaluate.
64 # Note that this is intentionally not called `system`,
65 # because `--argstr system` would only be passed to the ci/default.nix file!
66 evalSystem,
67 # The path to the `paths.json` file from `attrpathsSuperset`
68 attrpathFile ? "${attrpathsSuperset}/paths.json",
69 # The number of attributes per chunk, see ./README.md for more info.
70 chunkSize,
71 checkMeta ? true,
72 includeBroken ? true,
73 # Whether to just evaluate a single chunk for quick testing
74 quickTest ? false,
75 }:
76 let
77 singleChunk = writeShellScript "single-chunk" ''
78 set -euo pipefail
79 chunkSize=$1
80 myChunk=$2
81 system=$3
82 outputDir=$4
83
84 export NIX_SHOW_STATS=1
85 export NIX_SHOW_STATS_PATH="$outputDir/stats/$myChunk"
86 echo "Chunk $myChunk on $system start"
87 set +e
88 command time -o "$outputDir/timestats/$myChunk" \
89 -f "Chunk $myChunk on $system done [%MKB max resident, %Es elapsed] %C" \
90 nix-env -f "${nixpkgs}/pkgs/top-level/release-attrpaths-parallel.nix" \
91 --eval-system "$system" \
92 --option restrict-eval true \
93 --option allow-import-from-derivation false \
94 --query --available \
95 --no-name --attr-path --out-path \
96 --show-trace \
97 --arg chunkSize "$chunkSize" \
98 --arg myChunk "$myChunk" \
99 --arg attrpathFile "${attrpathFile}" \
100 --arg systems "[ \"$system\" ]" \
101 --arg checkMeta ${lib.boolToString checkMeta} \
102 --arg includeBroken ${lib.boolToString includeBroken} \
103 -I ${nixpkgs} \
104 -I ${attrpathFile} \
105 > "$outputDir/result/$myChunk" \
106 2> "$outputDir/stderr/$myChunk"
107 exitCode=$?
108 set -e
109 cat "$outputDir/stderr/$myChunk"
110 cat "$outputDir/timestats/$myChunk"
111 if (( exitCode != 0 )); then
112 echo "Evaluation failed with exit code $exitCode"
113 # This immediately halts all xargs processes
114 kill $PPID
115 elif [[ -s "$outputDir/stderr/$myChunk" ]]; then
116 echo "Nixpkgs on $system evaluated with warnings, aborting"
117 kill $PPID
118 fi
119 '';
120 in
121 runCommand "nixpkgs-eval-${evalSystem}"
122 {
123 nativeBuildInputs = [
124 nix
125 time
126 procps
127 jq
128 ];
129 env = {
130 inherit evalSystem chunkSize;
131 };
132 }
133 ''
134 export NIX_STATE_DIR=$(mktemp -d)
135 nix-store --init
136
137 echo "System: $evalSystem"
138 cores=$NIX_BUILD_CORES
139 echo "Cores: $cores"
140 attrCount=$(jq length "${attrpathFile}")
141 echo "Attribute count: $attrCount"
142 echo "Chunk size: $chunkSize"
143 # Same as `attrCount / chunkSize` but rounded up
144 chunkCount=$(( (attrCount - 1) / chunkSize + 1 ))
145 echo "Chunk count: $chunkCount"
146
147 mkdir $out
148
149 # Record and print stats on free memory and swap in the background
150 (
151 while true; do
152 availMemory=$(free -b | grep Mem | awk '{print $7}')
153 freeSwap=$(free -b | grep Swap | awk '{print $4}')
154 echo "Available memory: $(( availMemory / 1024 / 1024 )) MiB, free swap: $(( freeSwap / 1024 / 1024 )) MiB"
155
156 if [[ ! -f "$out/min-avail-memory" ]] || (( availMemory < $(<$out/min-avail-memory) )); then
157 echo "$availMemory" > $out/min-avail-memory
158 fi
159 if [[ ! -f $out/min-free-swap ]] || (( availMemory < $(<$out/min-free-swap) )); then
160 echo "$freeSwap" > $out/min-free-swap
161 fi
162 sleep 4
163 done
164 ) &
165
166 seq_end=$(( chunkCount - 1 ))
167
168 ${lib.optionalString quickTest ''
169 seq_end=0
170 ''}
171
172 chunkOutputDir=$(mktemp -d)
173 mkdir "$chunkOutputDir"/{result,stats,timestats,stderr}
174
175 seq -w 0 "$seq_end" |
176 command time -f "%e" -o "$out/total-time" \
177 xargs -I{} -P"$cores" \
178 ${singleChunk} "$chunkSize" {} "$evalSystem" "$chunkOutputDir"
179
180 cp -r "$chunkOutputDir"/stats $out/stats-by-chunk
181
182 if (( chunkSize * chunkCount != attrCount )); then
183 # A final incomplete chunk would mess up the stats, don't include it
184 rm "$chunkOutputDir"/stats/"$seq_end"
185 fi
186
187 # Make sure the glob doesn't break when there's no files
188 shopt -s nullglob
189 cat "$chunkOutputDir"/result/* > $out/paths
190 cat "$chunkOutputDir"/stats/* > $out/stats.jsonstream
191 '';
192
193 combine =
194 {
195 resultsDir,
196 }:
197 runCommand "combined-result"
198 {
199 nativeBuildInputs = [
200 jq
201 sta
202 ];
203 }
204 ''
205 mkdir -p $out
206
207 # Transform output paths to JSON
208 cat ${resultsDir}/*/paths |
209 jq --sort-keys --raw-input --slurp '
210 split("\n") |
211 map(select(. != "") | split(" ") | map(select(. != ""))) |
212 map(
213 {
214 key: .[0],
215 value: .[1] | split(";") | map(split("=") |
216 if length == 1 then
217 { key: "out", value: .[0] }
218 else
219 { key: .[0], value: .[1] }
220 end) | from_entries}
221 ) | from_entries
222 ' > $out/outpaths.json
223
224 # Computes min, mean, error, etc. for a list of values and outputs a JSON from that
225 statistics() {
226 local stat=$1
227 sta --transpose |
228 jq --raw-input --argjson stat "$stat" -n '
229 [
230 inputs |
231 split("\t") |
232 { key: .[0], value: (.[1] | fromjson) }
233 ] |
234 from_entries |
235 {
236 key: ($stat | join(".")),
237 value: .
238 }'
239 }
240
241 # Gets all available number stats (without .sizes because those are constant and not interesting)
242 readarray -t stats < <(jq -cs '.[0] | del(.sizes) | paths(type == "number")' ${resultsDir}/*/stats.jsonstream)
243
244 # Combines the statistics from all evaluations
245 {
246 echo "{ \"key\": \"minAvailMemory\", \"value\": $(cat ${resultsDir}/*/min-avail-memory | sta --brief --min) }"
247 echo "{ \"key\": \"minFreeSwap\", \"value\": $(cat ${resultsDir}/*/min-free-swap | sta --brief --min) }"
248 cat ${resultsDir}/*/total-time | statistics '["totalTime"]'
249 for stat in "''${stats[@]}"; do
250 cat ${resultsDir}/*/stats.jsonstream |
251 jq --argjson stat "$stat" 'getpath($stat)' |
252 statistics "$stat"
253 done
254 } |
255 jq -s from_entries > $out/stats.json
256
257 mkdir -p $out/stats
258
259 for d in ${resultsDir}/*; do
260 cp -r "$d"/stats-by-chunk $out/stats/$(basename "$d")
261 done
262 '';
263
264 compare = import ./compare {
265 inherit
266 lib
267 jq
268 runCommand
269 writeText
270 supportedSystems
271 python3
272 ;
273 };
274
275 full =
276 {
277 # Whether to evaluate on a specific set of systems, by default all are evaluated
278 evalSystems ? if quickTest then [ "x86_64-linux" ] else supportedSystems,
279 # The number of attributes per chunk, see ./README.md for more info.
280 chunkSize,
281 quickTest ? false,
282 }:
283 let
284 results = linkFarm "results" (
285 map (evalSystem: {
286 name = evalSystem;
287 path = singleSystem {
288 inherit quickTest evalSystem chunkSize;
289 };
290 }) evalSystems
291 );
292 in
293 combine {
294 resultsDir = results;
295 };
296
297in
298{
299 inherit
300 attrpathsSuperset
301 singleSystem
302 combine
303 compare
304 # The above three are used by separate VMs in a GitHub workflow,
305 # while the below is intended for testing on a single local machine
306 full
307 ;
308}