pyrox.dev
1#!/usr/bin/env nix-shell
2#!nix-shell -i bash -p sta jq bc nix -I nixpkgs=../..
3# shellcheck disable=SC2016
4
5# Benchmarks lib.fileset
6# Run:
7# [nixpkgs]$ lib/fileset/benchmark.sh HEAD
8
9set -euo pipefail
10shopt -s inherit_errexit dotglob
11
12if (( $# == 0 )); then
13 echo "Usage: $0 HEAD"
14 echo "Benchmarks the current tree against the HEAD commit. Any git ref will work."
15 exit 1
16fi
17compareTo=$1
18
19SCRIPT_FILE=$(readlink -f "${BASH_SOURCE[0]}")
20SCRIPT_DIR=$(dirname "$SCRIPT_FILE")
21
22nixpkgs=$(cd "$SCRIPT_DIR/../.."; pwd)
23
24tmp="$(mktemp -d)"
25clean_up() {
26 rm -rf "$tmp"
27}
28trap clean_up EXIT SIGINT SIGTERM
29work="$tmp/work"
30mkdir "$work"
31cd "$work"
32
33declare -a stats=(
34 ".envs.elements"
35 ".envs.number"
36 ".gc.totalBytes"
37 ".list.concats"
38 ".list.elements"
39 ".nrFunctionCalls"
40 ".nrLookups"
41 ".nrOpUpdates"
42 ".nrPrimOpCalls"
43 ".nrThunks"
44 ".sets.elements"
45 ".sets.number"
46 ".symbols.number"
47 ".values.number"
48)
49
50runs=10
51
52run() {
53 # Empty the file
54 : > cpuTimes
55
56 for i in $(seq 0 "$runs"); do
57 NIX_PATH=nixpkgs=$1 NIX_SHOW_STATS=1 NIX_SHOW_STATS_PATH=$tmp/stats.json \
58 nix-instantiate --eval --strict --show-trace >/dev/null \
59 --expr 'with import <nixpkgs/lib>; with fileset; '"$2"
60
61 # Only measure the time after the first run, one is warmup
62 if (( i > 0 )); then
63 jq '.cpuTime' "$tmp/stats.json" >> cpuTimes
64 fi
65 done
66
67 # Compute mean and standard deviation
68 read -r mean sd < <(sta --mean --sd --brief <cpuTimes)
69
70 jq --argjson mean "$mean" --argjson sd "$sd" \
71 '.cpuTimeMean = $mean | .cpuTimeSd = $sd' \
72 "$tmp/stats.json"
73}
74
75bench() {
76 echo "Benchmarking expression $1" >&2
77 #echo "Running benchmark on index" >&2
78 run "$nixpkgs" "$1" > "$tmp/new.json"
79 (
80 #echo "Checking out $compareTo" >&2
81 git -C "$nixpkgs" worktree add --quiet "$tmp/worktree" "$compareTo"
82 trap 'git -C "$nixpkgs" worktree remove "$tmp/worktree"' EXIT
83 #echo "Running benchmark on $compareTo" >&2
84 run "$tmp/worktree" "$1" > "$tmp/old.json"
85 )
86
87 read -r oldMean oldSd newMean newSd percentageMean percentageSd < \
88 <(jq -rn --slurpfile old "$tmp/old.json" --slurpfile new "$tmp/new.json" \
89 ' $old[0].cpuTimeMean as $om
90 | $old[0].cpuTimeSd as $os
91 | $new[0].cpuTimeMean as $nm
92 | $new[0].cpuTimeSd as $ns
93 | (100 / $om * $nm) as $pm
94 # Copied from https://github.com/sharkdp/hyperfine/blob/b38d550b89b1dab85139eada01c91a60798db9cc/src/benchmark/relative_speed.rs#L46-L53
95 | ($pm * pow(pow($ns / $nm; 2) + pow($os / $om; 2); 0.5)) as $ps
96 | [ $om, $os, $nm, $ns, $pm, $ps ]
97 | @sh')
98
99 echo -e "Mean CPU time $newMean (σ = $newSd) for $runs runs is \e[0;33m$percentageMean% (σ = $percentageSd%)\e[0m of the old value $oldMean (σ = $oldSd)" >&2
100
101 different=0
102 for stat in "${stats[@]}"; do
103 oldValue=$(jq "$stat" "$tmp/old.json")
104 newValue=$(jq "$stat" "$tmp/new.json")
105 if (( oldValue != newValue )); then
106 percent=$(bc <<< "scale=100; result = 100/$oldValue*$newValue; scale=4; result / 1")
107 if (( oldValue < newValue )); then
108 echo -e "Statistic $stat ($newValue) is \e[0;31m$percent% (+$(( newValue - oldValue )))\e[0m of the old value $oldValue" >&2
109 else
110 echo -e "Statistic $stat ($newValue) is \e[0;32m$percent% (-$(( oldValue - newValue )))\e[0m of the old value $oldValue" >&2
111 fi
112 (( different++ )) || true
113 fi
114 done
115 echo "$different stats differ between the current tree and $compareTo"
116 echo ""
117}
118
119# Create a fairly populated tree
120touch f{0..5}
121mkdir d{0..5}
122mkdir e{0..5}
123touch d{0..5}/f{0..5}
124mkdir -p d{0..5}/d{0..5}
125mkdir -p e{0..5}/e{0..5}
126touch d{0..5}/d{0..5}/f{0..5}
127mkdir -p d{0..5}/d{0..5}/d{0..5}
128mkdir -p e{0..5}/e{0..5}/e{0..5}
129touch d{0..5}/d{0..5}/d{0..5}/f{0..5}
130mkdir -p d{0..5}/d{0..5}/d{0..5}/d{0..5}
131mkdir -p e{0..5}/e{0..5}/e{0..5}/e{0..5}
132touch d{0..5}/d{0..5}/d{0..5}/d{0..5}/f{0..5}
133
134bench 'toSource { root = ./.; fileset = ./.; }'
135
136rm -rf -- *
137
138touch {0..1000}
139bench 'toSource { root = ./.; fileset = unions (mapAttrsToList (name: value: ./. + "/${name}") (builtins.readDir ./.)); }'
140rm -rf -- *