commit c9c9c093cfbc7b32be5579ca21312c6b5e093d66 · pyrox.dev/nixpkgs

+36 -5
lib/fileset/benchmark.sh
···

       45
       45
        
           ".values.number"

     

       46
       46
        
       )

     

       47
       47
        
       

     

       48
       48
       -
       # TODO: Measure time

     

       48
       48
       +
       runs=10

     

       49
       49
       +
       

     

       49
       50
        
       run() {

     

       50
       50
       -
           NIX_PATH=nixpkgs=$1 NIX_SHOW_STATS=1 NIX_SHOW_STATS_PATH=$tmp/stats.json \

     

       51
       51
       -
               nix-instantiate --eval --strict --show-trace >/dev/null \

     

       52
       52
       -
               --expr 'with import <nixpkgs/lib>; with fileset; '"$2"

     

       53
       53
       -
           cat "$tmp/stats.json"

     

       51
       51
       +
           # Empty the file

     

       52
       52
       +
           : > cpuTimes

     

       53
       53
       +
       

     

       54
       54
       +
           for i in $(seq 0 "$runs"); do

     

       55
       55
       +
               NIX_PATH=nixpkgs=$1 NIX_SHOW_STATS=1 NIX_SHOW_STATS_PATH=$tmp/stats.json \

     

       56
       56
       +
                   nix-instantiate --eval --strict --show-trace >/dev/null \

     

       57
       57
       +
                   --expr 'with import <nixpkgs/lib>; with fileset; '"$2"

     

       58
       58
       +
       

     

       59
       59
       +
               # Only measure the time after the first run, one is warmup

     

       60
       60
       +
               if (( i > 0 )); then

     

       61
       61
       +
                   jq '.cpuTime' "$tmp/stats.json" >> cpuTimes

     

       62
       62
       +
               fi

     

       63
       63
       +
           done

     

       64
       64
       +
       

     

       65
       65
       +
           # Compute mean and standard deviation

     

       66
       66
       +
           read -r mean sd < <(sta --mean --sd --brief <cpuTimes)

     

       67
       67
       +
       

     

       68
       68
       +
           jq --argjson mean "$mean" --argjson sd "$sd" \

     

       69
       69
       +
               '.cpuTimeMean = $mean | .cpuTimeSd = $sd' \

     

       70
       70
       +
               "$tmp/stats.json"

     

       54
       71
        
       }

     

       55
       72
        
       

     

       56
       73
        
       bench() {

     
···

       64
       81
        
               #echo "Running benchmark on $compareTo" >&2

     

       65
       82
        
               run "$tmp/worktree" "$1" > "$tmp/old.json"

     

       66
       83
        
           )

     

       84
       84
       +
       

     

       85
       85
       +
           read -r oldMean oldSd newMean newSd percentageMean percentageSd < \

     

       86
       86
       +
               <(jq -rn --slurpfile old "$tmp/old.json" --slurpfile new "$tmp/new.json" \

     

       87
       87
       +
               ' $old[0].cpuTimeMean as $om

     

       88
       88
       +
               | $old[0].cpuTimeSd as $os

     

       89
       89
       +
               | $new[0].cpuTimeMean as $nm

     

       90
       90
       +
               | $new[0].cpuTimeSd as $ns

     

       91
       91
       +
               | (100 / $om * $nm) as $pm

     

       92
       92
       +
               # Copied from https://github.com/sharkdp/hyperfine/blob/b38d550b89b1dab85139eada01c91a60798db9cc/src/benchmark/relative_speed.rs#L46-L53

     

       93
       93
       +
               | ($pm * pow(pow($ns / $nm; 2) + pow($os / $om; 2); 0.5)) as $ps

     

       94
       94
       +
               | [ $om, $os, $nm, $ns, $pm, $ps ]

     

       95
       95
       +
               | @sh')

     

       96
       96
       +
       

     

       97
       97
       +
           echo -e "Mean CPU time $newMean (σ = $newSd) for $runs runs is \e[0;33m$percentageMean% (σ = $percentageSd%)\e[0m of the old value $oldMean (σ = $oldSd)" >&2

     

       67
       98
        
       

     

       68
       99
        
           different=0

     

       69
       100
        
           for stat in "${stats[@]}"; do