louise
The Louise language: compiler, runtime, std, tooling
Clone avec HTTPS, un jeton personnel en mot de passe, ou avec SSH, une clé ajoutée dans vos paramètres.
Dernier runRéussi#88 · main · 15 min 30 s
0Étoiles
0Forks
40Issues
| 1 | #!/usr/bin/env bash |
| 2 | # Benchmark results over time (issue gabla108/louise#101), for a dedicated, quiet machine |
| 3 | # (.toboggan/workflows/bench.yml). Runs scripts/bench.sh and |
| 4 | # scripts/bench-compile.sh, counts each benchmark's user-space |
| 5 | # instructions with `perf stat` on one scheduler thread (the same to a few |
| 6 | # parts in 10^8 from run to run, where wall time varies by percents; |
| 7 | # skipped without perf), and writes the results as one CSV per commit: |
| 8 | # |
| 9 | # commit,benchmark,metric,value # metric: ms, compile_ms, instructions |
| 10 | # |
| 11 | # It then compares them with the previous run's and lists the |
| 12 | # regressions beyond a threshold, in REPORT (and on stdout). Regressions |
| 13 | # do not fail the script: timings are for people to judge. A benchmark |
| 14 | # whose output is wrong does (bench.sh checks it). |
| 15 | # |
| 16 | # scripts/bench-track.sh |
| 17 | # |
| 18 | # Environment: LOUISE_BENCH_DIR, where results are kept (default |
| 19 | # ~/louise-bench): <commit>.csv, <commit>.report, and `index`, the commits |
| 20 | # in the order they ran. BENCH_THRESHOLD_MS (default 10) and |
| 21 | # BENCH_THRESHOLD_INSTRUCTIONS (default 1), in percent. BENCH_RUNS, as |
| 22 | # for bench.sh (instructions are counted once). |
| 23 | set -euo pipefail |
| 24 | cd "$(dirname "$0")/.." |
| 25 | |
| 26 | dir="${LOUISE_BENCH_DIR:-$HOME/louise-bench}" |
| 27 | commit="$(git rev-parse HEAD)" |
| 28 | mkdir -p "$dir" |
| 29 | work="$(mktemp -d)" |
| 30 | trap 'rm -rf "$work"' EXIT |
| 31 | |
| 32 | # The toolchain as released (issue gabla108/louise#120). Results before this commit came |
| 33 | # from the `release` profile: the series steps there. |
| 34 | export LOUISE_PROFILE=dist |
| 35 | BENCH_BUILD_DIR="$work/programs" scripts/bench.sh --save "$work/run.txt" |
| 36 | scripts/bench-compile.sh --save "$work/compile.txt" |
| 37 | |
| 38 | csv="$work/results.csv" |
| 39 | echo "commit,benchmark,metric,value" > "$csv" |
| 40 | while read -r name ms; do echo "$commit,$name,ms,$ms"; done < "$work/run.txt" >> "$csv" |
| 41 | while read -r name ms; do echo "$commit,$name,compile_ms,$ms"; done < "$work/compile.txt" >> "$csv" |
| 42 | # User-space instructions a program runs, or nothing if perf cannot count |
| 43 | # them (not installed, no counters, or perf_event_paranoid above 2). |
| 44 | # - One scheduler thread: on 22, threads that steal or spin vary the count |
| 45 | # by 0.3-1.5%; on one, it is the same to a few parts in 10^8. |
| 46 | # - On a hybrid CPU perf reports one row per core type |
| 47 | # (`cpu_core/instructions/u`, `cpu_atom/instructions/u`), each counting |
| 48 | # while the program runs on that type: their sum is the total. |
| 49 | # `--no-scale`: by default perf scales each row up as if its counter had |
| 50 | # been multiplexed, which counted up to 30% too many, varying with the |
| 51 | # time spent on each core type. |
| 52 | instructions() { |
| 53 | command -v perf >/dev/null || return 0 |
| 54 | LOUISE_THREADS=1 perf stat --no-scale -x, -e instructions:u -o "$work/perf.txt" "$@" >/dev/null || return 1 |
| 55 | awk -F, '$3 ~ /instructions/ && $1 ~ /^[0-9]+$/ { n += $1; seen = 1 } |
| 56 | END { if (seen) print n }' "$work/perf.txt" |
| 57 | } |
| 58 | if [ -n "$(instructions true)" ]; then |
| 59 | for program in tests/bench/*.lse; do |
| 60 | name="$(basename "$program" .lse)" |
| 61 | echo "$commit,$name,instructions,$(instructions "$work/programs/$name")" >> "$csv" |
| 62 | done |
| 63 | else |
| 64 | echo "bench-track: perf cannot count instructions here, so they are not recorded" |
| 65 | fi |
| 66 | cp "$csv" "$dir/$commit.csv" |
| 67 | |
| 68 | # The previous run: the last commit in the index other than this one. |
| 69 | previous="" |
| 70 | if [ -f "$dir/index" ]; then |
| 71 | previous="$(grep -vx "$commit" "$dir/index" | tail -1 || true)" |
| 72 | fi |
| 73 | report="$dir/$commit.report" |
| 74 | if [ -z "$previous" ] || [ ! -f "$dir/$previous.csv" ]; then |
| 75 | echo "bench-track: no previous results to compare with" | tee "$report" |
| 76 | else |
| 77 | awk -F, -v ms="${BENCH_THRESHOLD_MS:-10}" -v ins="${BENCH_THRESHOLD_INSTRUCTIONS:-1}" \ |
| 78 | -v before="${previous:0:10}" -v after="${commit:0:10}" ' |
| 79 | FNR == 1 { next } |
| 80 | NR == FNR { old[$2 "," $3] = $4; next } |
| 81 | ($2 "," $3) in old { |
| 82 | o = old[$2 "," $3] |
| 83 | if (o <= 0) next |
| 84 | change = ($4 - o) * 100 / o |
| 85 | limit = ($3 == "instructions") ? ins : ms |
| 86 | if (change > limit) { |
| 87 | rows = rows sprintf("| `%s` | %s | %s | %s | +%.1f%% |\n", $2, $3, o, $4, change) |
| 88 | } |
| 89 | } |
| 90 | END { |
| 91 | if (rows == "") { printf "No regressions from %s to %s.\n", before, after; exit } |
| 92 | printf "Benchmark regressions from %s to %s (thresholds: %s%% for times, %s%% for instructions):\n\n", before, after, ms, ins |
| 93 | print "| Benchmark | Metric | Before | After | Change |" |
| 94 | print "|---|---|---|---|---|" |
| 95 | printf "%s", rows |
| 96 | }' "$dir/$previous.csv" "$csv" | tee "$report" |
| 97 | fi |
| 98 | grep -qx "$commit" "$dir/index" 2>/dev/null || echo "$commit" >> "$dir/index" |