tobogganSe connecter

louise

Clone avec HTTPS, un jeton personnel en mot de passe, ou avec SSH, une clé ajoutée dans vos paramètres.

98 lignes · 4473 octetsBrut · Historique
1#!/usr/bin/env bash
2# Benchmark results over time (issue #101), for a dedicated, quiet machine
3# (.toboggan/workflows/bench.yml). Runs scripts/bench.sh and
4# scripts/bench-compile.sh, counts each benchmark's user-space
5# instructions with `perf stat` on one scheduler thread (the same to a few
6# parts in 10^8 from run to run, where wall time varies by percents;
7# skipped without perf), and writes the results as one CSV per commit:
8#
9# commit,benchmark,metric,value # metric: ms, compile_ms, instructions
10#
11# It then compares them with the previous run's and lists the
12# regressions beyond a threshold, in REPORT (and on stdout). Regressions
13# do not fail the script: timings are for people to judge. A benchmark
14# whose output is wrong does (bench.sh checks it).
15#
16# scripts/bench-track.sh
17#
18# Environment: LOUISE_BENCH_DIR, where results are kept (default
19# ~/louise-bench): <commit>.csv, <commit>.report, and `index`, the commits
20# in the order they ran. BENCH_THRESHOLD_MS (default 10) and
21# BENCH_THRESHOLD_INSTRUCTIONS (default 1), in percent. BENCH_RUNS, as
22# for bench.sh (instructions are counted once).
23set -euo pipefail
24cd "$(dirname "$0")/.."
25
26dir="${LOUISE_BENCH_DIR:-$HOME/louise-bench}"
27commit="$(git rev-parse HEAD)"
28mkdir -p "$dir"
29work="$(mktemp -d)"
30trap 'rm -rf "$work"' EXIT
31
32# The toolchain as released (issue #120). Results before this commit came
33# from the `release` profile: the series steps there.
34export LOUISE_PROFILE=dist
35BENCH_BUILD_DIR="$work/programs" scripts/bench.sh --save "$work/run.txt"
36scripts/bench-compile.sh --save "$work/compile.txt"
37
38csv="$work/results.csv"
39echo "commit,benchmark,metric,value" > "$csv"
40while read -r name ms; do echo "$commit,$name,ms,$ms"; done < "$work/run.txt" >> "$csv"
41while read -r name ms; do echo "$commit,$name,compile_ms,$ms"; done < "$work/compile.txt" >> "$csv"
42# User-space instructions a program runs, or nothing if perf cannot count
43# them (not installed, no counters, or perf_event_paranoid above 2).
44# - One scheduler thread: on 22, threads that steal or spin vary the count
45# by 0.3-1.5%; on one, it is the same to a few parts in 10^8.
46# - On a hybrid CPU perf reports one row per core type
47# (`cpu_core/instructions/u`, `cpu_atom/instructions/u`), each counting
48# while the program runs on that type: their sum is the total.
49# `--no-scale`: by default perf scales each row up as if its counter had
50# been multiplexed, which counted up to 30% too many, varying with the
51# time spent on each core type.
52instructions() {
53 command -v perf >/dev/null || return 0
54 LOUISE_THREADS=1 perf stat --no-scale -x, -e instructions:u -o "$work/perf.txt" "$@" >/dev/null || return 1
55 awk -F, '$3 ~ /instructions/ && $1 ~ /^[0-9]+$/ { n += $1; seen = 1 }
56 END { if (seen) print n }' "$work/perf.txt"
57}
58if [ -n "$(instructions true)" ]; then
59 for program in tests/bench/*.lse; do
60 name="$(basename "$program" .lse)"
61 echo "$commit,$name,instructions,$(instructions "$work/programs/$name")" >> "$csv"
62 done
63else
64 echo "bench-track: perf cannot count instructions here, so they are not recorded"
65fi
66cp "$csv" "$dir/$commit.csv"
67
68# The previous run: the last commit in the index other than this one.
69previous=""
70if [ -f "$dir/index" ]; then
71 previous="$(grep -vx "$commit" "$dir/index" | tail -1 || true)"
72fi
73report="$dir/$commit.report"
74if [ -z "$previous" ] || [ ! -f "$dir/$previous.csv" ]; then
75 echo "bench-track: no previous results to compare with" | tee "$report"
76else
77 awk -F, -v ms="${BENCH_THRESHOLD_MS:-10}" -v ins="${BENCH_THRESHOLD_INSTRUCTIONS:-1}" \
78 -v before="${previous:0:10}" -v after="${commit:0:10}" '
79 FNR == 1 { next }
80 NR == FNR { old[$2 "," $3] = $4; next }
81 ($2 "," $3) in old {
82 o = old[$2 "," $3]
83 if (o <= 0) next
84 change = ($4 - o) * 100 / o
85 limit = ($3 == "instructions") ? ins : ms
86 if (change > limit) {
87 rows = rows sprintf("| `%s` | %s | %s | %s | +%.1f%% |\n", $2, $3, o, $4, change)
88 }
89 }
90 END {
91 if (rows == "") { printf "No regressions from %s to %s.\n", before, after; exit }
92 printf "Benchmark regressions from %s to %s (thresholds: %s%% for times, %s%% for instructions):\n\n", before, after, ms, ins
93 print "| Benchmark | Metric | Before | After | Change |"
94 print "|---|---|---|---|---|"
95 printf "%s", rows
96 }' "$dir/$previous.csv" "$csv" | tee "$report"
97fi
98grep -qx "$commit" "$dir/index" 2>/dev/null || echo "$commit" >> "$dir/index"