louise
The Louise language: compiler, runtime, std, tooling
Clone avec HTTPS, un jeton personnel en mot de passe, ou avec SSH, une clé ajoutée dans vos paramètres.
Dernier runRéussi#88 · main · 15 min 30 s
0Étoiles
0Forks
40Issues
| 1 | #!/usr/bin/env bash |
| 2 | # Build-to-build variance of the benchmarks (issue #193): with what |
| 3 | # layout does identical C run at the same speed? |
| 4 | # |
| 5 | # Builds every benchmark from scratch VARIANCE_RUNS times per |
| 6 | # configuration — a fresh build directory each, so ThinLTO places the |
| 7 | # code anew — once with the fixed layout and once without |
| 8 | # (LOUISE_NO_ALIGN=1), then runs all the builds interleaved (each in |
| 9 | # turn, VARIANCE_REPS rounds), so a slow drift of the machine touches |
| 10 | # every build alike. Reports, per benchmark and configuration, the |
| 11 | # spread between the builds' best times as a percentage of the best. |
| 12 | # The done-when: the aligned spread stays around 2%. Wall clock is what |
| 13 | # layout affects (instruction counts do not see it); pin the runs, or |
| 14 | # the scheduler's noise swamps the signal. |
| 15 | # |
| 16 | # scripts/bench-variance.sh |
| 17 | # |
| 18 | # Environment: VARIANCE_RUNS (builds per configuration, default 3), |
| 19 | # VARIANCE_REPS (interleaved rounds, default 5), BENCH_PIN (a CPU list |
| 20 | # for taskset, e.g. "0"), LOUISE_PROFILE (default dist), LOUISE_THREADS |
| 21 | # (default 1: layout, not scheduling, is what this measures). |
| 22 | set -euo pipefail |
| 23 | cd "$(dirname "$0")/.." |
| 24 | source scripts/lib.sh |
| 25 | |
| 26 | builds="${VARIANCE_RUNS:-3}" |
| 27 | reps="${VARIANCE_REPS:-5}" |
| 28 | export LOUISE_PROFILE="${LOUISE_PROFILE:-dist}" |
| 29 | export LOUISE_THREADS="${LOUISE_THREADS:-1}" |
| 30 | |
| 31 | build_release_louise |
| 32 | runner=() |
| 33 | if [ -n "${BENCH_PIN:-}" ]; then runner=(taskset -c "$BENCH_PIN"); fi |
| 34 | |
| 35 | work="$(mktemp -d)" |
| 36 | trap 'rm -rf "$work"' EXIT |
| 37 | |
| 38 | # perf_cycles OUTFILE CMD…: CMD under `perf stat`, its stdout to OUTFILE; |
| 39 | # prints the run's user cycles summed over core types (a hybrid CPU |
| 40 | # reports one row each; --no-scale, as scripts/bench-track.sh), nothing |
| 41 | # where perf cannot count. Wall clock has a 1 ms granularity, which is |
| 42 | # percents of the fastest benchmarks; cycles resolve the layout signal. |
| 43 | perf_cycles() { |
| 44 | local out=$1 |
| 45 | shift |
| 46 | perf stat --no-scale -x, -e cycles:u -o "$work/perf.txt" "$@" > "$out" 2>/dev/null || return 1 |
| 47 | awk -F, '$3 ~ /cycles/ && $1 ~ /^[0-9]+$/ { n += $1; seen = 1 } |
| 48 | END { if (seen) print n }' "$work/perf.txt" |
| 49 | } |
| 50 | have_perf=1 |
| 51 | if ! command -v perf >/dev/null || [ -z "$(perf_cycles /dev/null true)" ]; then |
| 52 | have_perf= |
| 53 | echo "bench-variance: perf cannot count cycles here; wall clock only" >&2 |
| 54 | fi |
| 55 | |
| 56 | programs=(tests/bench/*.lse) |
| 57 | for config in align noalign; do |
| 58 | for b in $(seq "$builds"); do |
| 59 | out="$work/$config.$b" |
| 60 | mkdir -p "$out" |
| 61 | for program in "${programs[@]}"; do |
| 62 | name="$(basename "$program" .lse)" |
| 63 | echo "bench-variance: building $config.$b/$name" >&2 |
| 64 | if [ "$config" = noalign ]; then |
| 65 | LOUISE_NO_ALIGN=1 "$LOUISE_BIN" build --release "$program" -o "$out/$name" >/dev/null |
| 66 | else |
| 67 | "$LOUISE_BIN" build --release "$program" -o "$out/$name" >/dev/null |
| 68 | fi |
| 69 | expected_output "$program" > "$out/$name.expected" |
| 70 | done |
| 71 | done |
| 72 | done |
| 73 | |
| 74 | for round in $(seq "$reps"); do |
| 75 | echo "bench-variance: round $round of $reps" >&2 |
| 76 | for program in "${programs[@]}"; do |
| 77 | name="$(basename "$program" .lse)" |
| 78 | for config in align noalign; do |
| 79 | for b in $(seq "$builds"); do |
| 80 | bin="$work/$config.$b/$name" |
| 81 | start=$(date +%s%N) |
| 82 | c="" |
| 83 | if [ -n "$have_perf" ]; then |
| 84 | c=$(perf_cycles "$bin.out" "${runner[@]}" "$bin") |
| 85 | else |
| 86 | "${runner[@]}" "$bin" > "$bin.out" |
| 87 | fi |
| 88 | ms=$((($(date +%s%N) - start) / 1000000)) |
| 89 | if ! diff -q "$bin.expected" "$bin.out" >/dev/null; then |
| 90 | echo "bench-variance: $name ($config.$b): wrong output" >&2 |
| 91 | exit 1 |
| 92 | fi |
| 93 | echo "$name $config $b $ms" >> "$work/runs.ms" |
| 94 | if [ -n "$c" ]; then echo "$name $config $b $c" >> "$work/runs.cycles"; fi |
| 95 | done |
| 96 | done |
| 97 | done |
| 98 | done |
| 99 | |
| 100 | report() { |
| 101 | printf "\n%-16s %18s %18s\n" benchmark "align $1" "no-align $1" |
| 102 | awk ' |
| 103 | { key = $1 SUBSEP $2 SUBSEP $3 |
| 104 | if (!(key in min) || $4 < min[key]) min[key] = $4 |
| 105 | names[$1] = 1 } |
| 106 | END { |
| 107 | for (key in min) { |
| 108 | split(key, a, SUBSEP) |
| 109 | nc = a[1] SUBSEP a[2] |
| 110 | if (!(nc in best) || min[key] < best[nc]) best[nc] = min[key] |
| 111 | if (min[key] > worst[nc]) worst[nc] = min[key] |
| 112 | } |
| 113 | for (n in names) { |
| 114 | sa = (worst[n, "align"] - best[n, "align"]) * 100.0 / best[n, "align"] |
| 115 | sn = (worst[n, "noalign"] - best[n, "noalign"]) * 100.0 / best[n, "noalign"] |
| 116 | printf "%-16s %17.2f%% %17.2f%%\n", n, sa, sn | "sort" |
| 117 | } |
| 118 | close("sort") |
| 119 | }' "$2" |
| 120 | } |
| 121 | report "wall spread" "$work/runs.ms" |
| 122 | if [ -f "$work/runs.cycles" ]; then report "cycles spread" "$work/runs.cycles"; fi |