tobogganSe connecter

louise

The Louise language: compiler, runtime, std, tooling

Clone avec HTTPS, un jeton personnel en mot de passe, ou avec SSH, une clé ajoutée dans vos paramètres.

122 lignes · 4556 octetsBrut · Historique
1#!/usr/bin/env bash
2# Build-to-build variance of the benchmarks (issue #193): with what
3# layout does identical C run at the same speed?
4#
5# Builds every benchmark from scratch VARIANCE_RUNS times per
6# configuration — a fresh build directory each, so ThinLTO places the
7# code anew — once with the fixed layout and once without
8# (LOUISE_NO_ALIGN=1), then runs all the builds interleaved (each in
9# turn, VARIANCE_REPS rounds), so a slow drift of the machine touches
10# every build alike. Reports, per benchmark and configuration, the
11# spread between the builds' best times as a percentage of the best.
12# The done-when: the aligned spread stays around 2%. Wall clock is what
13# layout affects (instruction counts do not see it); pin the runs, or
14# the scheduler's noise swamps the signal.
15#
16# scripts/bench-variance.sh
17#
18# Environment: VARIANCE_RUNS (builds per configuration, default 3),
19# VARIANCE_REPS (interleaved rounds, default 5), BENCH_PIN (a CPU list
20# for taskset, e.g. "0"), LOUISE_PROFILE (default dist), LOUISE_THREADS
21# (default 1: layout, not scheduling, is what this measures).
22set -euo pipefail
23cd "$(dirname "$0")/.."
24source scripts/lib.sh
25
26builds="${VARIANCE_RUNS:-3}"
27reps="${VARIANCE_REPS:-5}"
28export LOUISE_PROFILE="${LOUISE_PROFILE:-dist}"
29export LOUISE_THREADS="${LOUISE_THREADS:-1}"
30
31build_release_louise
32runner=()
33if [ -n "${BENCH_PIN:-}" ]; then runner=(taskset -c "$BENCH_PIN"); fi
34
35work="$(mktemp -d)"
36trap 'rm -rf "$work"' EXIT
37
38# perf_cycles OUTFILE CMD…: CMD under `perf stat`, its stdout to OUTFILE;
39# prints the run's user cycles summed over core types (a hybrid CPU
40# reports one row each; --no-scale, as scripts/bench-track.sh), nothing
41# where perf cannot count. Wall clock has a 1 ms granularity, which is
42# percents of the fastest benchmarks; cycles resolve the layout signal.
43perf_cycles() {
44 local out=$1
45 shift
46 perf stat --no-scale -x, -e cycles:u -o "$work/perf.txt" "$@" > "$out" 2>/dev/null || return 1
47 awk -F, '$3 ~ /cycles/ && $1 ~ /^[0-9]+$/ { n += $1; seen = 1 }
48 END { if (seen) print n }' "$work/perf.txt"
49}
50have_perf=1
51if ! command -v perf >/dev/null || [ -z "$(perf_cycles /dev/null true)" ]; then
52 have_perf=
53 echo "bench-variance: perf cannot count cycles here; wall clock only" >&2
54fi
55
56programs=(tests/bench/*.lse)
57for config in align noalign; do
58 for b in $(seq "$builds"); do
59 out="$work/$config.$b"
60 mkdir -p "$out"
61 for program in "${programs[@]}"; do
62 name="$(basename "$program" .lse)"
63 echo "bench-variance: building $config.$b/$name" >&2
64 if [ "$config" = noalign ]; then
65 LOUISE_NO_ALIGN=1 "$LOUISE_BIN" build --release "$program" -o "$out/$name" >/dev/null
66 else
67 "$LOUISE_BIN" build --release "$program" -o "$out/$name" >/dev/null
68 fi
69 expected_output "$program" > "$out/$name.expected"
70 done
71 done
72done
73
74for round in $(seq "$reps"); do
75 echo "bench-variance: round $round of $reps" >&2
76 for program in "${programs[@]}"; do
77 name="$(basename "$program" .lse)"
78 for config in align noalign; do
79 for b in $(seq "$builds"); do
80 bin="$work/$config.$b/$name"
81 start=$(date +%s%N)
82 c=""
83 if [ -n "$have_perf" ]; then
84 c=$(perf_cycles "$bin.out" "${runner[@]}" "$bin")
85 else
86 "${runner[@]}" "$bin" > "$bin.out"
87 fi
88 ms=$((($(date +%s%N) - start) / 1000000))
89 if ! diff -q "$bin.expected" "$bin.out" >/dev/null; then
90 echo "bench-variance: $name ($config.$b): wrong output" >&2
91 exit 1
92 fi
93 echo "$name $config $b $ms" >> "$work/runs.ms"
94 if [ -n "$c" ]; then echo "$name $config $b $c" >> "$work/runs.cycles"; fi
95 done
96 done
97 done
98done
99
100report() {
101 printf "\n%-16s %18s %18s\n" benchmark "align $1" "no-align $1"
102 awk '
103 { key = $1 SUBSEP $2 SUBSEP $3
104 if (!(key in min) || $4 < min[key]) min[key] = $4
105 names[$1] = 1 }
106 END {
107 for (key in min) {
108 split(key, a, SUBSEP)
109 nc = a[1] SUBSEP a[2]
110 if (!(nc in best) || min[key] < best[nc]) best[nc] = min[key]
111 if (min[key] > worst[nc]) worst[nc] = min[key]
112 }
113 for (n in names) {
114 sa = (worst[n, "align"] - best[n, "align"]) * 100.0 / best[n, "align"]
115 sn = (worst[n, "noalign"] - best[n, "noalign"]) * 100.0 / best[n, "noalign"]
116 printf "%-16s %17.2f%% %17.2f%%\n", n, sa, sn | "sort"
117 }
118 close("sort")
119 }' "$2"
120}
121report "wall spread" "$work/runs.ms"
122if [ -f "$work/runs.cycles" ]; then report "cycles spread" "$work/runs.cycles"; fi