#!/usr/bin/env bash # Build-to-build variance of the benchmarks (issue #193): with what # layout does identical C run at the same speed? # # Builds every benchmark from scratch VARIANCE_RUNS times per # configuration — a fresh build directory each, so ThinLTO places the # code anew — once with the fixed layout and once without # (LOUISE_NO_ALIGN=1), then runs all the builds interleaved (each in # turn, VARIANCE_REPS rounds), so a slow drift of the machine touches # every build alike. Reports, per benchmark and configuration, the # spread between the builds' best times as a percentage of the best. # The done-when: the aligned spread stays around 2%. Wall clock is what # layout affects (instruction counts do not see it); pin the runs, or # the scheduler's noise swamps the signal. # # scripts/bench-variance.sh # # Environment: VARIANCE_RUNS (builds per configuration, default 3), # VARIANCE_REPS (interleaved rounds, default 5), BENCH_PIN (a CPU list # for taskset, e.g. "0"), LOUISE_PROFILE (default dist), LOUISE_THREADS # (default 1: layout, not scheduling, is what this measures). set -euo pipefail cd "$(dirname "$0")/.." source scripts/lib.sh builds="${VARIANCE_RUNS:-3}" reps="${VARIANCE_REPS:-5}" export LOUISE_PROFILE="${LOUISE_PROFILE:-dist}" export LOUISE_THREADS="${LOUISE_THREADS:-1}" build_release_louise runner=() if [ -n "${BENCH_PIN:-}" ]; then runner=(taskset -c "$BENCH_PIN"); fi work="$(mktemp -d)" trap 'rm -rf "$work"' EXIT # perf_cycles OUTFILE CMD…: CMD under `perf stat`, its stdout to OUTFILE; # prints the run's user cycles summed over core types (a hybrid CPU # reports one row each; --no-scale, as scripts/bench-track.sh), nothing # where perf cannot count. Wall clock has a 1 ms granularity, which is # percents of the fastest benchmarks; cycles resolve the layout signal. perf_cycles() { local out=$1 shift perf stat --no-scale -x, -e cycles:u -o "$work/perf.txt" "$@" > "$out" 2>/dev/null || return 1 awk -F, '$3 ~ /cycles/ && $1 ~ /^[0-9]+$/ { n += $1; seen = 1 } END { if (seen) print n }' "$work/perf.txt" } have_perf=1 if ! command -v perf >/dev/null || [ -z "$(perf_cycles /dev/null true)" ]; then have_perf= echo "bench-variance: perf cannot count cycles here; wall clock only" >&2 fi programs=(tests/bench/*.lse) for config in align noalign; do for b in $(seq "$builds"); do out="$work/$config.$b" mkdir -p "$out" for program in "${programs[@]}"; do name="$(basename "$program" .lse)" echo "bench-variance: building $config.$b/$name" >&2 if [ "$config" = noalign ]; then LOUISE_NO_ALIGN=1 "$LOUISE_BIN" build --release "$program" -o "$out/$name" >/dev/null else "$LOUISE_BIN" build --release "$program" -o "$out/$name" >/dev/null fi expected_output "$program" > "$out/$name.expected" done done done for round in $(seq "$reps"); do echo "bench-variance: round $round of $reps" >&2 for program in "${programs[@]}"; do name="$(basename "$program" .lse)" for config in align noalign; do for b in $(seq "$builds"); do bin="$work/$config.$b/$name" start=$(date +%s%N) c="" if [ -n "$have_perf" ]; then c=$(perf_cycles "$bin.out" "${runner[@]}" "$bin") else "${runner[@]}" "$bin" > "$bin.out" fi ms=$((($(date +%s%N) - start) / 1000000)) if ! diff -q "$bin.expected" "$bin.out" >/dev/null; then echo "bench-variance: $name ($config.$b): wrong output" >&2 exit 1 fi echo "$name $config $b $ms" >> "$work/runs.ms" if [ -n "$c" ]; then echo "$name $config $b $c" >> "$work/runs.cycles"; fi done done done done report() { printf "\n%-16s %18s %18s\n" benchmark "align $1" "no-align $1" awk ' { key = $1 SUBSEP $2 SUBSEP $3 if (!(key in min) || $4 < min[key]) min[key] = $4 names[$1] = 1 } END { for (key in min) { split(key, a, SUBSEP) nc = a[1] SUBSEP a[2] if (!(nc in best) || min[key] < best[nc]) best[nc] = min[key] if (min[key] > worst[nc]) worst[nc] = min[key] } for (n in names) { sa = (worst[n, "align"] - best[n, "align"]) * 100.0 / best[n, "align"] sn = (worst[n, "noalign"] - best[n, "noalign"]) * 100.0 / best[n, "noalign"] printf "%-16s %17.2f%% %17.2f%%\n", n, sa, sn | "sort" } close("sort") }' "$2" } report "wall spread" "$work/runs.ms" if [ -f "$work/runs.cycles" ]; then report "cycles spread" "$work/runs.cycles"; fi