diff --git a/scripts/bench-utils.mjs b/scripts/bench-utils.mjs new file mode 100644 index 0000000..a124f3e --- /dev/null +++ b/scripts/bench-utils.mjs @@ -0,0 +1,66 @@ +/** + * Shared utilities for benchmark formatting scripts. + */ + +import { readFileSync } from "node:fs"; + +export function formatTime(ns) { + if (ns >= 1e6) return `${(ns / 1e6).toFixed(2)} ms`; + if (ns >= 1e3) return `${(ns / 1e3).toFixed(2)} µs`; + return `${ns.toFixed(2)} ns`; +} + +/** + * Classify a delta percentage (negative = experiment faster): + * ⚪ within ±2% — indistinguishable from run-to-run noise + * 🟢 faster by more than 2% + * 🟠 slower by 2–5% — worth a look, not necessarily real + * 🔴 slower by 5% or more + */ +export function deltaEmoji(pct) { + const abs = Math.abs(pct); + if (abs < 2) return "⚪"; + if (pct <= -5) return "🟢"; + if (pct >= 5) return "🔴"; + if (pct < 0) return "🟢"; + return "🟠"; +} + +export const DELTA_LEGEND = "🟢 faster · 🔴 slower · 🟠 slightly slower · ⚪ within 2%"; + +/** + * Parse benchmark JSON results into control/experiment pairs with deltas. + * Uses p50 (median) which is more robust to outliers than avg. + */ +export function parsePairs(json) { + const pairs = new Map(); + + for (const trial of json.benchmarks || []) { + for (const r of trial.runs || []) { + if (!r.stats) continue; + const m = r.name.match(/^(.+)\s+\((control|experiment)\)$/); + if (!m) continue; + const [, key, role] = m; + if (!pairs.has(key)) pairs.set(key, {}); + pairs.get(key)[role] = r.stats; + } + } + + const rows = []; + for (const [name, { control, experiment }] of pairs) { + if (!control || !experiment) continue; + const ctrlVal = control.p50 ?? control.avg; + const expVal = experiment.p50 ?? experiment.avg; + const delta = ((expVal - ctrlVal) / ctrlVal) * 100; + rows.push({ name, control: ctrlVal, experiment: expVal, delta }); + } + + return rows; +} + +/** + * Read and parse the benchmark JSON results file. + */ +export function readBenchJSON(path) { + return JSON.parse(readFileSync(path, "utf8")); +} diff --git a/scripts/format-bench-cli.mjs b/scripts/format-bench-cli.mjs index 5dc50ac..b169605 100644 --- a/scripts/format-bench-cli.mjs +++ b/scripts/format-bench-cli.mjs @@ -6,7 +6,7 @@ * BENCH_JSON_OUTPUT - Path to the JSON bench results */ -import { readFileSync } from "node:fs"; +import { DELTA_LEGEND, deltaEmoji, formatTime, parsePairs, readBenchJSON } from "./bench-utils.mjs"; const jsonPath = process.env.BENCH_JSON_OUTPUT; @@ -18,66 +18,13 @@ if (!jsonPath) { let json; try { - json = JSON.parse(readFileSync(jsonPath, "utf8")); + json = readBenchJSON(jsonPath); } catch (e) { console.error(`Could not read ${jsonPath}: ${e.message}`); process.exit(1); } -function formatTime(ns) { - if (ns >= 1e6) return `${(ns / 1e6).toFixed(2)} ms`; - if (ns >= 1e3) return `${(ns / 1e3).toFixed(2)} µs`; - - return `${ns.toFixed(2)} ns`; -} - -function deltaEmoji(pct) { - const abs = Math.abs(pct); - - if (abs < 1) return "⚪"; - if (pct <= -5) return "🟢"; - if (pct >= 5) return "🔴"; - - return "🟡"; -} - -// Group control/experiment pairs -const pairs = new Map(); - -for (const trial of json.benchmarks || []) { - for (const r of trial.runs || []) { - if (!r.stats) continue; - - const m = r.name.match(/^(.+)\s+\((control|experiment)\)$/); - - if (!m) continue; - - const [, key, role] = m; - - if (!pairs.has(key)) pairs.set(key, {}); - - pairs.get(key)[role] = r.stats; - } -} - -if (pairs.size === 0) { - console.log("No comparison data found."); - process.exit(0); -} - -// Build rows — use median (p50) which is far more robust to outliers from -// CPU frequency scaling, GC pauses, and other system noise than the mean. -const rows = []; - -for (const [name, { control, experiment }] of pairs) { - if (!control || !experiment) continue; - - const ctrlVal = control.p50 ?? control.avg; - const expVal = experiment.p50 ?? experiment.avg; - const delta = ((expVal - ctrlVal) / ctrlVal) * 100; - - rows.push({ name, control: ctrlVal, experiment: expVal, delta }); -} +const rows = parsePairs(json); if (rows.length === 0) { console.log("No comparison data found."); @@ -122,5 +69,5 @@ for (const row of rows) { } console.log(); -console.log("🟢 faster · 🔴 slower · 🟡 within 5% · ⚪ within 1%"); +console.log(DELTA_LEGEND); console.log(); diff --git a/scripts/format-bench-comment.mjs b/scripts/format-bench-comment.mjs index 80b0eae..b5bfec0 100644 --- a/scripts/format-bench-comment.mjs +++ b/scripts/format-bench-comment.mjs @@ -13,6 +13,7 @@ */ import { readFileSync } from "node:fs"; +import { DELTA_LEGEND, deltaEmoji, formatTime, parsePairs, readBenchJSON } from "./bench-utils.mjs"; const marker = ""; @@ -43,77 +44,30 @@ const jsonPath = process.env.BENCH_JSON_OUTPUT; if (jsonPath) { try { - const json = JSON.parse(readFileSync(jsonPath, "utf8")); - summarySection = buildSummary(json); + const rows = parsePairs(readBenchJSON(jsonPath)); + + if (rows.length > 0) { + const tableRows = rows.map(({ name, control, experiment, delta }) => { + const emoji = deltaEmoji(delta); + const sign = delta > 0 ? "+" : ""; + return `| ${emoji} | ${name} | ${formatTime(control)} | ${formatTime(experiment)} | ${sign}${delta.toFixed(1)}% |`; + }); + + summarySection = [ + "", + "| | Benchmark | Control (p50) | Experiment (p50) | Δ |", + "|---|---|---:|---:|---:|", + ...tableRows, + "", + `> ${DELTA_LEGEND}`, + "", + ].join("\n"); + } } catch { // JSON not available or malformed — skip summary } } -function formatTime(ns) { - if (ns >= 1e6) return `${(ns / 1e6).toFixed(2)} ms`; - if (ns >= 1e3) return `${(ns / 1e3).toFixed(2)} µs`; - return `${ns.toFixed(2)} ns`; -} - -function deltaEmoji(pct) { - const abs = Math.abs(pct); - // negative pct means experiment is faster (lower time = better) - if (abs < 1) return "⚪"; - if (pct <= -5) return "🟢"; - if (pct >= 5) return "🔴"; - return "🟡"; -} - -function buildSummary(json) { - const benchmarks = json.benchmarks || []; - - // In comparison mode, benchmarks come in pairs inside summary groups. - // Each benchmark alias is like "gts small (control)" / "gts small (experiment)". - // Group them by stripping the suffix. - const pairs = new Map(); - - for (const trial of benchmarks) { - for (const r of trial.runs || []) { - if (!r.stats) continue; - const m = r.name.match(/^(.+)\s+\((control|experiment)\)$/); - if (!m) continue; - const [, key, role] = m; - if (!pairs.has(key)) pairs.set(key, {}); - pairs.get(key)[role] = r.stats; - } - } - - if (pairs.size === 0) return ""; - - const rows = []; - for (const [name, { control, experiment }] of pairs) { - if (!control || !experiment) continue; - // Use p50 (median) — far more robust to GC pauses and noisy-neighbor - // spikes on shared CI runners than the mean. - const ctrlVal = control.p50 ?? control.avg; - const expVal = experiment.p50 ?? experiment.avg; - const delta = ((expVal - ctrlVal) / ctrlVal) * 100; - const emoji = deltaEmoji(delta); - const sign = delta > 0 ? "+" : ""; - rows.push( - `| ${emoji} | ${name} | ${formatTime(ctrlVal)} | ${formatTime(expVal)} | ${sign}${delta.toFixed(1)}% |`, - ); - } - - if (rows.length === 0) return ""; - - return [ - "", - "| | Benchmark | Control (p50) | Experiment (p50) | Δ |", - "|---|---|---:|---:|---:|", - ...rows, - "", - "> 🟢 faster · 🔴 slower · 🟡 within 5% · ⚪ within 1%", - "", - ].join("\n"); -} - // --------------------------------------------------------------------------- // Assemble comment // ---------------------------------------------------------------------------