Skip to content

Align bench delta formatting with ember-eslint-parser #71

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
NullVoxPopuli merged 1 commit into from
Jul 3, 2026
Merged

Align bench delta formatting with ember-eslint-parser #71

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 66 additions & 0 deletions scripts/bench-utils.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
/**
* Shared utilities for benchmark formatting scripts.
*/

import { readFileSync } from "node:fs";

export function formatTime(ns) {
if (ns >= 1e6) return `${(ns / 1e6).toFixed(2)} ms`;
if (ns >= 1e3) return `${(ns / 1e3).toFixed(2)} µs`;
return `${ns.toFixed(2)} ns`;
}

/**
* Classify a delta percentage (negative = experiment faster):
* ⚪ within ±2% — indistinguishable from run-to-run noise
* 🟢 faster by more than 2%
* 🟠 slower by 2–5% — worth a look, not necessarily real
* 🔴 slower by 5% or more
*/
export function deltaEmoji(pct) {
const abs = Math.abs(pct);
if (abs < 2) return "⚪";
if (pct <= -5) return "🟢";
if (pct >= 5) return "🔴";
if (pct < 0) return "🟢";
return "🟠";
}

export const DELTA_LEGEND = "🟢 faster · 🔴 slower · 🟠 slightly slower · ⚪ within 2%";

/**
* Parse benchmark JSON results into control/experiment pairs with deltas.
* Uses p50 (median) which is more robust to outliers than avg.
*/
export function parsePairs(json) {
const pairs = new Map();

for (const trial of json.benchmarks || []) {
for (const r of trial.runs || []) {
if (!r.stats) continue;
const m = r.name.match(/^(.+)\s+\((control|experiment)\)$/);
if (!m) continue;
const [, key, role] = m;
if (!pairs.has(key)) pairs.set(key, {});
pairs.get(key)[role] = r.stats;
}
}

const rows = [];
for (const [name, { control, experiment }] of pairs) {
if (!control || !experiment) continue;
const ctrlVal = control.p50 ?? control.avg;
const expVal = experiment.p50 ?? experiment.avg;
const delta = ((expVal - ctrlVal) / ctrlVal) * 100;
rows.push({ name, control: ctrlVal, experiment: expVal, delta });
}

return rows;
}

/**
* Read and parse the benchmark JSON results file.
*/
export function readBenchJSON(path) {
return JSON.parse(readFileSync(path, "utf8"));
}
61 changes: 4 additions & 57 deletions scripts/format-bench-cli.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
* BENCH_JSON_OUTPUT - Path to the JSON bench results
*/

import { readFileSync } from "node:fs";
import { DELTA_LEGEND, deltaEmoji, formatTime, parsePairs, readBenchJSON } from "./bench-utils.mjs";

const jsonPath = process.env.BENCH_JSON_OUTPUT;

Expand All @@ -18,66 +18,13 @@ if (!jsonPath) {
let json;

try {
json = JSON.parse(readFileSync(jsonPath, "utf8"));
json = readBenchJSON(jsonPath);
} catch (e) {
console.error(`Could not read ${jsonPath}: ${e.message}`);
process.exit(1);
}

function formatTime(ns) {
if (ns >= 1e6) return `${(ns / 1e6).toFixed(2)} ms`;
if (ns >= 1e3) return `${(ns / 1e3).toFixed(2)} µs`;

return `${ns.toFixed(2)} ns`;
}

function deltaEmoji(pct) {
const abs = Math.abs(pct);

if (abs < 1) return "⚪";
if (pct <= -5) return "🟢";
if (pct >= 5) return "🔴";

return "🟡";
}

// Group control/experiment pairs
const pairs = new Map();

for (const trial of json.benchmarks || []) {
for (const r of trial.runs || []) {
if (!r.stats) continue;

const m = r.name.match(/^(.+)\s+\((control|experiment)\)$/);

if (!m) continue;

const [, key, role] = m;

if (!pairs.has(key)) pairs.set(key, {});

pairs.get(key)[role] = r.stats;
}
}

if (pairs.size === 0) {
console.log("No comparison data found.");
process.exit(0);
}

// Build rows — use median (p50) which is far more robust to outliers from
// CPU frequency scaling, GC pauses, and other system noise than the mean.
const rows = [];

for (const [name, { control, experiment }] of pairs) {
if (!control || !experiment) continue;

const ctrlVal = control.p50 ?? control.avg;
const expVal = experiment.p50 ?? experiment.avg;
const delta = ((expVal - ctrlVal) / ctrlVal) * 100;

rows.push({ name, control: ctrlVal, experiment: expVal, delta });
}
const rows = parsePairs(json);

if (rows.length === 0) {
console.log("No comparison data found.");
Expand Down Expand Up @@ -122,5 +69,5 @@ for (const row of rows) {
}

console.log();
console.log("🟢 faster · 🔴 slower · 🟡 within 5% · ⚪ within 1%");
console.log(DELTA_LEGEND);
console.log();
86 changes: 20 additions & 66 deletions scripts/format-bench-comment.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
*/

import { readFileSync } from "node:fs";
import { DELTA_LEGEND, deltaEmoji, formatTime, parsePairs, readBenchJSON } from "./bench-utils.mjs";

const marker = "<!-- bench-compare -->";

Expand Down Expand Up @@ -43,77 +44,30 @@ const jsonPath = process.env.BENCH_JSON_OUTPUT;

if (jsonPath) {
try {
const json = JSON.parse(readFileSync(jsonPath, "utf8"));
summarySection = buildSummary(json);
const rows = parsePairs(readBenchJSON(jsonPath));

if (rows.length > 0) {
const tableRows = rows.map(({ name, control, experiment, delta }) => {
const emoji = deltaEmoji(delta);
const sign = delta > 0 ? "+" : "";
return `| ${emoji} | ${name} | ${formatTime(control)} | ${formatTime(experiment)} | ${sign}${delta.toFixed(1)}% |`;
});

summarySection = [
"",
"| | Benchmark | Control (p50) | Experiment (p50) | Δ |",
"|---|---|---:|---:|---:|",
...tableRows,
"",
`> ${DELTA_LEGEND}`,
"",
].join("\n");
}
} catch {
// JSON not available or malformed — skip summary
}
}

function formatTime(ns) {
if (ns >= 1e6) return `${(ns / 1e6).toFixed(2)} ms`;
if (ns >= 1e3) return `${(ns / 1e3).toFixed(2)} µs`;
return `${ns.toFixed(2)} ns`;
}

function deltaEmoji(pct) {
const abs = Math.abs(pct);
// negative pct means experiment is faster (lower time = better)
if (abs < 1) return "⚪";
if (pct <= -5) return "🟢";
if (pct >= 5) return "🔴";
return "🟡";
}

function buildSummary(json) {
const benchmarks = json.benchmarks || [];

// In comparison mode, benchmarks come in pairs inside summary groups.
// Each benchmark alias is like "gts small (control)" / "gts small (experiment)".
// Group them by stripping the suffix.
const pairs = new Map();

for (const trial of benchmarks) {
for (const r of trial.runs || []) {
if (!r.stats) continue;
const m = r.name.match(/^(.+)\s+\((control|experiment)\)$/);
if (!m) continue;
const [, key, role] = m;
if (!pairs.has(key)) pairs.set(key, {});
pairs.get(key)[role] = r.stats;
}
}

if (pairs.size === 0) return "";

const rows = [];
for (const [name, { control, experiment }] of pairs) {
if (!control || !experiment) continue;
// Use p50 (median) — far more robust to GC pauses and noisy-neighbor
// spikes on shared CI runners than the mean.
const ctrlVal = control.p50 ?? control.avg;
const expVal = experiment.p50 ?? experiment.avg;
const delta = ((expVal - ctrlVal) / ctrlVal) * 100;
const emoji = deltaEmoji(delta);
const sign = delta > 0 ? "+" : "";
rows.push(
`| ${emoji} | ${name} | ${formatTime(ctrlVal)} | ${formatTime(expVal)} | ${sign}${delta.toFixed(1)}% |`,
);
}

if (rows.length === 0) return "";

return [
"",
"| | Benchmark | Control (p50) | Experiment (p50) | Δ |",
"|---|---|---:|---:|---:|",
...rows,
"",
"> 🟢 faster · 🔴 slower · 🟡 within 5% · ⚪ within 1%",
"",
].join("\n");
}

// ---------------------------------------------------------------------------
// Assemble comment
// ---------------------------------------------------------------------------
Expand Down
Loading