diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 1fc5249..6c8fa3b 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -44,7 +44,7 @@ src/ # CLI tool (published to npm as prism-triage) reviewer.ts # multi-provider LLM review labels.ts # GitHub label management with rate limiting write-gate.ts # one dry-run-by-default gate every GitHub mutation funnels through (read-only ethos) - benchmark.ts # embedding provider benchmark tool + benchmark.ts # embedding provider benchmark tool (--out per-run results + cluster membership) config.ts # Zod-validated YAML + env config errors.ts # typed error classes types.ts # shared interfaces diff --git a/CHANGELOG.md b/CHANGELOG.md index de47d8e..cb18c81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,9 @@ all notable changes to pr-prism are documented here. - confirmed (identity) duplicate clusters now pick canonical by earliest-created instead of quality score: byte-identical duplicates are a which-was-first question, and a copied PR can outscore the original it was lifted from. fuzzy clusters keep the state/CI/score rule. starmap `canonical`/`contested` for confirmed clusters shift accordingly (value change, schema stays v1) ### added -- starmap items now carry `createdAt` alongside `updatedAt` (additive), so consumers can render and reason about which-was-first without re-fetching from github. NOTE for star-map: its importer rejects unknown fields, so it needs the coordinated importer patch before consuming a dataset with this field +- `prism benchmark --out ` writes a run's results to a chosen file. every run previously wrote `data/benchmark-results.json`, so a second run silently destroyed the first one's numbers - an hour of embedding lost to starting the next comparison. an empty or directory-shaped value is rejected rather than falling back to the default +- benchmark results now record `clusterMembership` per model per threshold. cluster counts cannot tell you whether a model finding more clusters is catching real duplicates or chaining unrelated items, and re-deriving membership means re-embedding the whole corpus +- starmap items now carry `createdAt` alongside `updatedAt` (additive), so consumers can render and reason about which-was-first without re-fetching from github. star-map's importer rejects unknown fields; the coordinated patch is star-map PR #11, which must land before it consumes a dataset carrying this field ## [3.1.0] — 2026-07-20 diff --git a/src/__tests__/benchmark.test.ts b/src/__tests__/benchmark.test.ts index f85edd6..5b0109c 100644 --- a/src/__tests__/benchmark.test.ts +++ b/src/__tests__/benchmark.test.ts @@ -6,6 +6,7 @@ import { benchmarkDatabasePath, computeClusterOverlap, ensureBenchmarkModels, + resolveBenchmarkOutPath, resolveBenchmarkProviderConfig, runBenchmarkForModel, } from "../benchmark.js"; @@ -373,3 +374,27 @@ describe.sequential("benchmark provider selection", () => { }); }); }); + +describe("resolveBenchmarkOutPath", () => { + it("defaults to data/benchmark-results.json under the working directory", () => { + expect(resolveBenchmarkOutPath(undefined, "/work")).toBe("/work/data/benchmark-results.json"); + }); + + it("resolves a relative path against the working directory", () => { + expect(resolveBenchmarkOutPath("out/local.json", "/work")).toBe("/work/out/local.json"); + }); + + it("keeps an absolute path as given", () => { + expect(resolveBenchmarkOutPath("/tmp/run-a.json", "/work")).toBe("/tmp/run-a.json"); + }); + + it("rejects a path that is empty or only whitespace", () => { + // A silent fall back to the default would overwrite the previous run, + // which is the exact data loss this option exists to prevent. + expect(() => resolveBenchmarkOutPath(" ", "/work")).toThrow(/--out/); + }); + + it("rejects a directory-looking path", () => { + expect(() => resolveBenchmarkOutPath("out/", "/work")).toThrow(/file path/); + }); +}); diff --git a/src/benchmark.ts b/src/benchmark.ts index 78950db..e5cef3b 100644 --- a/src/benchmark.ts +++ b/src/benchmark.ts @@ -1,6 +1,6 @@ import { execFileSync } from "node:child_process"; -import { writeFileSync } from "node:fs"; -import { resolve } from "node:path"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; import chalk from "chalk"; import Table from "cli-table3"; import ora from "ora"; @@ -42,6 +42,13 @@ export interface BenchmarkResult { models: ModelResult[]; thresholds: number[]; overlaps: Array<{ model: string; threshold: number; overlap: OverlapResult }>; + /** + * Cluster membership per model per threshold. Counts alone cannot answer + * whether a model that finds more clusters is catching real duplicates or + * chaining unrelated items, and re-deriving membership means re-embedding + * the corpus, so it is recorded here. + */ + clusterMembership: Array<{ model: string; threshold: number; clusters: SimplifiedCluster[] }>; timestamp: string; } @@ -50,6 +57,29 @@ export interface BenchmarkEndpointIdentity { fingerprint?: string; } +/** + * Where a benchmark run writes its results. + * + * Defaults to data/benchmark-results.json, which every run shares. Two runs in + * a row therefore leave only the second one's numbers on disk, so a comparison + * that took an hour to produce can be lost by starting the next one. Passing + * --out gives each run its own file. + * + * An empty or directory-shaped value is rejected rather than quietly falling + * back to the default, since the fallback is the data loss being avoided. + */ +export function resolveBenchmarkOutPath(out: string | undefined, cwd: string = process.cwd()): string { + if (out === undefined) return resolve(cwd, "data", "benchmark-results.json"); + const trimmed = out.trim(); + if (!trimmed) { + throw new Error("--out must be a file path, not an empty value"); + } + if (/[/\\]$/.test(trimmed)) { + throw new Error(`--out must be a file path, not a directory: ${out}`); + } + return resolve(cwd, trimmed); +} + export function benchmarkDatabasePath( repoFull: string, provider: string, @@ -351,6 +381,8 @@ export async function runBenchmarkForModel( // ── main benchmark runner ──────────────────────────────────────── export interface BenchmarkOptions { + /** Results file for this run; defaults to data/benchmark-results.json. */ + out?: string; repo?: string; models: string; thresholds?: string; @@ -469,6 +501,7 @@ export async function runBenchmark(opts: BenchmarkOptions): Promise { // ── overlap: each model vs baseline (first model) ─────────── const baseline = results[0]; const allOverlaps: Array<{ model: string; threshold: number; overlap: OverlapResult }> = []; + const clusterMembership: Array<{ model: string; threshold: number; clusters: SimplifiedCluster[] }> = []; // Load baseline clusters once per threshold for (const t of thresholds) { @@ -483,6 +516,7 @@ export async function runBenchmark(opts: BenchmarkOptions): Promise { items: c.items.map((item) => item.number), })); baseStore.close(); + clusterMembership.push({ model: baseline.model, threshold: t, clusters: baseSimplified }); for (let ri = 1; ri < results.length; ri++) { const r = results[ri]; @@ -497,6 +531,7 @@ export async function runBenchmark(opts: BenchmarkOptions): Promise { items: c.items.map((item) => item.number), })); store.close(); + clusterMembership.push({ model: r.model, threshold: t, clusters: simplified }); const overlap = computeClusterOverlap(baseSimplified, simplified); allOverlaps.push({ model: r.model, threshold: t, overlap }); @@ -530,10 +565,12 @@ export async function runBenchmark(opts: BenchmarkOptions): Promise { models: results, thresholds, overlaps: allOverlaps, + clusterMembership, timestamp: new Date().toISOString(), }; - const outPath = resolve(process.cwd(), "data", "benchmark-results.json"); + const outPath = resolveBenchmarkOutPath(opts.out); + mkdirSync(dirname(outPath), { recursive: true }); writeFileSync(outPath, JSON.stringify(benchmarkResult, null, 2)); console.log(chalk.green(`\nresults saved to ${outPath}`)); } diff --git a/src/cli.ts b/src/cli.ts index fefd80d..5b3564d 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -912,6 +912,7 @@ program .option("--dimensions ", "Output dimensions (falls back to EMBEDDING_DIMENSIONS)", parseBenchmarkDimensions) .option("--models ", "Comma-separated embedding models to compare", "nomic-embed-text,qwen3-embedding:0.6b") .option("--thresholds ", "Comma-separated similarity thresholds", "0.82,0.85") + .option("--out ", "Results file for this run (default: data/benchmark-results.json)") .action(async (opts) => { await runBenchmark({ repo: opts.repo, @@ -920,6 +921,7 @@ program dimensions: opts.dimensions, models: opts.models, thresholds: opts.thresholds, + out: opts.out, }); });