feat(benchmark): per-run output paths and cluster membership

ARCHITECTURE.md

@@ -44,7 +44,7 @@ src/                    # CLI tool (published to npm as prism-triage)
   reviewer.ts           # multi-provider LLM review
   labels.ts             # GitHub label management with rate limiting
   write-gate.ts         # one dry-run-by-default gate every GitHub mutation funnels through (read-only ethos)
-  benchmark.ts          # embedding provider benchmark tool
+  benchmark.ts          # embedding provider benchmark tool (--out per-run results + cluster membership)
   config.ts             # Zod-validated YAML + env config
   errors.ts             # typed error classes
   types.ts              # shared interfaces

CHANGELOG.md

@@ -8,7 +8,9 @@ all notable changes to pr-prism are documented here.
 - confirmed (identity) duplicate clusters now pick canonical by earliest-created instead of quality score: byte-identical duplicates are a which-was-first question, and a copied PR can outscore the original it was lifted from. fuzzy clusters keep the state/CI/score rule. starmap `canonical`/`contested` for confirmed clusters shift accordingly (value change, schema stays v1)
 
 ### added
-- starmap items now carry `createdAt` alongside `updatedAt` (additive), so consumers can render and reason about which-was-first without re-fetching from github. NOTE for star-map: its importer rejects unknown fields, so it needs the coordinated importer patch before consuming a dataset with this field
+- `prism benchmark --out <path>` writes a run's results to a chosen file. every run previously wrote `data/benchmark-results.json`, so a second run silently destroyed the first one's numbers - an hour of embedding lost to starting the next comparison. an empty or directory-shaped value is rejected rather than falling back to the default
+- benchmark results now record `clusterMembership` per model per threshold. cluster counts cannot tell you whether a model finding more clusters is catching real duplicates or chaining unrelated items, and re-deriving membership means re-embedding the whole corpus
+- starmap items now carry `createdAt` alongside `updatedAt` (additive), so consumers can render and reason about which-was-first without re-fetching from github. star-map's importer rejects unknown fields; the coordinated patch is star-map PR #11, which must land before it consumes a dataset carrying this field
 
 ## [3.1.0] — 2026-07-20
 

src/__tests__/benchmark.test.ts

@@ -6,6 +6,7 @@ import {
   benchmarkDatabasePath,
   computeClusterOverlap,
   ensureBenchmarkModels,
+  resolveBenchmarkOutPath,
   resolveBenchmarkProviderConfig,
   runBenchmarkForModel,
 } from "../benchmark.js";
@@ -373,3 +374,27 @@ describe.sequential("benchmark provider selection", () => {
     });
   });
 });
+
+describe("resolveBenchmarkOutPath", () => {
+  it("defaults to data/benchmark-results.json under the working directory", () => {
+    expect(resolveBenchmarkOutPath(undefined, "/work")).toBe("/work/data/benchmark-results.json");
+  });
+
+  it("resolves a relative path against the working directory", () => {
+    expect(resolveBenchmarkOutPath("out/local.json", "/work")).toBe("/work/out/local.json");
+  });
+
+  it("keeps an absolute path as given", () => {
+    expect(resolveBenchmarkOutPath("/tmp/run-a.json", "/work")).toBe("/tmp/run-a.json");
+  });
+
+  it("rejects a path that is empty or only whitespace", () => {
+    // A silent fall back to the default would overwrite the previous run,
+    // which is the exact data loss this option exists to prevent.
+    expect(() => resolveBenchmarkOutPath("   ", "/work")).toThrow(/--out/);
+  });
+
+  it("rejects a directory-looking path", () => {
+    expect(() => resolveBenchmarkOutPath("out/", "/work")).toThrow(/file path/);
+  });
+});

src/benchmark.ts

@@ -1,6 +1,6 @@
 import { execFileSync } from "node:child_process";
-import { writeFileSync } from "node:fs";
-import { resolve } from "node:path";
+import { mkdirSync, writeFileSync } from "node:fs";
+import { dirname, resolve } from "node:path";
 import chalk from "chalk";
 import Table from "cli-table3";
 import ora from "ora";
@@ -42,6 +42,13 @@ export interface BenchmarkResult {
   models: ModelResult[];
   thresholds: number[];
   overlaps: Array<{ model: string; threshold: number; overlap: OverlapResult }>;
+  /**
+   * Cluster membership per model per threshold. Counts alone cannot answer
+   * whether a model that finds more clusters is catching real duplicates or
+   * chaining unrelated items, and re-deriving membership means re-embedding
+   * the corpus, so it is recorded here.
+   */
+  clusterMembership: Array<{ model: string; threshold: number; clusters: SimplifiedCluster[] }>;
   timestamp: string;
 }
 
@@ -50,6 +57,29 @@ export interface BenchmarkEndpointIdentity {
   fingerprint?: string;
 }
 
+/**
+ * Where a benchmark run writes its results.
+ *
+ * Defaults to data/benchmark-results.json, which every run shares. Two runs in
+ * a row therefore leave only the second one's numbers on disk, so a comparison
+ * that took an hour to produce can be lost by starting the next one. Passing
+ * --out gives each run its own file.
+ *
+ * An empty or directory-shaped value is rejected rather than quietly falling
+ * back to the default, since the fallback is the data loss being avoided.
+ */
+export function resolveBenchmarkOutPath(out: string | undefined, cwd: string = process.cwd()): string {
+  if (out === undefined) return resolve(cwd, "data", "benchmark-results.json");
+  const trimmed = out.trim();
+  if (!trimmed) {
+    throw new Error("--out must be a file path, not an empty value");
+  }
+  if (/[/\\]$/.test(trimmed)) {
+    throw new Error(`--out must be a file path, not a directory: ${out}`);
+  }
+  return resolve(cwd, trimmed);
+}
+
 export function benchmarkDatabasePath(
   repoFull: string,
   provider: string,
@@ -351,6 +381,8 @@ export async function runBenchmarkForModel(
 // ── main benchmark runner ────────────────────────────────────────
 
 export interface BenchmarkOptions {
+  /** Results file for this run; defaults to data/benchmark-results.json. */
+  out?: string;
   repo?: string;
   models: string;
   thresholds?: string;
@@ -469,6 +501,7 @@ export async function runBenchmark(opts: BenchmarkOptions): Promise<void> {
   // ── overlap: each model vs baseline (first model) ───────────
   const baseline = results[0];
   const allOverlaps: Array<{ model: string; threshold: number; overlap: OverlapResult }> = [];
+  const clusterMembership: Array<{ model: string; threshold: number; clusters: SimplifiedCluster[] }> = [];
 
   // Load baseline clusters once per threshold
   for (const t of thresholds) {
@@ -483,6 +516,7 @@ export async function runBenchmark(opts: BenchmarkOptions): Promise<void> {
       items: c.items.map((item) => item.number),
     }));
     baseStore.close();
+    clusterMembership.push({ model: baseline.model, threshold: t, clusters: baseSimplified });
 
     for (let ri = 1; ri < results.length; ri++) {
       const r = results[ri];
@@ -497,6 +531,7 @@ export async function runBenchmark(opts: BenchmarkOptions): Promise<void> {
         items: c.items.map((item) => item.number),
       }));
       store.close();
+      clusterMembership.push({ model: r.model, threshold: t, clusters: simplified });
 
       const overlap = computeClusterOverlap(baseSimplified, simplified);
       allOverlaps.push({ model: r.model, threshold: t, overlap });
@@ -530,10 +565,12 @@ export async function runBenchmark(opts: BenchmarkOptions): Promise<void> {
     models: results,
     thresholds,
     overlaps: allOverlaps,
+    clusterMembership,
     timestamp: new Date().toISOString(),
   };
 
-  const outPath = resolve(process.cwd(), "data", "benchmark-results.json");
+  const outPath = resolveBenchmarkOutPath(opts.out);
+  mkdirSync(dirname(outPath), { recursive: true });
   writeFileSync(outPath, JSON.stringify(benchmarkResult, null, 2));
   console.log(chalk.green(`\nresults saved to ${outPath}`));
 }

src/cli.ts

@@ -912,6 +912,7 @@ program
   .option("--dimensions <integer>", "Output dimensions (falls back to EMBEDDING_DIMENSIONS)", parseBenchmarkDimensions)
   .option("--models <models>", "Comma-separated embedding models to compare", "nomic-embed-text,qwen3-embedding:0.6b")
   .option("--thresholds <thresholds>", "Comma-separated similarity thresholds", "0.82,0.85")
+  .option("--out <path>", "Results file for this run (default: data/benchmark-results.json)")
   .action(async (opts) => {
     await runBenchmark({
       repo: opts.repo,
@@ -920,6 +921,7 @@ program
       dimensions: opts.dimensions,
       models: opts.models,
       thresholds: opts.thresholds,
+      out: opts.out,
     });
   });