diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d2f24b1e..52800260 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,33 @@ jobs: run: bun run lint - name: Test - run: bun test --coverage + run: bun test --coverage ./tests/ + + playground-e2e: + name: Playground E2E (Playwright) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - name: Install dependencies + run: bun install + + - name: Cache Playwright browsers + uses: actions/cache@v4 + with: + path: ~/.cache/ms-playwright + key: playwright-${{ runner.os }}-${{ hashFiles('bun.lock') }} + + - name: Install Playwright browsers + run: bunx playwright install --with-deps chromium + + - name: Run Playwright playground tests + run: bun run test:e2e build: name: Build diff --git a/bun.lock b/bun.lock index 163b75ec..982a9f46 100644 --- a/bun.lock +++ b/bun.lock @@ -8,6 +8,7 @@ "@biomejs/biome": "^1.9.4", "@types/bun": "^1.1.14", "fast-check": "^3.22.0", + "playwright": "1.59.1", }, "peerDependencies": { "typescript": "^5.7.0", @@ -41,6 +42,12 @@ "fast-check": ["fast-check@3.23.2", "", { "dependencies": { "pure-rand": "^6.1.0" } }, "sha512-h5+1OzzfCC3Ef7VbtKdcv7zsstUQwUDlYpUTvjeUsJAssPgLn7QzbboPtL5ro04Mq0rPOsMzl7q5hIbRs2wD1A=="], + "fsevents": ["fsevents@2.3.2", "", { "os": "darwin" }, "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA=="], + + "playwright": ["playwright@1.59.1", "", { "dependencies": { "playwright-core": "1.59.1" }, "optionalDependencies": { "fsevents": "2.3.2" }, "bin": { "playwright": "cli.js" } }, "sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw=="], + + "playwright-core": ["playwright-core@1.59.1", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg=="], + "pure-rand": ["pure-rand@6.1.0", "", {}, "sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA=="], "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], diff --git a/docs/playground.md b/docs/playground.md index 7f08e62b..409c5c04 100644 --- a/docs/playground.md +++ b/docs/playground.md @@ -118,6 +118,54 @@ cp node_modules/typescript/lib/typescript.js ./playground/dist/typescript.js The CI pipeline (`pages.yml`) runs this automatically during deployment. +## End-to-End Cell Execution Tests + +To make sure every code cell on every playground page actually works (no +TypeScript errors, no runtime errors, real output), the project ships a +Playwright-based test suite under `tests-e2e/playground-cells.test.ts`. + +It launches headless Chromium, navigates to every `playground/*.html` page, +clicks **▶ Run** on every `.playground-block`, and asserts that the cell +output is not an error and is not the "(no output …)" sentinel. + +```bash +bun install +bunx playwright install --with-deps chromium +bun run test:e2e +``` + +CI runs this in the dedicated `playground-e2e` job (see `.github/workflows/ci.yml`). + +### Known-failures allowlist + +A large number of pages currently have at least one broken cell — most often +because: + +1. The "TypeScript" cell actually contains Python source (so TS lexing fails + on the `import pandas as pd` line). +2. A cell references a variable defined in a previous cell. **Each cell runs + in its own `new Function()` scope, so nothing persists between cells** — + every cell needs its own `import { … } from "tsb"` and its own data setup. +3. A cell never calls `console.log()` — the playground only shows what the + user explicitly logs. + +The file `tests-e2e/known-failures.json` enumerates the (file → cell numbers) +that are currently broken so CI can pass while progress is made. Each entry +should be **removed from the allowlist as the corresponding cell is fixed** — +the test framework also fails if a cell now passes but is still listed +(forward-progress check). + +### Authoring rule + +Every cell on every playground page **must** be self-contained: + +- Import everything it uses from `"tsb"` directly inside the cell. +- Re-declare any helper data it depends on inside the cell. +- Call `console.log(…)` (or `console.warn` / `console.error`) so output is + visible. + +See `playground/merge_ordered.html` for the canonical pattern. + ## Non-Goals (Current Scope) - **Infinite loop protection**: long-running or infinite loops will hang the diff --git a/package.json b/package.json index 848ae544..076a8a67 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,8 @@ } }, "scripts": { - "test": "bun test", + "test": "bun test ./tests/", + "test:e2e": "bun test --timeout 600000 tests-e2e", "lint": "biome check .", "lint:fix": "biome check --write .", "typecheck": "tsc --noEmit", @@ -23,8 +24,9 @@ }, "devDependencies": { "@biomejs/biome": "^1.9.4", + "@types/bun": "^1.1.14", "fast-check": "^3.22.0", - "@types/bun": "^1.1.14" + "playwright": "1.59.1" }, "peerDependencies": { "typescript": "^5.7.0" diff --git a/playground/hash_pandas_object.html b/playground/hash_pandas_object.html index 134212d7..4f8a20d7 100644 --- a/playground/hash_pandas_object.html +++ b/playground/hash_pandas_object.html @@ -244,7 +244,7 @@
import { DataFrame, hashPandasObject } from "tsb";
-const df = new DataFrame({
+const df = DataFrame.fromColumns({
id: [1, 2, 3],
name: ["Alice", "Bob", "Alice"],
age: [30, 25, 30],
@@ -276,7 +276,7 @@ 3 · Deduplication with hashes
import { DataFrame, hashPandasObject } from "tsb";
-const df = new DataFrame({
+const df = DataFrame.fromColumns({
a: [1, 2, 1, 3],
b: ["x", "y", "x", "z"],
});
diff --git a/playground/merge_ordered.html b/playground/merge_ordered.html
index 8f1a2c2f..423261b7 100644
--- a/playground/merge_ordered.html
+++ b/playground/merge_ordered.html
@@ -176,13 +176,13 @@ Basic outer ordered merge
});
const result = mergeOrdered(left, right, { on: "date" });
+console.log(result.toString());
// date | price | volume
// 1 | 10 | null
// 2 | null | 200
// 3 | 30 | 300
// 5 | 50 | null
-// 6 | null | 600
-console.log(result);
+// 6 | null | 600
Click ▶ Run to execute
Ctrl+Enter to run · Tab to indent
@@ -198,17 +198,28 @@ Forward-fill after merge
-
Click ▶ Run to execute
Ctrl+Enter to run · Tab to indent
@@ -224,8 +235,14 @@ Inner join variant
- // Only rows where both DataFrames have a key
-console.log(mergeOrdered(left, right, { on: "date", how: "inner" }));
+ import { DataFrame, mergeOrdered } from "tsb";
+
+const left = DataFrame.fromColumns({ date: [1, 3, 5], price: [10, 30, 50] });
+const right = DataFrame.fromColumns({ date: [2, 3, 6], volume: [200, 300, 600] });
+
+// Only rows where both DataFrames have a key
+const result = mergeOrdered(left, right, { on: "date", how: "inner" });
+console.log(result.toString());
// date | price | volume
// 3 | 30 | 300
Click ▶ Run to execute
@@ -243,10 +260,13 @@ Different key column names per side
- const left2 = DataFrame.fromColumns({ t_left: [1, 3, 5], a: [10, 30, 50] });
+ import { DataFrame, mergeOrdered } from "tsb";
+
+const left2 = DataFrame.fromColumns({ t_left: [1, 3, 5], a: [10, 30, 50] });
const right2 = DataFrame.fromColumns({ t_right: [2, 3, 6], b: [200, 300, 600] });
-console.log(mergeOrdered(left2, right2, { left_on: "t_left", right_on: "t_right" }));
+const result = mergeOrdered(left2, right2, { left_on: "t_left", right_on: "t_right" });
+console.log(result.toString());
// t_left | a | b
// 1 | 10 | null
// 2 | null | 200
@@ -268,7 +288,9 @@ Group-wise ordered merge (left_by / right_by)
- // Perform the ordered merge independently for each group
+ import { DataFrame, mergeOrdered } from "tsb";
+
+// Perform the ordered merge independently for each group
const left3 = DataFrame.fromColumns({
grp: ["A", "A", "B", "B"],
k: [1, 3, 1, 3],
@@ -280,19 +302,19 @@ Group-wise ordered merge (left_by / right_by)
b: [20, 30, 200, 300],
});
-mergeOrdered(left3, right3, {
+const result = mergeOrdered(left3, right3, {
on: "k",
left_by: "grp",
right_by: "grp",
});
+console.log(result.toString());
// grp | k | a | b
// A | 1 | 10 | null
// A | 2 | null | 20
// A | 3 | 30 | 30
// B | 1 | 100 | null
// B | 2 | null | 200
-// B | 3 | 300 | 300
-console.log(right3);
+// B | 3 | 300 | 300
Click ▶ Run to execute
Ctrl+Enter to run · Tab to indent
@@ -308,10 +330,13 @@ Overlapping non-key columns — suffixes
- const left4 = DataFrame.fromColumns({ k: [1, 2, 3], val: [10, 20, 30] });
+ import { DataFrame, mergeOrdered } from "tsb";
+
+const left4 = DataFrame.fromColumns({ k: [1, 2, 3], val: [10, 20, 30] });
const right4 = DataFrame.fromColumns({ k: [2, 3, 4], val: [200, 300, 400] });
-console.log(mergeOrdered(left4, right4, { on: "k", suffixes: ["_L", "_R"] }));
+const result = mergeOrdered(left4, right4, { on: "k", suffixes: ["_L", "_R"] });
+console.log(result.toString());
// k | val_L | val_R
// 1 | 10 | null
// 2 | 20 | 200
diff --git a/tests-e2e/known-failures.json b/tests-e2e/known-failures.json
new file mode 100644
index 00000000..09251f9e
--- /dev/null
+++ b/tests-e2e/known-failures.json
@@ -0,0 +1,63 @@
+{
+ "align.html": [2, 3, 5, 6, 7],
+ "api_types.html": [2, 3, 4, 5, 6, 7],
+ "assign.html": [2, 3, 4, 5],
+ "at_iat.html": [1, 3, 4, 5, 7, 8, 9],
+ "attrs.html": [2, 3, 4, 5, 6],
+ "between.html": [1, 3, 5, 7],
+ "clip_advanced.html": [1, 2, 3, 4, 5, 6],
+ "corrwith.html": [1, 3, 5, 7],
+ "crosstab.html": [2, 4, 6, 8, 10, 12],
+ "cut.html": [2, 3, 4, 5, 6, 8],
+ "cut_bins_to_frame.html": [1],
+ "cut_qcut.html": [1, 2, 4, 5, 6, 7],
+ "datetime_tz.html": [2, 4, 6, 8, 10],
+ "dot_matmul.html": [1, 2],
+ "eval_query.html": [1, 2, 3],
+ "excel.html": [1, 2, 3, 4],
+ "factorize.html": [2, 4, 6, 8],
+ "filter.html": [1, 3, 5, 7, 9],
+ "get_dummies.html": [2, 4, 6, 8],
+ "infer_dtype.html": [5],
+ "insert_pop.html": [6],
+ "interpolate.html": [2, 6],
+ "join.html": [2, 3, 4, 5, 6],
+ "json_normalize.html": [2, 3],
+ "memory_usage.html": [1, 2, 3, 4, 5],
+ "merge_asof.html": [2, 3, 4, 5, 6, 7],
+ "mode.html": [1, 2, 3, 4, 5, 6],
+ "named_agg.html": [2, 3, 5, 6, 7],
+ "natsort.html": [2, 6],
+ "nunique.html": [2, 3, 5],
+ "pipe_apply.html": [1, 3],
+ "pivot_table.html": [2, 3, 4],
+ "pow_mod.html": [5],
+ "quantile.html": [1, 2, 3, 4, 5, 6, 7],
+ "reduce_ops.html": [1, 3, 4, 6],
+ "reindex.html": [6],
+ "rolling_apply.html": [1, 2, 3, 4, 5, 6],
+ "scalar_extract.html": [1, 3, 5, 7, 9, 11],
+ "searchsorted.html": [6],
+ "select_dtypes.html": [2, 3, 4, 5],
+ "sem_var.html": [2, 3, 5],
+ "skew_kurt.html": [1, 2, 3, 4, 5, 6],
+ "sort_ops.html": [1, 3, 4, 5, 6, 7, 9, 10, 11],
+ "str_findall_and_json_denormalize.html": [2, 3, 4, 5, 6, 8, 9, 10, 11, 12],
+ "str_get_dummies.html": [2, 4, 6, 8],
+ "style.html": [
+ 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25
+ ],
+ "swaplevel.html": [1, 2, 3, 4, 5, 6, 7, 8],
+ "testing.html": [1, 3, 4, 6, 8],
+ "to_datetime.html": [2],
+ "to_from_dict.html": [1, 2],
+ "to_numeric.html": [1, 2, 3, 4, 5],
+ "to_timedelta.html": [2],
+ "transform_agg.html": [1, 2],
+ "truncate.html": [1, 3, 5, 7],
+ "update.html": [1, 3, 5],
+ "value_counts_full.html": [1, 3, 4, 5, 6],
+ "wide_to_long.html": [1, 2, 3, 4],
+ "window_extended.html": [1, 3, 5],
+ "xs.html": [1]
+}
diff --git a/tests-e2e/playground-cells.test.ts b/tests-e2e/playground-cells.test.ts
new file mode 100644
index 00000000..9d8df8a5
--- /dev/null
+++ b/tests-e2e/playground-cells.test.ts
@@ -0,0 +1,276 @@
+/**
+ * Playground page execution tests (Playwright + Chromium).
+ *
+ * For every interactive playground page under `playground/`, this test
+ * launches a real headless browser, clicks ▶ Run on every code cell, and
+ * asserts that:
+ *
+ * 1. The cell does not produce a TypeScript or runtime error
+ * (the playground runtime renders these with a leading "❌" and
+ * sets the .error class on `.playground-output`).
+ * 2. The cell produces non-empty output (i.e. it actually called
+ * console.log / console.error / console.warn — no
+ * "(no output — add console.log() …)" sentinel).
+ *
+ * This catches the class of bugs called out in the playground-test issue —
+ * for example, pages where cells reference variables/imports from a previous
+ * cell (cells run in isolated `new Function` scopes so nothing persists), or
+ * pages where the "TypeScript" cell actually contains Python source.
+ *
+ * ── Known failures allowlist ─────────────────────────────────────────
+ *
+ * Many existing pages currently fail. To make the test useful immediately
+ * (catch *new* regressions, surface the existing breakage) without blocking
+ * CI on a 78-page rewrite, the file `tests-e2e/known-failures.json`
+ * enumerates the (file → cell numbers) that are currently broken.
+ *
+ * - A failing cell that IS in the allowlist is reported but does not
+ * fail the test (encoded as a passing assertion with `console.warn`).
+ * - A failing cell that is NOT in the allowlist fails the test
+ * (regression).
+ * - A passing cell that IS in the allowlist also fails the test —
+ * please remove it from the allowlist (forward progress).
+ *
+ * As pages get fixed, their entries should be removed from
+ * `known-failures.json`.
+ *
+ * ── Running locally ──────────────────────────────────────────────────
+ *
+ * bun install
+ * bunx playwright install chromium
+ * bun run test:e2e
+ */
+
+import { afterAll, beforeAll, describe, expect, it } from "bun:test";
+import { readFileSync, readdirSync } from "node:fs";
+import { join } from "node:path";
+import { type Browser, type BrowserContext, chromium } from "playwright";
+
+const PROJECT_ROOT = join(import.meta.dir, "..");
+const PLAYGROUND_DIR = join(PROJECT_ROOT, "playground");
+const KNOWN_FAILURES_PATH = join(import.meta.dir, "known-failures.json");
+
+// Pages that are intentionally not interactive playgrounds.
+const NON_PLAYGROUND_PAGES = new Set(["index.html", "benchmarks.html"]);
+
+const PORT = 3399;
+const BASE_URL = `http://localhost:${PORT}`;
+
+// Number of pages tested in parallel. Chosen empirically: low enough that a
+// shared GitHub Actions runner (2 vCPU) doesn't thrash on V8 isolates and
+// network I/O, high enough that the full suite (~130 pages) finishes well
+// inside a minute. Increasing this past ~8 yields diminishing returns and
+// risks flaky timeouts on constrained runners.
+const PAGE_CONCURRENCY = 6;
+
+type KnownFailures = Record;
+
+function loadKnownFailures(): KnownFailures {
+ const raw = readFileSync(KNOWN_FAILURES_PATH, "utf8");
+ return JSON.parse(raw) as KnownFailures;
+}
+
+function listPlaygroundHtmlFiles(): string[] {
+ return readdirSync(PLAYGROUND_DIR)
+ .filter((f) => f.endsWith(".html"))
+ .filter((f) => !NON_PLAYGROUND_PAGES.has(f))
+ .sort();
+}
+
+type CellOutcome = {
+ cellIndex: number; // 1-based to match human-readable allowlist entries
+ ok: boolean;
+ reason: string; // empty when ok
+};
+
+type ServerHandle = { kill: () => void };
+
+async function startPlaygroundServer(): Promise {
+ // Re-use the project's serve.ts but on a non-default port via env override.
+ // serve.ts hard-codes the port, so we instead build the bundle once and
+ // start a minimal Bun.serve on PORT here.
+ await Bun.$`mkdir -p ${join(PLAYGROUND_DIR, "dist")}`.quiet();
+ const buildResult = await Bun.build({
+ entrypoints: [join(PROJECT_ROOT, "src", "index.ts")],
+ outdir: join(PLAYGROUND_DIR, "dist"),
+ target: "browser",
+ minify: true,
+ });
+ if (!buildResult.success) {
+ for (const log of buildResult.logs) console.error(log);
+ throw new Error("Failed to build tsb browser bundle");
+ }
+ const tsSrc = join(PROJECT_ROOT, "node_modules", "typescript", "lib", "typescript.js");
+ await Bun.write(join(PLAYGROUND_DIR, "dist", "typescript.js"), Bun.file(tsSrc));
+
+ const MIME: Record = {
+ ".html": "text/html",
+ ".js": "text/javascript",
+ ".css": "text/css",
+ ".json": "application/json",
+ ".svg": "image/svg+xml",
+ };
+ const server = Bun.serve({
+ port: PORT,
+ fetch(req: Request): Response {
+ const url = new URL(req.url);
+ const pathname = url.pathname === "/" ? "/index.html" : url.pathname;
+ const filePath = join(PLAYGROUND_DIR, pathname);
+ const dotIdx = filePath.lastIndexOf(".");
+ const ext = dotIdx >= 0 ? filePath.slice(dotIdx) : "";
+ return new Response(Bun.file(filePath), {
+ headers: { "Content-Type": MIME[ext] ?? "application/octet-stream" },
+ });
+ },
+ error(): Response {
+ return new Response("Not found", { status: 404 });
+ },
+ });
+ return { kill: () => server.stop(true) };
+}
+
+async function executePageCells(ctx: BrowserContext, file: string): Promise {
+ const page = await ctx.newPage();
+ const pageErrors: string[] = [];
+ page.on("pageerror", (err) => pageErrors.push(err.message));
+ const outcomes: CellOutcome[] = [];
+ try {
+ await page.goto(`${BASE_URL}/${file}`, {
+ waitUntil: "networkidle",
+ timeout: 30000,
+ });
+ // Wait until all Run buttons are enabled (signals runtime is initialized).
+ await page.waitForFunction(
+ () => {
+ const btns = document.querySelectorAll(".playground-run");
+ if (btns.length === 0) return false;
+ return Array.from(btns).every((b) => !(b as HTMLButtonElement).disabled);
+ },
+ { timeout: 25000 },
+ );
+ const blocks = await page.locator(".playground-block").all();
+ for (let i = 0; i < blocks.length; i++) {
+ const block = blocks[i];
+ if (!block) continue;
+ const cellIndex = i + 1;
+ try {
+ await block.locator(".playground-run").click();
+ // Output is updated synchronously inside the click handler, but give
+ // the event loop a tick for the DOM update to flush.
+ await page.waitForTimeout(50);
+ const outputLoc = block.locator(".playground-output").first();
+ const text = (await outputLoc.textContent()) ?? "";
+ const cls = (await outputLoc.getAttribute("class")) ?? "";
+ const trimmed = text.trim();
+ if (cls.includes("error") || trimmed.startsWith("❌")) {
+ outcomes.push({
+ cellIndex,
+ ok: false,
+ reason: `error: ${trimmed.slice(0, 200)}`,
+ });
+ } else if (
+ trimmed === "" ||
+ trimmed.startsWith("(no output") ||
+ trimmed.startsWith("Click ▶")
+ ) {
+ outcomes.push({ cellIndex, ok: false, reason: "no output produced" });
+ } else {
+ outcomes.push({ cellIndex, ok: true, reason: "" });
+ }
+ } catch (e) {
+ const msg = e instanceof Error ? e.message : String(e);
+ outcomes.push({ cellIndex, ok: false, reason: `playwright error: ${msg}` });
+ }
+ }
+ } finally {
+ if (pageErrors.length > 0) {
+ // Surface uncaught page errors as a synthetic cell-0 failure.
+ outcomes.unshift({
+ cellIndex: 0,
+ ok: false,
+ reason: `page error(s): ${pageErrors.join(" || ").slice(0, 300)}`,
+ });
+ }
+ await page.close();
+ }
+ return outcomes;
+}
+
+let server: ServerHandle | null = null;
+let browser: Browser | null = null;
+let context: BrowserContext | null = null;
+const allOutcomes: Map = new Map();
+const knownFailures = loadKnownFailures();
+const files = listPlaygroundHtmlFiles();
+
+beforeAll(async () => {
+ server = await startPlaygroundServer();
+ browser = await chromium.launch();
+ context = await browser.newContext();
+
+ // Pre-execute every page in parallel (bounded concurrency) to keep total
+ // wall-clock under a few minutes. We still create a per-page `it()` below
+ // so failures are reported with file granularity.
+ let nextIdx = 0;
+ async function worker(): Promise {
+ while (nextIdx < files.length) {
+ const i = nextIdx++;
+ const file = files[i];
+ if (!file || !context) continue;
+ const outcomes = await executePageCells(context, file);
+ allOutcomes.set(file, outcomes);
+ }
+ }
+ await Promise.all(Array.from({ length: PAGE_CONCURRENCY }, worker));
+}, 600_000);
+
+afterAll(async () => {
+ if (context) await context.close();
+ if (browser) await browser.close();
+ if (server) server.kill();
+});
+
+describe("playground page execution", () => {
+ it("discovered playground pages", () => {
+ expect(files.length).toBeGreaterThan(0);
+ });
+
+ for (const file of files) {
+ it(`every cell on ${file} produces non-error, non-empty output`, () => {
+ const outcomes = allOutcomes.get(file);
+ expect(outcomes, `no outcomes recorded for ${file}`).toBeDefined();
+ const allowedCells = new Set(knownFailures[file] ?? []);
+ const unexpectedFailures: string[] = [];
+ const unexpectedPasses: number[] = [];
+
+ for (const o of outcomes ?? []) {
+ if (o.cellIndex === 0) {
+ // Page-level failure (e.g., uncaught page error) — never allowlist.
+ unexpectedFailures.push(`page-level: ${o.reason}`);
+ continue;
+ }
+ const inAllowlist = allowedCells.has(o.cellIndex);
+ if (!o.ok && !inAllowlist) {
+ unexpectedFailures.push(`cell ${o.cellIndex}: ${o.reason}`);
+ } else if (o.ok && inAllowlist) {
+ unexpectedPasses.push(o.cellIndex);
+ }
+ }
+
+ const messages: string[] = [];
+ if (unexpectedFailures.length > 0) {
+ messages.push(
+ `${unexpectedFailures.length} regression(s) in ${file}:\n - ${unexpectedFailures.join("\n - ")}`,
+ );
+ }
+ if (unexpectedPasses.length > 0) {
+ messages.push(
+ `${unexpectedPasses.length} cell(s) in ${file} now PASS but are still listed in tests-e2e/known-failures.json — please remove them: [${unexpectedPasses.join(", ")}]`,
+ );
+ }
+ if (messages.length > 0) {
+ throw new Error(messages.join("\n\n"));
+ }
+ });
+ }
+});