Merge pull request #259 from githubnext/copilot/add-playwright-tests-for-playground

Add Playwright tests that execute every playground cell, fix merge_ordered

Author: mrjf

.github/workflows/ci.yml

@@ -36,7 +36,33 @@ jobs:
         run: bun run lint
 
       - name: Test
-        run: bun test --coverage
+        run: bun test --coverage ./tests/
+
+  playground-e2e:
+    name: Playground E2E (Playwright)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: latest
+
+      - name: Install dependencies
+        run: bun install
+
+      - name: Cache Playwright browsers
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-${{ runner.os }}-${{ hashFiles('bun.lock') }}
+
+      - name: Install Playwright browsers
+        run: bunx playwright install --with-deps chromium
+
+      - name: Run Playwright playground tests
+        run: bun run test:e2e
 
   build:
     name: Build

bun.lock

@@ -118,6 +118,54 @@ cp node_modules/typescript/lib/typescript.js ./playground/dist/typescript.js
 
 The CI pipeline (`pages.yml`) runs this automatically during deployment.
 
+## End-to-End Cell Execution Tests
+
+To make sure every code cell on every playground page actually works (no
+TypeScript errors, no runtime errors, real output), the project ships a
+Playwright-based test suite under `tests-e2e/playground-cells.test.ts`.
+
+It launches headless Chromium, navigates to every `playground/*.html` page,
+clicks **▶ Run** on every `.playground-block`, and asserts that the cell
+output is not an error and is not the "(no output …)" sentinel.
+
+```bash
+bun install
+bunx playwright install --with-deps chromium
+bun run test:e2e
+```
+
+CI runs this in the dedicated `playground-e2e` job (see `.github/workflows/ci.yml`).
+
+### Known-failures allowlist
+
+A large number of pages currently have at least one broken cell — most often
+because:
+
+1. The "TypeScript" cell actually contains Python source (so TS lexing fails
+   on the `import pandas as pd` line).
+2. A cell references a variable defined in a previous cell. **Each cell runs
+   in its own `new Function()` scope, so nothing persists between cells** —
+   every cell needs its own `import { … } from "tsb"` and its own data setup.
+3. A cell never calls `console.log()` — the playground only shows what the
+   user explicitly logs.
+
+The file `tests-e2e/known-failures.json` enumerates the (file → cell numbers)
+that are currently broken so CI can pass while progress is made. Each entry
+should be **removed from the allowlist as the corresponding cell is fixed** —
+the test framework also fails if a cell now passes but is still listed
+(forward-progress check).
+
+### Authoring rule
+
+Every cell on every playground page **must** be self-contained:
+
+- Import everything it uses from `"tsb"` directly inside the cell.
+- Re-declare any helper data it depends on inside the cell.
+- Call `console.log(…)` (or `console.warn` / `console.error`) so output is
+  visible.
+
+See `playground/merge_ordered.html` for the canonical pattern.
+
 ## Non-Goals (Current Scope)
 
 - **Infinite loop protection**: long-running or infinite loops will hang the

docs/playground.md

@@ -14,7 +14,8 @@
     }
   },
   "scripts": {
-    "test": "bun test",
+    "test": "bun test ./tests/",
+    "test:e2e": "bun test --timeout 600000 tests-e2e",
     "lint": "biome check .",
     "lint:fix": "biome check --write .",
     "typecheck": "tsc --noEmit",
@@ -23,8 +24,9 @@
   },
   "devDependencies": {
     "@biomejs/biome": "^1.9.4",
+    "@types/bun": "^1.1.14",
     "fast-check": "^3.22.0",
-    "@types/bun": "^1.1.14"
+    "playwright": "1.59.1"
   },
   "peerDependencies": {
     "typescript": "^5.7.0"

package.json

@@ -244,7 +244,7 @@ <h2>2 · DataFrame row hashing</h2>
       </div>
       <pre class="playground-editor" contenteditable="true" spellcheck="false">import { DataFrame, hashPandasObject } from "tsb";
 
-const df = new DataFrame({
+const df = DataFrame.fromColumns({
   id:   [1, 2, 3],
   name: ["Alice", "Bob", "Alice"],
   age:  [30, 25, 30],
@@ -276,7 +276,7 @@ <h2>3 · Deduplication with hashes</h2>
       </div>
       <pre class="playground-editor" contenteditable="true" spellcheck="false">import { DataFrame, hashPandasObject } from "tsb";
 
-const df = new DataFrame({
+const df = DataFrame.fromColumns({
   a: [1, 2, 1, 3],
   b: ["x", "y", "x", "z"],
 });

playground/hash_pandas_object.html

@@ -176,13 +176,13 @@ <h2>Basic outer ordered merge</h2>
 });
 
 const result = mergeOrdered(left, right, { on: "date" });
+console.log(result.toString());
 // date | price | volume
 //    1 |    10 |   null
 //    2 |  null |    200
 //    3 |    30 |    300
 //    5 |    50 |   null
-//    6 |  null |    600
-console.log(result);</textarea>
+//    6 |  null |    600</textarea>
       <div class="playground-output">Click ▶ Run to execute</div>
       <div class="playground-hint">Ctrl+Enter to run · Tab to indent</div>
     </div>
@@ -198,17 +198,28 @@ <h2>Forward-fill after merge</h2>
           <button class="playground-reset">↺ Reset</button>
         </div>
       </div>
-      <textarea class="playground-editor" spellcheck="false">const result = mergeOrdered(left, right, {
+      <textarea class="playground-editor" spellcheck="false">import { DataFrame, mergeOrdered } from "tsb";
+
+const left = DataFrame.fromColumns({
+  date:  [1, 3, 5],
+  price: [10, 30, 50],
+});
+const right = DataFrame.fromColumns({
+  date:   [2, 3, 6],
+  volume: [200, 300, 600],
+});
+
+const result = mergeOrdered(left, right, {
   on: "date",
   fill_method: "ffill",
 });
+console.log(result.toString());
 // date | price | volume
 //    1 |    10 |   null   ← no earlier price to fill
 //    2 |    10 |    200   ← price carried forward from date=1
 //    3 |    30 |    300
 //    5 |    50 |    300   ← volume carried forward from date=3
-//    6 |    50 |    600
-console.log(result);</textarea>
+//    6 |    50 |    600</textarea>
       <div class="playground-output">Click ▶ Run to execute</div>
       <div class="playground-hint">Ctrl+Enter to run · Tab to indent</div>
     </div>
@@ -224,8 +235,14 @@ <h2>Inner join variant</h2>
           <button class="playground-reset">↺ Reset</button>
         </div>
       </div>
-      <textarea class="playground-editor" spellcheck="false">// Only rows where both DataFrames have a key
-console.log(mergeOrdered(left, right, { on: "date", how: "inner" }));
+      <textarea class="playground-editor" spellcheck="false">import { DataFrame, mergeOrdered } from "tsb";
+
+const left  = DataFrame.fromColumns({ date: [1, 3, 5], price:  [10, 30, 50]    });
+const right = DataFrame.fromColumns({ date: [2, 3, 6], volume: [200, 300, 600] });
+
+// Only rows where both DataFrames have a key
+const result = mergeOrdered(left, right, { on: "date", how: "inner" });
+console.log(result.toString());
 // date | price | volume
 //    3 |    30 |    300</textarea>
       <div class="playground-output">Click ▶ Run to execute</div>
@@ -243,10 +260,13 @@ <h2>Different key column names per side</h2>
           <button class="playground-reset">↺ Reset</button>
         </div>
       </div>
-      <textarea class="playground-editor" spellcheck="false">const left2 = DataFrame.fromColumns({ t_left:  [1, 3, 5], a: [10, 30, 50] });
+      <textarea class="playground-editor" spellcheck="false">import { DataFrame, mergeOrdered } from "tsb";
+
+const left2  = DataFrame.fromColumns({ t_left:  [1, 3, 5], a: [10, 30, 50] });
 const right2 = DataFrame.fromColumns({ t_right: [2, 3, 6], b: [200, 300, 600] });
 
-console.log(mergeOrdered(left2, right2, { left_on: "t_left", right_on: "t_right" }));
+const result = mergeOrdered(left2, right2, { left_on: "t_left", right_on: "t_right" });
+console.log(result.toString());
 // t_left | a    | b
 //      1 |   10 | null
 //      2 | null |  200
@@ -268,7 +288,9 @@ <h2>Group-wise ordered merge (left_by / right_by)</h2>
           <button class="playground-reset">↺ Reset</button>
         </div>
       </div>
-      <textarea class="playground-editor" spellcheck="false">// Perform the ordered merge independently for each group
+      <textarea class="playground-editor" spellcheck="false">import { DataFrame, mergeOrdered } from "tsb";
+
+// Perform the ordered merge independently for each group
 const left3 = DataFrame.fromColumns({
   grp: ["A", "A", "B", "B"],
   k:   [1,   3,   1,   3],
@@ -280,19 +302,19 @@ <h2>Group-wise ordered merge (left_by / right_by)</h2>
   b:   [20,  30, 200, 300],
 });
 
-mergeOrdered(left3, right3, {
+const result = mergeOrdered(left3, right3, {
   on: "k",
   left_by: "grp",
   right_by: "grp",
 });
+console.log(result.toString());
 // grp | k | a    | b
 //   A | 1 |   10 | null
 //   A | 2 | null |   20
 //   A | 3 |   30 |   30
 //   B | 1 |  100 | null
 //   B | 2 | null |  200
-//   B | 3 |  300 |  300
-console.log(right3);</textarea>
+//   B | 3 |  300 |  300</textarea>
       <div class="playground-output">Click ▶ Run to execute</div>
       <div class="playground-hint">Ctrl+Enter to run · Tab to indent</div>
     </div>
@@ -308,10 +330,13 @@ <h2>Overlapping non-key columns — suffixes</h2>
           <button class="playground-reset">↺ Reset</button>
         </div>
       </div>
-      <textarea class="playground-editor" spellcheck="false">const left4 = DataFrame.fromColumns({ k: [1, 2, 3], val: [10, 20, 30] });
+      <textarea class="playground-editor" spellcheck="false">import { DataFrame, mergeOrdered } from "tsb";
+
+const left4  = DataFrame.fromColumns({ k: [1, 2, 3], val: [10, 20, 30] });
 const right4 = DataFrame.fromColumns({ k: [2, 3, 4], val: [200, 300, 400] });
 
-console.log(mergeOrdered(left4, right4, { on: "k", suffixes: ["_L", "_R"] }));
+const result = mergeOrdered(left4, right4, { on: "k", suffixes: ["_L", "_R"] });
+console.log(result.toString());
 // k | val_L | val_R
 // 1 |    10 |  null
 // 2 |    20 |   200

playground/merge_ordered.html

@@ -0,0 +1,63 @@
+{
+  "align.html": [2, 3, 5, 6, 7],
+  "api_types.html": [2, 3, 4, 5, 6, 7],
+  "assign.html": [2, 3, 4, 5],
+  "at_iat.html": [1, 3, 4, 5, 7, 8, 9],
+  "attrs.html": [2, 3, 4, 5, 6],
+  "between.html": [1, 3, 5, 7],
+  "clip_advanced.html": [1, 2, 3, 4, 5, 6],
+  "corrwith.html": [1, 3, 5, 7],
+  "crosstab.html": [2, 4, 6, 8, 10, 12],
+  "cut.html": [2, 3, 4, 5, 6, 8],
+  "cut_bins_to_frame.html": [1],
+  "cut_qcut.html": [1, 2, 4, 5, 6, 7],
+  "datetime_tz.html": [2, 4, 6, 8, 10],
+  "dot_matmul.html": [1, 2],
+  "eval_query.html": [1, 2, 3],
+  "excel.html": [1, 2, 3, 4],
+  "factorize.html": [2, 4, 6, 8],
+  "filter.html": [1, 3, 5, 7, 9],
+  "get_dummies.html": [2, 4, 6, 8],
+  "infer_dtype.html": [5],
+  "insert_pop.html": [6],
+  "interpolate.html": [2, 6],
+  "join.html": [2, 3, 4, 5, 6],
+  "json_normalize.html": [2, 3],
+  "memory_usage.html": [1, 2, 3, 4, 5],
+  "merge_asof.html": [2, 3, 4, 5, 6, 7],
+  "mode.html": [1, 2, 3, 4, 5, 6],
+  "named_agg.html": [2, 3, 5, 6, 7],
+  "natsort.html": [2, 6],
+  "nunique.html": [2, 3, 5],
+  "pipe_apply.html": [1, 3],
+  "pivot_table.html": [2, 3, 4],
+  "pow_mod.html": [5],
+  "quantile.html": [1, 2, 3, 4, 5, 6, 7],
+  "reduce_ops.html": [1, 3, 4, 6],
+  "reindex.html": [6],
+  "rolling_apply.html": [1, 2, 3, 4, 5, 6],
+  "scalar_extract.html": [1, 3, 5, 7, 9, 11],
+  "searchsorted.html": [6],
+  "select_dtypes.html": [2, 3, 4, 5],
+  "sem_var.html": [2, 3, 5],
+  "skew_kurt.html": [1, 2, 3, 4, 5, 6],
+  "sort_ops.html": [1, 3, 4, 5, 6, 7, 9, 10, 11],
+  "str_findall_and_json_denormalize.html": [2, 3, 4, 5, 6, 8, 9, 10, 11, 12],
+  "str_get_dummies.html": [2, 4, 6, 8],
+  "style.html": [
+    2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25
+  ],
+  "swaplevel.html": [1, 2, 3, 4, 5, 6, 7, 8],
+  "testing.html": [1, 3, 4, 6, 8],
+  "to_datetime.html": [2],
+  "to_from_dict.html": [1, 2],
+  "to_numeric.html": [1, 2, 3, 4, 5],
+  "to_timedelta.html": [2],
+  "transform_agg.html": [1, 2],
+  "truncate.html": [1, 3, 5, 7],
+  "update.html": [1, 3, 5],
+  "value_counts_full.html": [1, 3, 4, 5, 6],
+  "wide_to_long.html": [1, 2, 3, 4],
+  "window_extended.html": [1, 3, 5],
+  "xs.html": [1]
+}