test: add a11y and lighthouse

README.md

@@ -120,8 +120,10 @@ Test
 End-to-End tests
 
     pnpm playwright install chromium
+
     pnpm test:e2e --project=chromium
-    pnpm test:e2e:dev --project=chromium
+    pnpm test:e2e --project=a11y
+    pnpm test:e2e --project=lighthouse
 
 Run format, lint, typecheck and test
 

e2e/a11y.ts

@@ -0,0 +1,41 @@
+import AxeBuilder from "@axe-core/playwright";
+import { type Page, expect, test } from "@playwright/test";
+import type axe from "axe-core";
+import { createHtmlReport } from "axe-html-reporter";
+
+import { attachReport, clientCredentials, throwOnConsoleError } from "./utils";
+
+const htmlReport = async (name: string, results: axe.AxeResults) => {
+  const reportFileName = `${test.info().project.name}-${name}.html`;
+
+  createHtmlReport({
+    results,
+    options: {
+      outputDir: "reports/a11y",
+      reportFileName
+    }
+  });
+
+  return `reports/a11y/${reportFileName}`;
+};
+
+const a11y = async (name: string, page: Page) => {
+  const results = await new AxeBuilder({ page }).analyze();
+  const path = await htmlReport(name, results);
+  await attachReport("axe-report", path);
+
+  return results;
+};
+
+test.describe("accessibility", () => {
+  test.describe.configure({ mode: "serial" });
+
+  test("home", async ({ page }) => {
+    throwOnConsoleError(page);
+    await clientCredentials(page);
+
+    await page.goto("/");
+    const results = await a11y("home", page);
+    expect(results.violations).toEqual([]);
+  });
+});

e2e/lighthouse.ts

@@ -0,0 +1,56 @@
+import test, { type BrowserContext, type Page, chromium } from "@playwright/test";
+import { playAudit } from "playwright-lighthouse";
+
+import { attachReport, clientCredentials } from "./utils";
+
+const threshold = process.env.LIGHTHOUSE_THRESHOLD
+  ? Number(process.env.LIGHTHOUSE_THRESHOLD)
+  : 0;
+
+const lighthouse = async (page: Page, name: string) => {
+  await playAudit({
+    page,
+    port: 9222,
+    thresholds: {
+      performance: threshold,
+      accessibility: threshold,
+      "best-practices": threshold,
+      seo: threshold,
+      pwa: threshold
+    },
+    reports: {
+      name,
+      directory: "reports/lighthouse/",
+      formats: {
+        json: false,
+        html: true
+      }
+    }
+  });
+
+  await attachReport("lighthouse-report", `reports/lighthouse/${name}.html`);
+};
+
+test.describe("lighthouse", () => {
+  test.describe.configure({ mode: "serial" });
+
+  let context: BrowserContext;
+
+  test.beforeAll(async () => {
+    context = await chromium.launchPersistentContext(".playwright/chromium-lighthouse", {
+      args: ["--remote-debugging-port=9222", "--ignore-certificate-errors"]
+    });
+  });
+
+  test.afterAll(async () => {
+    await context.close();
+  });
+
+  test("home", async () => {
+    const page = await context.newPage();
+    await clientCredentials(page);
+
+    await page.goto("/");
+    await lighthouse(page, "home");
+  });
+});

e2e/utils.ts

@@ -1,4 +1,4 @@
-import { type Page } from "@playwright/test";
+import test, { type Page } from "@playwright/test";
 
 export const throwOnConsoleError = (page: Page): void => {
   page.on("console", (message) => {
@@ -22,3 +22,10 @@ export const clientCredentials = async (page: Page): Promise<void> => {
     await route.continue({ headers });
   });
 };
+
+/// NOTE: currently forced download, see https://github.com/microsoft/playwright/pull/33267
+export const attachReport = (name: string, path: string): Promise<void> =>
+  test.info().attach(name, {
+    path,
+    contentType: "text/html"
+  });

package.json

@@ -47,6 +47,7 @@
     "zod": "3.24.1"
   },
   "devDependencies": {
+    "@axe-core/playwright": "4.10.1",
     "@eslint/js": "9.17.0",
     "@playwright/test": "1.49.1",
     "@testing-library/jest-dom": "6.6.3",
@@ -56,13 +57,17 @@
     "@types/lodash-es": "4.17.12",
     "@vitest/coverage-v8": "2.1.8",
     "autoprefixer": "10.4.20",
+    "axe-core": "4.10.2",
+    "axe-html-reporter": "2.2.11",
     "better-sqlite3": "11.7.0",
     "drizzle-kit": "0.30.1",
     "eslint": "9.17.0",
     "eslint-plugin-jsx-a11y": "6.10.2",
     "eslint-plugin-solid": "0.14.5",
     "fast-check": "3.23.2",
     "globals": "15.14.0",
+    "lighthouse": "12.3.0",
+    "playwright-lighthouse": "4.0.0",
     "postcss": "8.4.49",
     "prettier": "3.4.2",
     "prettier-plugin-sql": "0.18.1",

playwright.config.ts

@@ -20,6 +20,7 @@ export default defineConfig({
   use: {
     baseURL: process.env.BASE_URL ?? "http://localhost:3000",
     trace: "on-first-retry",
+    screenshot: "only-on-failure",
     viewport: { width: 1920, height: 1080 }
   },
   reporter: [
@@ -38,6 +39,32 @@ export default defineConfig({
     {
       name: "webkit",
       use: { ...devices["Desktop Safari"] }
+    },
+    {
+      name: "iphone-15",
+      use: { ...devices["iPhone 15"] }
+    },
+    {
+      name: "a11y",
+      testMatch: "e2e/a11y.ts",
+      use: { ...devices["Desktop Chrome"], screenshot: "on" }
+    },
+    {
+      name: "a11y-dark",
+      testMatch: "e2e/a11y.ts",
+      use: { ...devices["Desktop Chrome"], screenshot: "on", colorScheme: "dark" }
+    },
+    {
+      name: "a11y-mobile",
+      testMatch: "e2e/a11y.ts",
+      use: { ...devices["iPhone 15"], screenshot: "on", colorScheme: "dark" }
+    },
+    {
+      name: "lighthouse",
+      testMatch: "e2e/lighthouse.ts",
+      use: {
+        screenshot: "off"
+      }
     }
   ]
 });