Beta E2E (browser) #106
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Beta E2E (browser) | |
| # Manual gate for "is beta safe to promote to production?" and reusable job | |
| # for the scheduled beta health check. | |
| # | |
| # The public lane is sharded one host per runner. Measured: a page load against | |
| # a beta host costs 1-2s from a laptop and 20-40s from a GitHub runner, because | |
| # the fleet sits behind one CDN that throttles bursty datacenter traffic. On a | |
| # single runner the sweep took ~28 minutes with scheduling already 83% | |
| # efficient, so the time was raw network wait rather than queueing. Sharding | |
| # makes the fleet cost the slowest single host and spreads the requests over | |
| # sixteen source addresses: measured 1704s -> 437s. | |
| # | |
| # Drives the deployed beta fleet with a real browser. Two gating lanes decide | |
| # the answer; a third advisory lane reports real-but-not-blocking findings so a | |
| # red run always means "do not promote". | |
| # | |
| # The workflow itself has no push or schedule trigger. The authenticated lane | |
| # spends model tokens against the credential selected by its caller, and most | |
| # beta apps share a database with production, so the explicit manual entrypoint | |
| # remains a deliberate promotion decision. The separate scheduled wrapper | |
| # calls this workflow on a six-hour cadence with the hosted sites' shared | |
| # credential and reports failures to one deduplicated issue. | |
| on: | |
| # Callable from another workflow - used by the production deploy's opt-in | |
| # pre-flight gate and the six-hour scheduled wrapper. Still never fires on | |
| # its own: there is no push, schedule, or pull_request trigger here. | |
| workflow_call: | |
| inputs: | |
| apps: | |
| description: 'Beta apps to test ("all", or e.g. "slides,analytics")' | |
| required: false | |
| default: "all" | |
| type: string | |
| lane: | |
| description: "Which lanes to run (public, authed, public+authed, or signup)" | |
| required: false | |
| default: "public+authed" | |
| type: string | |
| signup_apps: | |
| description: "Apps for the signup canary" | |
| required: false | |
| default: "all" | |
| type: string | |
| signup_environments: | |
| description: "Environments for the signup canary" | |
| required: false | |
| default: "beta" | |
| type: string | |
| grep: | |
| description: "Only run tests whose title matches this (optional)" | |
| required: false | |
| default: "" | |
| type: string | |
| key_source: | |
| description: "Which OpenAI credential the agent turns bill" | |
| required: false | |
| default: dedicated | |
| type: string | |
| secrets: | |
| BETA_E2E_EMAIL: | |
| required: false | |
| BETA_E2E_SESSION_TOKENS: | |
| required: false | |
| BETA_E2E_SESSION_TOKEN_CRM: | |
| required: false | |
| BETA_E2E_SESSION_TOKEN_CHAT: | |
| required: false | |
| BETA_E2E_OPENAI_API_KEY: | |
| required: false | |
| OPENAI_API_KEY: | |
| required: false | |
| MAILOSAUR_API_KEY: | |
| required: false | |
| MAILOSAUR_SERVER_ID: | |
| required: false | |
| PAGERDUTY_ROUTING_KEY: | |
| required: false | |
| outputs: | |
| signup_outcome: | |
| description: "Signup lane result: success, failure, or inconclusive." | |
| value: ${{ jobs.signup.outputs.outcome }} | |
| workflow_dispatch: | |
| inputs: | |
| apps: | |
| description: 'Beta apps to test ("all", or e.g. "slides,analytics,chat")' | |
| required: false | |
| default: "all" | |
| lane: | |
| description: "Which lanes to run" | |
| required: false | |
| default: "public+authed" | |
| type: choice | |
| options: | |
| - public | |
| - authed | |
| - public+authed | |
| - signup | |
| signup_apps: | |
| description: "Apps for the signup canary" | |
| required: false | |
| default: "all" | |
| type: string | |
| signup_environments: | |
| description: "Environments for the signup canary" | |
| required: false | |
| default: "beta" | |
| type: string | |
| key_source: | |
| description: "Which OpenAI credential the agent turns bill" | |
| required: false | |
| default: dedicated | |
| type: choice | |
| options: | |
| # A key created for this suite with its own spend limit, stored as | |
| # BETA_E2E_OPENAI_API_KEY. Keeps this suite's cost attributable. | |
| - dedicated | |
| # The repository's shared OPENAI_API_KEY. Works, but pools this | |
| # suite's spend with every other consumer of that key. | |
| - shared | |
| grep: | |
| description: "Only run tests whose title matches this (optional)" | |
| required: false | |
| default: "" | |
| concurrency: | |
| # One promotion check at a time. Two concurrent runs would sign in as the | |
| # same identity and interleave agent turns in the same threads. | |
| group: beta-e2e | |
| cancel-in-progress: false | |
| permissions: | |
| contents: read | |
| jobs: | |
| discover: | |
| name: Resolve host shards | |
| if: ${{ inputs.lane != 'signup' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| outputs: | |
| matrix: ${{ steps.matrix.outputs.matrix }} | |
| apps: ${{ steps.matrix.outputs.apps }} | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| with: | |
| fetch-depth: 1 | |
| sparse-checkout: scripts/netlify-beta-sites.json | |
| sparse-checkout-cone-mode: false | |
| - name: Build the host matrix | |
| id: matrix | |
| env: | |
| REQUESTED_APPS: ${{ inputs.apps }} | |
| run: | | |
| set -euo pipefail | |
| node <<'NODE' | |
| const fs = require("node:fs"); | |
| const sites = JSON.parse( | |
| fs.readFileSync("scripts/netlify-beta-sites.json", "utf8"), | |
| ); | |
| const known = sites | |
| .filter((site) => site.e2e !== false) | |
| .map((site) => site.id); | |
| const raw = (process.env.REQUESTED_APPS || "all").trim(); | |
| let apps; | |
| if (!raw || raw === "all") { | |
| apps = [...new Set(known)]; | |
| } else { | |
| apps = [ | |
| ...new Set( | |
| raw | |
| .split(",") | |
| .map((value) => value.trim().toLowerCase()) | |
| .filter(Boolean), | |
| ), | |
| ]; | |
| const unknown = apps.filter((id) => !known.includes(id)); | |
| if (unknown.length > 0) { | |
| throw new Error( | |
| `Unknown app(s): ${unknown.join(", ")}. Known: ${known.join(", ")}`, | |
| ); | |
| } | |
| } | |
| if (apps.length === 0) { | |
| throw new Error("The app selection resolved to nothing to test."); | |
| } | |
| const matrix = { app: apps }; | |
| fs.appendFileSync( | |
| process.env.GITHUB_OUTPUT, | |
| `matrix=${JSON.stringify(matrix)}\napps=${apps.join(",")}\n`, | |
| ); | |
| console.log(`Sharding ${apps.length} host(s): ${apps.join(", ")}`); | |
| NODE | |
| public: | |
| # Lane jobs below are ordered because several beta apps share production | |
| # databases; always() keeps later evidence running after ordinary failure. | |
| name: ${{ matrix.app }} public sweep | |
| needs: discover | |
| if: ${{ needs.discover.result == 'success' && inputs.lane != 'authed' && inputs.lane != 'signup' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 25 | |
| strategy: | |
| fail-fast: false | |
| # Keep public shards bounded: independent hosts can still share backend | |
| # capacity, and the fleet-wide lanes run after this matrix completes. | |
| max-parallel: 4 | |
| matrix: ${{ fromJSON(needs.discover.outputs.matrix) }} | |
| env: | |
| BETA_E2E_APPS: ${{ matrix.app }} | |
| BETA_E2E_GREP: ${{ inputs.grep || '' }} | |
| BETA_E2E_AUTHED: "0" | |
| BETA_E2E_REPORT_SLOT: public-${{ matrix.app }} | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| - uses: ./.github/actions/beta-e2e-setup | |
| # A composite action rather than repeated steps: the same seven-step | |
| # preamble now runs in four places. | |
| - name: Public sweep | |
| run: | | |
| if [ -n "$BETA_E2E_GREP" ]; then | |
| pnpm e2e:beta --project=public --grep "$BETA_E2E_GREP" --pass-with-no-tests | |
| else | |
| pnpm e2e:beta --project=public | |
| fi | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| if: ${{ always() }} | |
| with: | |
| name: beta-e2e-public-${{ matrix.app }}-${{ github.run_id }} | |
| path: | | |
| e2e/beta/playwright-report/ | |
| e2e/beta/test-results/ | |
| if-no-files-found: warn | |
| retention-days: 7 | |
| fleet: | |
| name: Fleet-wide checks | |
| needs: [discover, public] | |
| if: ${{ always() && !cancelled() && needs.discover.result == 'success' && inputs.lane != 'authed' && inputs.lane != 'signup' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| env: | |
| BETA_E2E_APPS: ${{ needs.discover.outputs.apps }} | |
| BETA_E2E_GREP: ${{ inputs.grep || '' }} | |
| BETA_E2E_AUTHED: "0" | |
| BETA_E2E_REPORT_SLOT: fleet | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| - uses: ./.github/actions/beta-e2e-setup | |
| - name: Cross-host checks | |
| run: | | |
| if [ -n "$BETA_E2E_GREP" ]; then | |
| pnpm e2e:beta --project=fleet --grep "$BETA_E2E_GREP" --pass-with-no-tests | |
| else | |
| pnpm e2e:beta --project=fleet | |
| fi | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| if: ${{ always() }} | |
| with: | |
| name: beta-e2e-fleet-${{ github.run_id }} | |
| path: e2e/beta/playwright-report/ | |
| if-no-files-found: warn | |
| retention-days: 7 | |
| authed: | |
| name: Authenticated ${{ matrix.cluster }} | |
| needs: [discover, advisory] | |
| if: ${{ always() && !cancelled() && needs.discover.result == 'success' && inputs.lane != 'public' && inputs.lane != 'signup' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 45 | |
| strategy: | |
| fail-fast: false | |
| # These clusters share the e2e identity and production-backed databases. | |
| # Keep them serialized while giving each failure its own gating signal. | |
| max-parallel: 1 | |
| # Keep the authenticated projects declarative so guard:beta-e2e-suite | |
| # can verify that each cluster remains independently visible. The steps | |
| # below skip unrelated clusters for a Design-only selection. | |
| matrix: | |
| include: | |
| - cluster: registry | |
| project: registry | |
| - cluster: chat | |
| project: chat | |
| - cluster: journeys | |
| project: journeys | |
| - cluster: design | |
| project: design | |
| env: | |
| BETA_E2E_APPS: ${{ matrix.cluster == 'design' && 'design' || needs.discover.outputs.apps }} | |
| BETA_E2E_GREP: ${{ inputs.grep || '' }} | |
| BETA_E2E_AUTHED: "1" | |
| BETA_E2E_CLUSTER: ${{ matrix.cluster }} | |
| BETA_E2E_REPORT_SLOT: authed-${{ matrix.cluster }} | |
| BETA_E2E_EMAIL: ${{ secrets.BETA_E2E_EMAIL }} | |
| BETA_E2E_SESSION_TOKENS: ${{ secrets.BETA_E2E_SESSION_TOKENS }} | |
| BETA_E2E_SESSION_TOKEN_CRM: ${{ secrets.BETA_E2E_SESSION_TOKEN_CRM }} | |
| BETA_E2E_SESSION_TOKEN_CHAT: ${{ secrets.BETA_E2E_SESSION_TOKEN_CHAT }} | |
| BETA_E2E_OPENAI_API_KEY: ${{ secrets.BETA_E2E_OPENAI_API_KEY }} | |
| # The repository-wide key is available only for an explicit shared-key | |
| # dispatch. Dedicated runs must not expose it to setup or test code. | |
| BETA_E2E_SHARED_OPENAI_API_KEY: ${{ inputs.key_source == 'shared' && secrets.OPENAI_API_KEY || '' }} | |
| BETA_E2E_ALLOW_SHARED_KEY: ${{ inputs.key_source == 'shared' }} | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| if: ${{ matrix.cluster == 'design' && contains(needs.discover.outputs.apps, 'design') || matrix.cluster != 'design' && needs.discover.outputs.apps != 'design' }} | |
| - uses: ./.github/actions/beta-e2e-setup | |
| if: ${{ matrix.cluster == 'design' && contains(needs.discover.outputs.apps, 'design') || matrix.cluster != 'design' && needs.discover.outputs.apps != 'design' }} | |
| - name: Authenticated ${{ matrix.cluster }} | |
| if: ${{ matrix.cluster == 'design' && contains(needs.discover.outputs.apps, 'design') || matrix.cluster != 'design' && needs.discover.outputs.apps != 'design' }} | |
| # Global setup fails loudly if a credential is missing rather than | |
| # degrading to an anonymous run that would pass without testing | |
| # anything. The clusters are separate so a provider failure cannot | |
| # hide registry or journey regressions. | |
| # A title filter can select only public/fleet tests, though. Ask | |
| # Playwright for the authenticated selection first with auth disabled; | |
| # --list does not run globalSetup, so an empty selection skips the | |
| # credentialed setup without turning a requested auth run into a pass. | |
| run: | | |
| if [ -n "$BETA_E2E_GREP" ]; then | |
| selection_file="$(mktemp)" | |
| trap 'rm -f "$selection_file"' EXIT | |
| set +e | |
| BETA_E2E_AUTHED=0 pnpm e2e:beta \ | |
| --project=${{ matrix.project }} \ | |
| --grep "$BETA_E2E_GREP" --list >"$selection_file" 2>&1 | |
| selection_status="$?" | |
| set -e | |
| cat "$selection_file" | |
| if [ "$selection_status" -ne 0 ]; then | |
| if grep -q "Error: No tests found" "$selection_file" && grep -q "Total: 0 tests in 0 files" "$selection_file"; then | |
| echo "No authenticated tests match BETA_E2E_GREP; skipping authenticated setup." | |
| exit 0 | |
| fi | |
| echo "Authenticated test selection failed before execution." >&2 | |
| exit "$selection_status" | |
| fi | |
| pnpm e2e:beta --project=${{ matrix.project }} --grep "$BETA_E2E_GREP" --pass-with-no-tests | |
| else | |
| pnpm e2e:beta --project=${{ matrix.project }} | |
| fi | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| if: ${{ always() && (matrix.cluster == 'design' && contains(needs.discover.outputs.apps, 'design') || matrix.cluster != 'design' && needs.discover.outputs.apps != 'design') }} | |
| # The authenticated lanes record no trace and no video (see the | |
| # playwright config): a trace captures real request headers and would | |
| # publish the e2e account's live session cookie to anyone with repo | |
| # read access. e2e/beta/.auth is deliberately not in this list. | |
| with: | |
| name: beta-e2e-authed-${{ matrix.cluster }}-${{ github.run_id }} | |
| path: | | |
| e2e/beta/playwright-report/ | |
| e2e/beta/test-results/ | |
| if-no-files-found: warn | |
| retention-days: 7 | |
| advisory: | |
| name: Advisory findings (non-gating) | |
| if: ${{ always() && !cancelled() && needs.discover.result == 'success' && inputs.lane != 'authed' && inputs.lane != 'signup' }} | |
| needs: [discover, fleet] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| env: | |
| BETA_E2E_APPS: ${{ needs.discover.outputs.apps }} | |
| BETA_E2E_GREP: ${{ inputs.grep || '' }} | |
| BETA_E2E_AUTHED: "0" | |
| BETA_E2E_REPORT_SLOT: advisory | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| - uses: ./.github/actions/beta-e2e-setup | |
| - name: Advisory findings | |
| continue-on-error: true | |
| # Always runs: it costs no credentials and no tokens, and findings that | |
| # stop being reported stop being fixed. Step-level continue-on-error | |
| # keeps the reusable workflow green while preserving the failed step | |
| # and uploaded report for inspection. | |
| run: | | |
| if [ -n "$BETA_E2E_GREP" ]; then | |
| pnpm e2e:beta --project=advisory --grep "$BETA_E2E_GREP" --pass-with-no-tests | |
| else | |
| pnpm e2e:beta --project=advisory | |
| fi | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| if: ${{ always() }} | |
| with: | |
| name: beta-e2e-advisory-${{ github.run_id }} | |
| path: e2e/beta/playwright-report/ | |
| if-no-files-found: warn | |
| retention-days: 7 | |
| signup: | |
| name: Signup canary | |
| if: ${{ inputs.lane == 'signup' }} | |
| outputs: | |
| outcome: ${{ steps.classify.outputs.outcome }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 180 | |
| env: | |
| SIGNUP_E2E_APPS: ${{ inputs.signup_apps || 'all' }} | |
| SIGNUP_E2E_ENVIRONMENTS: ${{ inputs.signup_environments || 'beta' }} | |
| MAILOSAUR_API_KEY: ${{ secrets.MAILOSAUR_API_KEY }} | |
| MAILOSAUR_SERVER_ID: ${{ secrets.MAILOSAUR_SERVER_ID }} | |
| PAGERDUTY_ROUTING_KEY: ${{ secrets.PAGERDUTY_ROUTING_KEY }} | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| - uses: ./.github/actions/beta-e2e-setup | |
| - name: Test signup email provider contracts | |
| run: pnpm exec tsx --test e2e/signup/lib/mailosaur.spec.ts | |
| - name: Typecheck signup suite | |
| run: pnpm typecheck:e2e:signup | |
| - name: Run full signup flow | |
| id: signup | |
| run: pnpm e2e:signup | |
| - name: Classify signup result | |
| id: classify | |
| if: ${{ always() }} | |
| run: | | |
| set -euo pipefail | |
| if [ "${{ steps.signup.outcome }}" != "success" ]; then | |
| echo "outcome=failure" >> "$GITHUB_OUTPUT" | |
| elif find e2e/signup/test-results -type f -name 'mailosaur-inconclusive.txt' -print -quit | grep -q .; then | |
| echo "outcome=inconclusive" >> "$GITHUB_OUTPUT" | |
| echo "INCONCLUSIVE: Mailosaur rate-limited the verification inbox; no product result was recorded." >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| echo "outcome=success" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Page signup health on-call | |
| if: ${{ failure() && steps.signup.outcome == 'failure' && steps.classify.outputs.outcome == 'failure' }} | |
| run: | | |
| set -euo pipefail | |
| if [ -z "${PAGERDUTY_ROUTING_KEY:-}" ]; then | |
| echo "::warning::PAGERDUTY_ROUTING_KEY is not configured; the GitHub issue fallback is active but no page was sent." | |
| exit 0 | |
| fi | |
| jq -n \ | |
| --arg routing_key "$PAGERDUTY_ROUTING_KEY" \ | |
| --arg dedup_key "agent-native-signup-e2e" \ | |
| --arg source "${{ github.repository }}" \ | |
| --arg run_url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ | |
| '{routing_key: $routing_key, event_action: "trigger", dedup_key: $dedup_key, payload: {summary: "Agent-Native signup E2E is failing", source: $source, severity: "critical", custom_details: {run_url: $run_url, apps: env.SIGNUP_E2E_APPS, environments: env.SIGNUP_E2E_ENVIRONMENTS}}}' \ | |
| | curl --fail-with-body --connect-timeout 10 --max-time 20 --silent --show-error \ | |
| --header 'Content-Type: application/json' \ | |
| --data-binary @- \ | |
| https://events.pagerduty.com/v2/enqueue | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| if: ${{ always() }} | |
| with: | |
| name: signup-e2e-${{ github.run_id }} | |
| path: e2e/signup/test-results/ | |
| if-no-files-found: warn | |
| retention-days: 14 |