beta-e2e.yml

name: Beta E2E (browser)

# Manual gate for "is beta safe to promote to production?" and reusable job
# for the scheduled beta health check.
#
# The public lane is sharded one host per runner. Measured: a page load against
# a beta host costs 1-2s from a laptop and 20-40s from a GitHub runner, because
# the fleet sits behind one CDN that throttles bursty datacenter traffic. On a
# single runner the sweep took ~28 minutes with scheduling already 83%
# efficient, so the time was raw network wait rather than queueing. Sharding
# makes the fleet cost the slowest single host and spreads the requests over
# sixteen source addresses: measured 1704s -> 437s.
#
# Drives the deployed beta fleet with a real browser. Two gating lanes decide
# the answer; a third advisory lane reports real-but-not-blocking findings so a
# red run always means "do not promote".
#
# The workflow itself has no push or schedule trigger. The authenticated lane
# spends model tokens against the credential selected by its caller, and most
# beta apps share a database with production, so the explicit manual entrypoint
# remains a deliberate promotion decision. The separate scheduled wrapper
# calls this workflow on a six-hour cadence with the hosted sites' shared
# credential and reports failures to one deduplicated issue.

on:
  # Callable from another workflow - used by the production deploy's opt-in
  # pre-flight gate and the six-hour scheduled wrapper. Still never fires on
  # its own: there is no push, schedule, or pull_request trigger here.
  workflow_call:
    inputs:
      apps:
        description: 'Beta apps to test ("all", or e.g. "slides,analytics")'
        required: false
        default: "all"
        type: string
      lane:
        description: "Which lanes to run (public, authed, public+authed, or signup)"
        required: false
        default: "public+authed"
        type: string
      signup_apps:
        description: "Apps for the signup canary"
        required: false
        default: "all"
        type: string
      signup_environments:
        description: "Environments for the signup canary"
        required: false
        default: "beta"
        type: string
      grep:
        description: "Only run tests whose title matches this (optional)"
        required: false
        default: ""
        type: string
      key_source:
        description: "Which OpenAI credential the agent turns bill"
        required: false
        default: dedicated
        type: string
    secrets:
      BETA_E2E_EMAIL:
        required: false
      BETA_E2E_SESSION_TOKENS:
        required: false
      BETA_E2E_SESSION_TOKEN_CRM:
        required: false
      BETA_E2E_SESSION_TOKEN_CHAT:
        required: false
      BETA_E2E_OPENAI_API_KEY:
        required: false
      OPENAI_API_KEY:
        required: false
      MAILOSAUR_API_KEY:
        required: false
      MAILOSAUR_SERVER_ID:
        required: false
      PAGERDUTY_ROUTING_KEY:
        required: false
    outputs:
      signup_outcome:
        description: "Signup lane result: success, failure, or inconclusive."
        value: ${{ jobs.signup.outputs.outcome }}
  workflow_dispatch:
    inputs:
      apps:
        description: 'Beta apps to test ("all", or e.g. "slides,analytics,chat")'
        required: false
        default: "all"
      lane:
        description: "Which lanes to run"
        required: false
        default: "public+authed"
        type: choice
        options:
          - public
          - authed
          - public+authed
          - signup
      signup_apps:
        description: "Apps for the signup canary"
        required: false
        default: "all"
        type: string
      signup_environments:
        description: "Environments for the signup canary"
        required: false
        default: "beta"
        type: string
      key_source:
        description: "Which OpenAI credential the agent turns bill"
        required: false
        default: dedicated
        type: choice
        options:
          # A key created for this suite with its own spend limit, stored as
          # BETA_E2E_OPENAI_API_KEY. Keeps this suite's cost attributable.
          - dedicated
          # The repository's shared OPENAI_API_KEY. Works, but pools this
          # suite's spend with every other consumer of that key.
          - shared
      grep:
        description: "Only run tests whose title matches this (optional)"
        required: false
        default: ""

concurrency:
  # One promotion check at a time. Two concurrent runs would sign in as the
  # same identity and interleave agent turns in the same threads.
  group: beta-e2e
  cancel-in-progress: false

permissions:
  contents: read

jobs:
  discover:
    name: Resolve host shards
    if: ${{ inputs.lane != 'signup' }}
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      matrix: ${{ steps.matrix.outputs.matrix }}
      apps: ${{ steps.matrix.outputs.apps }}
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
        with:
          fetch-depth: 1
          sparse-checkout: scripts/netlify-beta-sites.json
          sparse-checkout-cone-mode: false

      - name: Build the host matrix
        id: matrix
        env:
          REQUESTED_APPS: ${{ inputs.apps }}
        run: |
          set -euo pipefail
          node <<'NODE'
          const fs = require("node:fs");
          const sites = JSON.parse(
            fs.readFileSync("scripts/netlify-beta-sites.json", "utf8"),
          );
          const known = sites
            .filter((site) => site.e2e !== false)
            .map((site) => site.id);
          const raw = (process.env.REQUESTED_APPS || "all").trim();
          let apps;
          if (!raw || raw === "all") {
            apps = [...new Set(known)];
          } else {
            apps = [
              ...new Set(
                raw
                  .split(",")
                  .map((value) => value.trim().toLowerCase())
                  .filter(Boolean),
              ),
            ];
            const unknown = apps.filter((id) => !known.includes(id));
            if (unknown.length > 0) {
              throw new Error(
                `Unknown app(s): ${unknown.join(", ")}. Known: ${known.join(", ")}`,
              );
            }
          }
          if (apps.length === 0) {
            throw new Error("The app selection resolved to nothing to test.");
          }
          const matrix = { app: apps };
          fs.appendFileSync(
            process.env.GITHUB_OUTPUT,
            `matrix=${JSON.stringify(matrix)}\napps=${apps.join(",")}\n`,
          );
          console.log(`Sharding ${apps.length} host(s): ${apps.join(", ")}`);
          NODE

  public:
    # Lane jobs below are ordered because several beta apps share production
    # databases; always() keeps later evidence running after ordinary failure.
    name: ${{ matrix.app }} public sweep
    needs: discover
    if: ${{ needs.discover.result == 'success' && inputs.lane != 'authed' && inputs.lane != 'signup' }}
    runs-on: ubuntu-latest
    timeout-minutes: 25
    strategy:
      fail-fast: false
      # Keep public shards bounded: independent hosts can still share backend
      # capacity, and the fleet-wide lanes run after this matrix completes.
      max-parallel: 4
      matrix: ${{ fromJSON(needs.discover.outputs.matrix) }}
    env:
      BETA_E2E_APPS: ${{ matrix.app }}
      BETA_E2E_GREP: ${{ inputs.grep || '' }}
      BETA_E2E_AUTHED: "0"
      BETA_E2E_REPORT_SLOT: public-${{ matrix.app }}
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
      - uses: ./.github/actions/beta-e2e-setup
        # A composite action rather than repeated steps: the same seven-step
        # preamble now runs in four places.
      - name: Public sweep
        run: |
          if [ -n "$BETA_E2E_GREP" ]; then
            pnpm e2e:beta --project=public --grep "$BETA_E2E_GREP" --pass-with-no-tests
          else
            pnpm e2e:beta --project=public
          fi
      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: ${{ always() }}
        with:
          name: beta-e2e-public-${{ matrix.app }}-${{ github.run_id }}
          path: |
            e2e/beta/playwright-report/
            e2e/beta/test-results/
          if-no-files-found: warn
          retention-days: 7

  fleet:
    name: Fleet-wide checks
    needs: [discover, public]
    if: ${{ always() && !cancelled() && needs.discover.result == 'success' && inputs.lane != 'authed' && inputs.lane != 'signup' }}
    runs-on: ubuntu-latest
    timeout-minutes: 20
    env:
      BETA_E2E_APPS: ${{ needs.discover.outputs.apps }}
      BETA_E2E_GREP: ${{ inputs.grep || '' }}
      BETA_E2E_AUTHED: "0"
      BETA_E2E_REPORT_SLOT: fleet
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
      - uses: ./.github/actions/beta-e2e-setup
      - name: Cross-host checks
        run: |
          if [ -n "$BETA_E2E_GREP" ]; then
            pnpm e2e:beta --project=fleet --grep "$BETA_E2E_GREP" --pass-with-no-tests
          else
            pnpm e2e:beta --project=fleet
          fi
      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: ${{ always() }}
        with:
          name: beta-e2e-fleet-${{ github.run_id }}
          path: e2e/beta/playwright-report/
          if-no-files-found: warn
          retention-days: 7

  authed:
    name: Authenticated ${{ matrix.cluster }}
    needs: [discover, advisory]
    if: ${{ always() && !cancelled() && needs.discover.result == 'success' && inputs.lane != 'public' && inputs.lane != 'signup' }}
    runs-on: ubuntu-latest
    timeout-minutes: 45
    strategy:
      fail-fast: false
      # These clusters share the e2e identity and production-backed databases.
      # Keep them serialized while giving each failure its own gating signal.
      max-parallel: 1
      # Keep the authenticated projects declarative so guard:beta-e2e-suite
      # can verify that each cluster remains independently visible. The steps
      # below skip unrelated clusters for a Design-only selection.
      matrix:
        include:
          - cluster: registry
            project: registry
          - cluster: chat
            project: chat
          - cluster: journeys
            project: journeys
          - cluster: design
            project: design
    env:
      BETA_E2E_APPS: ${{ matrix.cluster == 'design' && 'design' || needs.discover.outputs.apps }}
      BETA_E2E_GREP: ${{ inputs.grep || '' }}
      BETA_E2E_AUTHED: "1"
      BETA_E2E_CLUSTER: ${{ matrix.cluster }}
      BETA_E2E_REPORT_SLOT: authed-${{ matrix.cluster }}
      BETA_E2E_EMAIL: ${{ secrets.BETA_E2E_EMAIL }}
      BETA_E2E_SESSION_TOKENS: ${{ secrets.BETA_E2E_SESSION_TOKENS }}
      BETA_E2E_SESSION_TOKEN_CRM: ${{ secrets.BETA_E2E_SESSION_TOKEN_CRM }}
      BETA_E2E_SESSION_TOKEN_CHAT: ${{ secrets.BETA_E2E_SESSION_TOKEN_CHAT }}
      BETA_E2E_OPENAI_API_KEY: ${{ secrets.BETA_E2E_OPENAI_API_KEY }}
      # The repository-wide key is available only for an explicit shared-key
      # dispatch. Dedicated runs must not expose it to setup or test code.
      BETA_E2E_SHARED_OPENAI_API_KEY: ${{ inputs.key_source == 'shared' && secrets.OPENAI_API_KEY || '' }}
      BETA_E2E_ALLOW_SHARED_KEY: ${{ inputs.key_source == 'shared' }}
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
        if: ${{ matrix.cluster == 'design' && contains(needs.discover.outputs.apps, 'design') || matrix.cluster != 'design' && needs.discover.outputs.apps != 'design' }}
      - uses: ./.github/actions/beta-e2e-setup
        if: ${{ matrix.cluster == 'design' && contains(needs.discover.outputs.apps, 'design') || matrix.cluster != 'design' && needs.discover.outputs.apps != 'design' }}
      - name: Authenticated ${{ matrix.cluster }}
        if: ${{ matrix.cluster == 'design' && contains(needs.discover.outputs.apps, 'design') || matrix.cluster != 'design' && needs.discover.outputs.apps != 'design' }}
        # Global setup fails loudly if a credential is missing rather than
        # degrading to an anonymous run that would pass without testing
        # anything. The clusters are separate so a provider failure cannot
        # hide registry or journey regressions.
        # A title filter can select only public/fleet tests, though. Ask
        # Playwright for the authenticated selection first with auth disabled;
        # --list does not run globalSetup, so an empty selection skips the
        # credentialed setup without turning a requested auth run into a pass.
        run: |
          if [ -n "$BETA_E2E_GREP" ]; then
            selection_file="$(mktemp)"
            trap 'rm -f "$selection_file"' EXIT
            set +e
            BETA_E2E_AUTHED=0 pnpm e2e:beta \
              --project=${{ matrix.project }} \
              --grep "$BETA_E2E_GREP" --list >"$selection_file" 2>&1
            selection_status="$?"
            set -e
            cat "$selection_file"
            if [ "$selection_status" -ne 0 ]; then
              if grep -q "Error: No tests found" "$selection_file" && grep -q "Total: 0 tests in 0 files" "$selection_file"; then
                echo "No authenticated tests match BETA_E2E_GREP; skipping authenticated setup."
                exit 0
              fi
              echo "Authenticated test selection failed before execution." >&2
              exit "$selection_status"
            fi
            pnpm e2e:beta --project=${{ matrix.project }} --grep "$BETA_E2E_GREP" --pass-with-no-tests
          else
            pnpm e2e:beta --project=${{ matrix.project }}
          fi
      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: ${{ always() && (matrix.cluster == 'design' && contains(needs.discover.outputs.apps, 'design') || matrix.cluster != 'design' && needs.discover.outputs.apps != 'design') }}
        # The authenticated lanes record no trace and no video (see the
        # playwright config): a trace captures real request headers and would
        # publish the e2e account's live session cookie to anyone with repo
        # read access. e2e/beta/.auth is deliberately not in this list.
        with:
          name: beta-e2e-authed-${{ matrix.cluster }}-${{ github.run_id }}
          path: |
            e2e/beta/playwright-report/
            e2e/beta/test-results/
          if-no-files-found: warn
          retention-days: 7

  advisory:
    name: Advisory findings (non-gating)
    if: ${{ always() && !cancelled() && needs.discover.result == 'success' && inputs.lane != 'authed' && inputs.lane != 'signup' }}
    needs: [discover, fleet]
    runs-on: ubuntu-latest
    timeout-minutes: 30
    env:
      BETA_E2E_APPS: ${{ needs.discover.outputs.apps }}
      BETA_E2E_GREP: ${{ inputs.grep || '' }}
      BETA_E2E_AUTHED: "0"
      BETA_E2E_REPORT_SLOT: advisory
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
      - uses: ./.github/actions/beta-e2e-setup
      - name: Advisory findings
        continue-on-error: true
        # Always runs: it costs no credentials and no tokens, and findings that
        # stop being reported stop being fixed. Step-level continue-on-error
        # keeps the reusable workflow green while preserving the failed step
        # and uploaded report for inspection.
        run: |
          if [ -n "$BETA_E2E_GREP" ]; then
            pnpm e2e:beta --project=advisory --grep "$BETA_E2E_GREP" --pass-with-no-tests
          else
            pnpm e2e:beta --project=advisory
          fi
      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: ${{ always() }}
        with:
          name: beta-e2e-advisory-${{ github.run_id }}
          path: e2e/beta/playwright-report/
          if-no-files-found: warn
          retention-days: 7

  signup:
    name: Signup canary
    if: ${{ inputs.lane == 'signup' }}
    outputs:
      outcome: ${{ steps.classify.outputs.outcome }}
    runs-on: ubuntu-latest
    timeout-minutes: 180
    env:
      SIGNUP_E2E_APPS: ${{ inputs.signup_apps || 'all' }}
      SIGNUP_E2E_ENVIRONMENTS: ${{ inputs.signup_environments || 'beta' }}
      MAILOSAUR_API_KEY: ${{ secrets.MAILOSAUR_API_KEY }}
      MAILOSAUR_SERVER_ID: ${{ secrets.MAILOSAUR_SERVER_ID }}
      PAGERDUTY_ROUTING_KEY: ${{ secrets.PAGERDUTY_ROUTING_KEY }}
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
      - uses: ./.github/actions/beta-e2e-setup
      - name: Test signup email provider contracts
        run: pnpm exec tsx --test e2e/signup/lib/mailosaur.spec.ts
      - name: Typecheck signup suite
        run: pnpm typecheck:e2e:signup
      - name: Run full signup flow
        id: signup
        run: pnpm e2e:signup
      - name: Classify signup result
        id: classify
        if: ${{ always() }}
        run: |
          set -euo pipefail
          if [ "${{ steps.signup.outcome }}" != "success" ]; then
            echo "outcome=failure" >> "$GITHUB_OUTPUT"
          elif find e2e/signup/test-results -type f -name 'mailosaur-inconclusive.txt' -print -quit | grep -q .; then
            echo "outcome=inconclusive" >> "$GITHUB_OUTPUT"
            echo "INCONCLUSIVE: Mailosaur rate-limited the verification inbox; no product result was recorded." >> "$GITHUB_STEP_SUMMARY"
          else
            echo "outcome=success" >> "$GITHUB_OUTPUT"
          fi
      - name: Page signup health on-call
        if: ${{ failure() && steps.signup.outcome == 'failure' && steps.classify.outputs.outcome == 'failure' }}
        run: |
          set -euo pipefail
          if [ -z "${PAGERDUTY_ROUTING_KEY:-}" ]; then
            echo "::warning::PAGERDUTY_ROUTING_KEY is not configured; the GitHub issue fallback is active but no page was sent."
            exit 0
          fi
          jq -n \
            --arg routing_key "$PAGERDUTY_ROUTING_KEY" \
            --arg dedup_key "agent-native-signup-e2e" \
            --arg source "${{ github.repository }}" \
            --arg run_url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
            '{routing_key: $routing_key, event_action: "trigger", dedup_key: $dedup_key, payload: {summary: "Agent-Native signup E2E is failing", source: $source, severity: "critical", custom_details: {run_url: $run_url, apps: env.SIGNUP_E2E_APPS, environments: env.SIGNUP_E2E_ENVIRONMENTS}}}' \
            | curl --fail-with-body --connect-timeout 10 --max-time 20 --silent --show-error \
                --header 'Content-Type: application/json' \
                --data-binary @- \
                https://events.pagerduty.com/v2/enqueue
      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: ${{ always() }}
        with:
          name: signup-e2e-${{ github.run_id }}
          path: e2e/signup/test-results/
          if-no-files-found: warn
          retention-days: 14