Explain inter-rater reliability before the first reviewer pair completes #640
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI - Deploy & Test | |
| # Directory filters only: packages/web/public/ holds files consumed by | |
| # Workers Static Assets at deploy time, so never filter by extension. | |
| on: | |
| push: | |
| branches: [main] | |
| paths-ignore: ['infra/**', 'packages/docs/**', '.claude/**', '*.md'] | |
| pull_request: | |
| branches: [main] | |
| paths-ignore: ['infra/**', 'packages/docs/**', '.claude/**', '*.md'] | |
| # Cancel in-progress runs on the same branch | |
| concurrency: | |
| group: ci-${{ github.ref }} | |
| cancel-in-progress: true | |
| # The Cloudflare vitest pool starts one workerd per test file up to this limit. | |
| # Uncapped, the instances starve each other and the heavier workers tests blow | |
| # their 10s budget. | |
| env: | |
| VITEST_MAX_WORKERS: 4 | |
| # --------------------------------------------------------------- | |
| # Prerequisites (one-time setup): | |
| # | |
| # 1. Cloudflare resources: | |
| # wrangler d1 create corates-db-staging | |
| # wrangler r2 bucket create corates-pdfs-staging | |
| # wrangler queues create corates-emails-staging | |
| # wrangler queues create corates-emails-staging-dlq | |
| # | |
| # 2. Update database_id in the packages/web/wrangler.jsonc staging env | |
| # with the ID from step 1. | |
| # | |
| # 3. DNS: Add CNAME record for staging.corates.org | |
| # pointing to your workers custom domain. | |
| # | |
| # 4. Wrangler secrets for staging (run from packages/web): | |
| # wrangler secret put AUTH_SECRET --env staging | |
| # wrangler secret put STRIPE_SECRET_KEY --env staging | |
| # wrangler secret put STRIPE_WEBHOOK_SECRET_AUTH --env staging | |
| # wrangler secret put GOOGLE_CLIENT_SECRET --env staging | |
| # wrangler secret put ORCID_CLIENT_SECRET --env staging | |
| # wrangler secret put POSTMARK_SERVER_TOKEN --env staging | |
| # wrangler secret put ADMIN_EMAIL --env staging | |
| # wrangler secret put GOOGLE_CLIENT_ID --env staging | |
| # wrangler secret put ORCID_CLIENT_ID --env staging | |
| # wrangler secret put EMAIL_FROM --env staging | |
| # | |
| # 5. GitHub repository secrets: | |
| # CLOUDFLARE_API_TOKEN - API token with Workers + D1 + R2 permissions | |
| # | |
| # 6. GitHub repository variables (Settings > Variables): | |
| # VITE_GOOGLE_PICKER_API_KEY | |
| # VITE_GOOGLE_PICKER_APP_ID | |
| # VITE_SENTRY_DSN (production client-side Sentry DSN) | |
| # --------------------------------------------------------------- | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup workspace | |
| uses: ./.github/actions/setup-workspace | |
| # workers' test helpers import this gitignored generated module | |
| - name: Generate test migration SQL | |
| run: pnpm --filter workers db:generate:test | |
| - name: Lint | |
| run: pnpm lint | |
| typecheck: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup workspace | |
| uses: ./.github/actions/setup-workspace | |
| - name: Generate test migration SQL | |
| run: pnpm --filter workers db:generate:test | |
| - name: Typecheck | |
| run: pnpm typecheck | |
| test-unit: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup workspace | |
| uses: ./.github/actions/setup-workspace | |
| - name: Shared and workers tests | |
| run: pnpm --filter @corates/shared --filter @corates/workers test | |
| - name: Web unit tests | |
| run: pnpm --filter web test:unit | |
| test-server: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup workspace | |
| uses: ./.github/actions/setup-workspace | |
| - name: Web server tests | |
| run: pnpm --filter web test:server | |
| # On push to main, decide whether to skip e2e: | |
| # - merged PR has the `skip-e2e` label, or | |
| # - commit message contains `[skip e2e]` (direct pushes / squash title) | |
| resolve-skip-e2e: | |
| if: github.event_name == 'push' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: read | |
| outputs: | |
| skip_e2e: ${{ steps.check.outputs.skip_e2e }} | |
| steps: | |
| - name: Check skip-e2e triggers | |
| id: check | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const message = context.payload.head_commit?.message ?? ''; | |
| const skipFromCommit = message.toLowerCase().includes('[skip e2e]'); | |
| // This job gates the production deploy, so an API blip on the | |
| // label lookup must not fail it. A failure falls back to running | |
| // e2e, which is the safe direction. | |
| let mergedPr; | |
| let skipFromLabel = false; | |
| try { | |
| const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| commit_sha: context.sha, | |
| }); | |
| mergedPr = prs.find(pr => pr.merged_at) ?? prs[0]; | |
| const labels = mergedPr?.labels?.map(label => label.name) ?? []; | |
| skipFromLabel = labels.includes('skip-e2e'); | |
| } catch (error) { | |
| core.warning(`Could not read PR labels (${error.message}); running e2e`); | |
| } | |
| const skip = skipFromCommit || skipFromLabel; | |
| core.setOutput('skip_e2e', skip ? 'true' : 'false'); | |
| if (skipFromCommit) { | |
| core.notice('E2E skipped via [skip e2e] in commit message'); | |
| } else if (skipFromLabel) { | |
| core.notice(`E2E skipped via skip-e2e label on PR #${mergedPr.number}`); | |
| } | |
| # Builds on PRs as a check; on push also deploys the build to staging. Staging | |
| # deliberately does not wait on the check jobs; deploy-production does. | |
| build-web: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup workspace | |
| uses: ./.github/actions/setup-workspace | |
| - name: Build web for staging | |
| env: | |
| CLOUDFLARE_ENV: staging | |
| VITE_API_URL: 'https://staging.corates.org' | |
| VITE_PUBLIC_APP_URL: 'https://staging.corates.org' | |
| VITE_BASEPATH: '/' | |
| # The e2e job runs against this deployment and seeds through the | |
| # in-page dev seams (window.__devSeed / __devLegacy), which only | |
| # exist when the dev panel is compiled in. Staging is a test | |
| # environment (DEV_MODE is already true on its worker). | |
| VITE_DEV_PANEL: 'true' | |
| VITE_GOOGLE_PICKER_API_KEY: ${{ vars.VITE_GOOGLE_PICKER_API_KEY }} | |
| VITE_GOOGLE_PICKER_APP_ID: ${{ vars.VITE_GOOGLE_PICKER_APP_ID }} | |
| run: pnpm --filter web build | |
| # build.target says what to emit but never verifies it | |
| - name: Check client bundle against the browser floor | |
| run: pnpm --filter web check:client-syntax | |
| - name: Migrate staging database | |
| if: github.event_name == 'push' | |
| env: | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| CLOUDFLARE_ACCOUNT_ID: 2ab59c3eb44887bfa2bac00afd32290a | |
| run: pnpm --filter web db:migrate:staging | |
| - name: Deploy web to staging | |
| if: github.event_name == 'push' | |
| env: | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| working-directory: packages/web | |
| run: pnpm exec wrangler deploy | |
| # Confirm the deploy is live before handing off to the e2e job. Runs here | |
| # (not in the e2e container) so we can rely on curl being present. | |
| - name: Wait for staging to be ready | |
| if: github.event_name == 'push' | |
| run: | | |
| for i in $(seq 1 30); do | |
| if curl -sf https://staging.corates.org/api/test/health > /dev/null 2>&1; then | |
| echo "Staging is ready" | |
| exit 0 | |
| fi | |
| echo "Waiting for staging... (attempt $i/30)" | |
| sleep 5 | |
| done | |
| echo "Staging did not become ready in time" | |
| exit 1 | |
| # E2E against staging. Chromium and its system libraries are installed per | |
| # job from the workspace's @playwright/test, so they can never drift apart. | |
| e2e: | |
| needs: [build-web, resolve-skip-e2e] | |
| if: | | |
| needs.build-web.result == 'success' && | |
| needs.resolve-skip-e2e.outputs.skip_e2e != 'true' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup workspace | |
| uses: ./.github/actions/setup-workspace | |
| with: | |
| build-workspace: 'false' | |
| - name: Install Chromium | |
| run: pnpm --filter web exec playwright install --with-deps chromium | |
| # Full pass, a healing re-run of a few failures, then the job summary. | |
| - name: Run e2e tests | |
| env: | |
| PLAYWRIGHT_BASE_URL: 'https://staging.corates.org' | |
| working-directory: packages/web | |
| run: bash scripts/e2e-ci.sh | |
| # always(), not failure(): a test that fails then passes on retry leaves a | |
| # trace behind but keeps the job green, and that is exactly the run worth | |
| # diagnosing. A clean run still uploads the handful of debug screenshots | |
| # the rob2/robins specs write into test-results/ unconditionally. The | |
| # first-pass directory only exists when the healing re-run happened and | |
| # holds the traces of the failures it healed. | |
| - name: Upload test artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: playwright-report | |
| path: | | |
| packages/web/test-results/ | |
| packages/web/test-results-first-pass/ | |
| if-no-files-found: ignore | |
| retention-days: 7 | |
| deploy-production: | |
| needs: [lint, typecheck, test-unit, test-server, build-web, resolve-skip-e2e, e2e] | |
| if: | | |
| always() && !cancelled() && | |
| github.event_name == 'push' && | |
| needs.lint.result == 'success' && | |
| needs.typecheck.result == 'success' && | |
| needs.test-unit.result == 'success' && | |
| needs.test-server.result == 'success' && | |
| needs.build-web.result == 'success' && | |
| (needs.e2e.result == 'success' || needs.resolve-skip-e2e.outputs.skip_e2e == 'true') | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup workspace | |
| uses: ./.github/actions/setup-workspace | |
| - name: Build web for production | |
| env: | |
| CLOUDFLARE_ENV: production | |
| VITE_API_URL: 'https://corates.org' | |
| VITE_PUBLIC_APP_URL: 'https://corates.org' | |
| VITE_BASEPATH: '/' | |
| VITE_DEV_PANEL: 'false' | |
| VITE_GOOGLE_PICKER_API_KEY: ${{ vars.VITE_GOOGLE_PICKER_API_KEY }} | |
| VITE_GOOGLE_PICKER_APP_ID: ${{ vars.VITE_GOOGLE_PICKER_APP_ID }} | |
| VITE_SENTRY_DSN: ${{ vars.VITE_SENTRY_DSN }} | |
| SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} | |
| run: pnpm --filter web build | |
| - name: Migrate production database | |
| env: | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| CLOUDFLARE_ACCOUNT_ID: 2ab59c3eb44887bfa2bac00afd32290a | |
| run: pnpm --filter web db:migrate:prod | |
| - name: Deploy web to production | |
| env: | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| working-directory: packages/web | |
| run: pnpm exec wrangler deploy |