test.yml

# Module 6 — Test (v2: second isolated Claude Code process, tool-restricted tester)
#
# Originally PRD §4 specified Codex as out-of-distribution tester. Amended to a
# second isolated Claude Code process (see .github/actions/test/action.yml header
# and docs/architecture.md §"Module 6 engine amendment"). Triggers when `verified`
# label is applied to an issue (Module 5 self-verify passed). Runs the test
# composite action, posts a test report audit comment on the linked PR, and
# transitions labels:
#   verified → testing → tested (passed) or test:failed (failed)
#
# SECURITY POSTURE (PRD §7):
#   - permissions: contents:read + issues:write + pull-requests:write
#     (S1: NO contents:write — test is read + test execution only)
#   - S5: claude-code-action with allow Read/Grep/Glob/Bash, deny Write/Edit
#     (NO --dangerously-skip-permissions)
#   - S4: no secrets printed in workflow; LLM_API_KEY never printed
#   - S3: never writes to main
#   - PR #16 lesson: label writes use CLAUDE_DEV_PAT (so downstream workflows fire)
#   - On failure: test:failed label + failure comment, stage:failed fallback
#
# Triggers:
#   - issues.labeled with label == verified

name: test

on:
  issues:
    types: [labeled]

permissions:
  contents: read
  issues: write
  pull-requests: write

concurrency:
  group: test-issue-${{ github.event.issue.number }}
  cancel-in-progress: false

jobs:
  test:
    if: ${{ github.event.label.name == 'verified' }}
    runs-on: ubuntu-latest
    permissions:
      contents: read
      issues: write
      pull-requests: write
    env:
      # PR #138 root cause: claude-code-action@v1 reads env.ANTHROPIC_BASE_URL
      # at the JOB level. Step-level env on composite actions does NOT
      # propagate to the SDK call. Without this, SDK defaults to
      # api.anthropic.com and GLM passthrough key gets 401.
      ANTHROPIC_BASE_URL: ${{ vars.ANTHROPIC_BASE_URL }}
    timeout-minutes: ${{ fromJSON(vars.TEST_TIME_BUDGET_MIN || '30') }}
    outputs:
      test-status: ${{ steps.test.outputs.test-status }}
      test-report: ${{ steps.test.outputs.test-report }}
      commit-sha: ${{ steps.test.outputs.commit-sha }}
    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false

      - name: Discover PR number from issue audit comments
        id: pr
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          NUMBER: ${{ github.event.issue.number }}
        run: |
          set -eu
          # Search issue comments for "PR:" link posted by develop workflow.
          # develop posts "🚀 Development branch pushed... PR: <url>"
          pr_url=$(gh issue view "$NUMBER" --repo "$GITHUB_REPOSITORY" --json comments \
            --jq '.comments | map(.body) | join("\n")' \
            | grep -oE 'https://github\.com/[^/]+/[^/]+/pull/[0-9]+' \
            | head -n 1 || echo "")

          if [ -z "$pr_url" ]; then
            echo "::error::Could not find PR URL in issue #$NUMBER comments. Has develop (Module 4) run?"
            exit 1
          fi

          pr_number=$(printf '%s' "$pr_url" | grep -oE '[0-9]+$' || echo "")
          if [ -z "$pr_number" ]; then
            echo "::error::Could not extract PR number from URL: $pr_url"
            exit 1
          fi

          # Discover head branch from PR
          head_branch=$(gh pr view "$pr_number" --repo "$GITHUB_REPOSITORY" --json headRefName --jq '.headRefName' 2>/dev/null || echo "")
          base_branch=$(gh pr view "$pr_number" --repo "$GITHUB_REPOSITORY" --json baseRefName --jq '.baseRefName' 2>/dev/null || echo "dev")

          echo "pr-url=$pr_url" >> "$GITHUB_OUTPUT"
          echo "pr-number=$pr_number" >> "$GITHUB_OUTPUT"
          echo "head-branch=$head_branch" >> "$GITHUB_OUTPUT"
          echo "base-branch=$base_branch" >> "$GITHUB_OUTPUT"

          echo "Discovered PR: $pr_url (head=$head_branch, base=$base_branch)"

      - name: Apply testing label on issue (Module 6 active)
        env:
          GH_TOKEN: ${{ secrets.CLAUDE_DEV_PAT }}
          NUMBER: ${{ github.event.issue.number }}
        run: |
          set -eu
          gh issue edit "$NUMBER" --repo "$GITHUB_REPOSITORY" --add-label "testing" || true

      - name: "Verify LLM_API_KEY secret exists (S4: never print value)"
        shell: bash
        run: |
          if [ -z "${{ secrets.LLM_API_KEY }}" ]; then
            echo "::error::LLM_API_KEY secret is not set or is empty. Module 6 v2 uses this key for GLM passthrough via ANTHROPIC_BASE_URL."
            exit 1
          fi
          echo "::notice::LLM_API_KEY secret is configured (value not printed per S4)."

      - name: Run test composite (Claude engine, v2)
        id: test
        uses: ./.github/actions/test
        with:
          issue-number: ${{ github.event.issue.number }}
          pr-url: ${{ steps.pr.outputs.pr-url }}
          pr-number: ${{ steps.pr.outputs.pr-number }}
          head-branch: ${{ steps.pr.outputs.head-branch }}
          base-branch: ${{ steps.pr.outputs.base-branch }}
          repo-token: ${{ secrets.GITHUB_TOKEN }}
          api-key: ${{ secrets.LLM_API_KEY }}
          api-key-backup: ${{ secrets.LLM_API_KEY_BACKUP }}
          anthropic-base-url: ${{ vars.ANTHROPIC_BASE_URL || '' }}
          model: ${{ vars.TEST_MODEL || '' }}
          max-turns: ${{ vars.TEST_MAX_TURNS || '10' }}
          context-budget-paths: ${{ vars.TEST_CONTEXT_PATHS || '' }}

      - name: Post test report audit comment on PR
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          PR_NUMBER: ${{ steps.pr.outputs.pr-number }}
          STATUS: ${{ steps.test.outputs.test-status }}
          REPORT: ${{ steps.test.outputs.test-report }}
          SHA: ${{ steps.test.outputs.commit-sha }}
          ISSUE_NUMBER: ${{ github.event.issue.number }}
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        run: |
          set -eu
          icon="✅"
          if [ "$STATUS" = "failed" ]; then icon="❌"; fi

          gh pr comment "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" --body "$(cat <<AUDIT_EOF
          $icon **Test: \`$STATUS\`**

          | Field | Value |
          |---|---|
          | Issue | #$ISSUE_NUMBER |
          | Commit | \`${SHA:0:12}\` |
          | Workflow run | $RUN_URL |

          ## Test report

          $REPORT

          ---
          Triggered by test (Module 6, v2) | Engine: Claude (tool-restricted tester, PRD §4 amendment)
          AUDIT_EOF
          )"

      - name: Post audit comment on issue
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          NUMBER: ${{ github.event.issue.number }}
          STATUS: ${{ steps.test.outputs.test-status }}
          REPORT: ${{ steps.test.outputs.test-report }}
          PR_URL: ${{ steps.pr.outputs.pr-url }}
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        run: |
          set -eu
          icon="✅"
          if [ "$STATUS" = "failed" ]; then icon="❌"; fi

          gh issue comment "$NUMBER" --repo "$GITHUB_REPOSITORY" --body "$(cat <<AUDIT_EOF
          $icon **Test: \`$STATUS\`**

          | Field | Value |
          |---|---|
          | PR | $PR_URL |
          | Workflow run | $RUN_URL |

          $REPORT
          AUDIT_EOF
          )"

      - name: Transition labels (passed → tested)
        if: steps.test.outputs.test-status == 'passed'
        env:
          GH_TOKEN: ${{ secrets.CLAUDE_DEV_PAT }}
          NUMBER: ${{ github.event.issue.number }}
        run: |
          set -eu
          # PR #16 lesson: use CLAUDE_DEV_PAT so downstream workflows fire.
          gh issue edit "$NUMBER" --repo "$GITHUB_REPOSITORY" --remove-label "testing" || true
          gh issue edit "$NUMBER" --repo "$GITHUB_REPOSITORY" --add-label "tested"

      - name: Retry or escalate (test:failed → retry N/3 or stage:failed)
        id: retry
        if: steps.test.outputs.test-status == 'failed'
        env:
          GH_TOKEN: ${{ secrets.CLAUDE_DEV_PAT }}
          NUMBER: ${{ github.event.issue.number }}
          REPORT: ${{ steps.test.outputs.test-report }}
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
          TEST_RETRY_MAX: ${{ vars.TEST_RETRY_MAX || '3' }}
        run: |
          set -eu
          # Remove testing label (applied at start of this job).
          gh issue edit "$NUMBER" --repo "$GITHUB_REPOSITORY" --remove-label "testing" || true

          # Determine current retry count from test:retry-N labels.
          labels=$(gh issue view "$NUMBER" --repo "$GITHUB_REPOSITORY" --json labels --jq '.labels | map(.name) | join(",")')
          retry_count=0
          if printf '%s' "$labels" | grep -q "test:retry-3"; then retry_count=3; fi
          if [ "$retry_count" -eq 0 ] && printf '%s' "$labels" | grep -q "test:retry-2"; then retry_count=2; fi
          if [ "$retry_count" -eq 0 ] && printf '%s' "$labels" | grep -q "test:retry-1"; then retry_count=1; fi

          max_retries="${TEST_RETRY_MAX:-3}"
          echo "retry_count=$retry_count max=$max_retries"

          if [ "$retry_count" -ge "$max_retries" ]; then
            # Max retries exhausted — escalate to maintainer.
            echo "decision=escalate" >> "$GITHUB_OUTPUT"
            gh issue edit "$NUMBER" --repo "$GITHUB_REPOSITORY" --add-label "stage:failed"
            gh issue comment "$NUMBER" --repo "$GITHUB_REPOSITORY" --body "$(cat <<EOF
          ❌ **Test retries exhausted (${retry_count}/${max_retries})**

          Module 6 failed $retry_count time(s). The pipeline cannot automatically recover.
          A maintainer must triage manually.

          | Field | Value |
          |---|---|
          | Last workflow run | $RUN_URL |
          | Retry labels | $labels |

          EOF
          )"
          else
            # Retry: remove test:failed (if present), clean up intermediate labels,
            # add retry counter.
            # M1+M2 restore (issue #118): the test:retry-N label is now the
            # dispatch signal — code-generate.yml triggers on issues.labeled
            # with test:retry-* and re-runs Module 4 with the prior test report
            # fetched from the audit comment below.
            next=$((retry_count + 1))
            echo "decision=retry" >> "$GITHUB_OUTPUT"
            echo "retry_label=test:retry-${next}" >> "$GITHUB_OUTPUT"

            gh issue edit "$NUMBER" --repo "$GITHUB_REPOSITORY" --remove-label "test:failed" 2>/dev/null || true
            gh issue edit "$NUMBER" --repo "$GITHUB_REPOSITORY" --remove-label "verified" 2>/dev/null || true
            # Clean up any prior test:retry-N labels so only the latest counter remains
            for old_n in 1 2 3 4 5; do
              [ "$old_n" = "$next" ] && continue
              gh issue edit "$NUMBER" --repo "$GITHUB_REPOSITORY" --remove-label "test:retry-$old_n" 2>/dev/null || true
            done
            gh issue edit "$NUMBER" --repo "$GITHUB_REPOSITORY" --add-label "test:retry-${next}"

            # Post audit comment with retry count, failure summary, and run URL.
            # code-generate.yml parses this comment body to reconstruct the
            # prior-test-report for Claude on the retry pass.
            failure_summary="$(printf '%s' "$REPORT" | head -n 6)"
            gh issue comment "$NUMBER" --repo "$GITHUB_REPOSITORY" --body "$(cat <<EOF
          🔄 **Auto-retry ${next}/${max_retries}**

          | Field | Value |
          |---|---|
          | Retry | ${next} of ${max_retries} |
          | Prior workflow run | $RUN_URL |
          | Retry label | test:retry-${next} |

          ## Failure summary (prior Module 6 run)

          ${failure_summary}

          ---
          code-generate.yml will auto-trigger on the \`test:retry-${next}\` label,
          fetch this comment as the prior-test-report, and re-run Module 4 to
          fix the failure.
          EOF
          )"
          fi

      - name: Comment failure details on PR (test:failed)
        if: steps.test.outputs.test-status == 'failed'
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          PR_NUMBER: ${{ steps.pr.outputs.pr-number }}
          REPORT: ${{ steps.test.outputs.test-report }}
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        run: |
          set -eu
          gh pr comment "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" --body "$(cat <<PR_BODY
          ❌ **Test failed**

          The test suite did not pass. See the test report below.

          ## Failures

          $REPORT

          ---
          [Workflow run]($RUN_URL) | Triggered by test (Module 6)
          PR_BODY
          )"

  on-failure:
    needs: test
    if: ${{ always() && needs.test.result == 'failure' }}
    runs-on: ubuntu-latest
    permissions:
      issues: write
    steps:
      - name: Mark stage:failed + clean intermediate labels
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          NUMBER: ${{ github.event.issue.number }}
        run: |
          if [ -z "$NUMBER" ]; then
            echo "::warning::No issue number on this event."
            exit 0
          fi
          gh issue edit "$NUMBER" --repo "$GITHUB_REPOSITORY" --add-label "stage:failed" || true
          # M3 fix: test job applies `testing` before the composite runs (line 93).
          # On composite failure that label is never removed, leaving the issue in
          # a contradictory state (testing + stage:failed + verified). Mirror the
          # cleanup pattern verify.yml:306 uses for `verifying`.
          gh issue edit "$NUMBER" --repo "$GITHUB_REPOSITORY" --remove-label "testing" 2>/dev/null || true
          gh issue comment "$NUMBER" --repo "$GITHUB_REPOSITORY" \
            --body "⚠️ Test job failed (infrastructure/engine error — not a test failure). See [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details. A maintainer will triage manually." || true