Nightly E2E Tests #281
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly E2E Tests | |
| on: | |
| schedule: | |
| - cron: '0 3 * * *' # Run nightly at 03:00 UTC | |
| workflow_dispatch: {} | |
| concurrency: | |
| group: ${{ github.workflow }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: write | |
| issues: write | |
| pull-requests: write | |
| actions: write | |
| jobs: | |
| clean: | |
| runs-on: ubuntu-latest | |
| name: "Clean test resources" | |
| outputs: | |
| testsuites: ${{ steps.plan.outputs.testsuites }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: GitHub CLI - show auth status | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| gh auth status | |
| - name: Run cleanup script | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| chmod +x ./clean.sh | |
| ./clean.sh | |
| - name: Upload cleanup log artifact | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: cleanup-log | |
| path: cleanup-*.log | |
| - name: Plan | |
| id: plan | |
| run: | | |
| # trigered test suites do not work since issues/comments created from actions | |
| # do not trigger actions | |
| echo 'testsuites=["test-claude-* --workflow-dispatch-only", "test-codex-* --workflow-dispatch-only", "test-copilot-* --workflow-dispatch-only", "test-custom-* --workflow-dispatch-only"]' >> "$GITHUB_OUTPUT" | |
| e2e: | |
| runs-on: ubuntu-latest | |
| needs: clean | |
| strategy: | |
| matrix: | |
| testsuite: ${{ fromJSON(needs.clean.outputs.testsuites) }} | |
| fail-fast: false | |
| name: "E2E Tests: ${{ matrix.testsuite }}" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: 'recursive' | |
| fetch-depth: 0 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v5 | |
| - name: Set up Go | |
| uses: actions/setup-go@v5 | |
| - name: GitHub CLI - show auth status | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| gh auth status | |
| - name: List files for debugging | |
| run: | | |
| ls -la | |
| pwd | |
| - name: Run e2e tests and publish output to job summary | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| timeout-minutes: 60 | |
| run: | | |
| set -o pipefail | |
| chmod +x ./e2e.sh | |
| # Run the e2e script, tee output to both stdout and log, capture exit code | |
| set +e | |
| ./e2e.sh ${{ matrix.testsuite }} 2>&1 | tee e2e-output.log | |
| rc=${PIPESTATUS[0]} | |
| set -e | |
| echo "\n## E2E test run output (${{ matrix.testsuite }})" >> "$GITHUB_STEP_SUMMARY" | |
| echo '``````bash' >> "$GITHUB_STEP_SUMMARY" | |
| sed -n '1,50000p' e2e-output.log >> "$GITHUB_STEP_SUMMARY" || true | |
| echo '``````' >> "$GITHUB_STEP_SUMMARY" | |
| echo "Detailed log attached as artifact: e2e-output-${{ matrix.testsuite }}.log" | |
| # Fail the job if the script exited non-zero so GH shows failure status | |
| if [[ $rc -ne 0 ]]; then | |
| echo "E2E script failed with exit code $rc" | |
| exit $rc | |
| fi |