orchestrai(!orc): show the ReportPortal link in the job summary #1801
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright Advanced Micro Devices, Inc. | |
| # | |
| # SPDX-License-Identifier: MIT | |
| name: Test Playbooks | |
| on: | |
| schedule: | |
| - cron: '0 8 * * *' # 01:00 PDT (UTC-7); 02:00 PST (UTC-8) | |
| workflow_dispatch: | |
| inputs: | |
| playbook_id: | |
| description: 'Specific playbook ID to test (leave empty to test all playbooks)' | |
| required: false | |
| type: string | |
| pull_request: | |
| permissions: | |
| contents: read | |
| # Required so the failure-reporting step can open issues on behalf of the | |
| # workflow. Note we still gate the issue creation step itself on | |
| # github.ref == 'refs/heads/main' so PR runs never create issues. | |
| issues: write | |
| jobs: | |
| detect-changes: | |
| name: Detect Changed Playbooks | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.build-matrix.outputs.matrix }} | |
| has_entries: ${{ steps.build-matrix.outputs.has_entries }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Detect changed playbooks | |
| id: detect | |
| run: | | |
| if [ -n "${{ github.event.inputs.playbook_id }}" ]; then | |
| echo "playbooks=[\"${{ github.event.inputs.playbook_id }}\"]" >> $GITHUB_OUTPUT | |
| elif [ "${{ github.event_name }}" = "pull_request" ]; then | |
| CHANGED_FILES=$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} -- 'playbooks/core/*' 'playbooks/supplemental/*' 2>/dev/null || echo "") | |
| PLAYBOOK_IDS=$(echo "$CHANGED_FILES" | grep -E '^playbooks/(core|supplemental)/[^/]+/' | sed 's|playbooks/[^/]*/\([^/]*\)/.*|\1|' | sort -u | jq -R -s -c 'split("\n") | map(select(length > 0))') | |
| if [ "$PLAYBOOK_IDS" = "[]" ] || [ -z "$PLAYBOOK_IDS" ]; then | |
| echo "playbooks=[]" >> $GITHUB_OUTPUT | |
| else | |
| echo "playbooks=$PLAYBOOK_IDS" >> $GITHUB_OUTPUT | |
| fi | |
| else | |
| ALL_PLAYBOOKS=$(find playbooks/core playbooks/supplemental -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sed 's|playbooks/[^/]*/||' | sort -u | jq -R -s -c 'split("\n") | map(select(length > 0))') | |
| if [ "$ALL_PLAYBOOKS" = "[]" ] || [ -z "$ALL_PLAYBOOKS" ]; then | |
| echo "playbooks=[]" >> $GITHUB_OUTPUT | |
| else | |
| echo "playbooks=$ALL_PLAYBOOKS" >> $GITHUB_OUTPUT | |
| fi | |
| fi | |
| - name: Build test matrix from playbook metadata | |
| id: build-matrix | |
| env: | |
| PLAYBOOKS: ${{ steps.detect.outputs.playbooks }} | |
| run: | | |
| if [ "$PLAYBOOKS" = "[]" ] || [ -z "$PLAYBOOKS" ]; then | |
| echo "matrix=[]" >> $GITHUB_OUTPUT | |
| echo "has_entries=false" >> $GITHUB_OUTPUT | |
| exit 0 | |
| fi | |
| MATRIX=$(python3 << 'PYEOF' | |
| import json, os | |
| from pathlib import Path | |
| playbooks = json.loads(os.environ["PLAYBOOKS"]) | |
| matrix = [] | |
| for pb_id in playbooks: | |
| for cat in ["core", "supplemental"]: | |
| pb_file = Path(f"playbooks/{cat}/{pb_id}/playbook.json") | |
| if pb_file.exists(): | |
| meta = json.loads(pb_file.read_text()) | |
| tested = meta.get("tested_platforms", {}) | |
| if not tested: | |
| continue | |
| required = meta.get("required_platforms", {}) | |
| for device, platform_list in tested.items(): | |
| required_platforms = set(required.get(device, [])) | |
| for platform in platform_list: | |
| os_label = "Windows" if platform == "windows" else "Linux" | |
| matrix.append({ | |
| "playbook": pb_id, | |
| "platform": platform, | |
| "arch": device, | |
| "runner": json.dumps(["self-hosted", os_label, device]), | |
| "required": platform in required_platforms, | |
| }) | |
| break | |
| print(json.dumps(matrix)) | |
| PYEOF | |
| ) | |
| if [ "$MATRIX" = "[]" ] || [ -z "$MATRIX" ]; then | |
| echo "matrix=[]" >> $GITHUB_OUTPUT | |
| echo "has_entries=false" >> $GITHUB_OUTPUT | |
| else | |
| echo "matrix=$MATRIX" >> $GITHUB_OUTPUT | |
| echo "has_entries=true" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Display test matrix | |
| run: | | |
| echo "Test matrix: ${{ steps.build-matrix.outputs.matrix }}" | |
| test-playbooks: | |
| name: "Test ${{ matrix.playbook }} (${{ matrix.platform }}/${{ matrix.arch }})${{ matrix.required && ' ✦' || '' }}" | |
| needs: detect-changes | |
| if: needs.detect-changes.outputs.has_entries == 'true' | |
| runs-on: ${{ fromJson(matrix.runner) }} | |
| continue-on-error: ${{ !matrix.required }} | |
| env: | |
| PYTHONUTF8: 1 | |
| PYTHONIOENCODING: utf-8 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson(needs.detect-changes.outputs.matrix) }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ (matrix.playbook == 'open-webui-chat' || matrix.playbook == 'vllm-inference') && '3.12' || '3.13' }} | |
| - name: Install test dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install pyyaml | |
| # Free the GPU before every job. A prior lemonade/LM Studio playbook can | |
| # strand a model on the shared runner (its unload teardown is skipped when | |
| # an earlier test in that playbook fails), which OOMs the next job. This | |
| # runs regardless of what ran before, so it also protects non-lemonade jobs. | |
| # Best-effort and time-bounded: a stuck unload must never wedge the job | |
| # (see the ollama note below), so these steps are continue-on-error with a | |
| # short timeout. | |
| - name: Free GPU before tests (Linux) | |
| if: matrix.platform == 'linux' | |
| shell: bash | |
| continue-on-error: true | |
| timeout-minutes: 5 | |
| run: | | |
| lemonade unload || true | |
| lms unload --all || true | |
| # Only unload ollama models if the server is already running. Running any | |
| # ollama command otherwise starts the server, which can inherit this | |
| # step's stdout and block it. This leaves the server running, same as the | |
| # lemonade/lms unloads above. | |
| if pgrep -x ollama >/dev/null 2>&1; then | |
| ollama ps 2>/dev/null | awk 'NR>1{print $1}' | while read -r m; do [ -n "$m" ] && ollama stop "$m" 2>/dev/null || true; done || true | |
| fi | |
| - name: Free GPU before tests (Windows) | |
| if: matrix.platform == 'windows' | |
| shell: powershell | |
| continue-on-error: true | |
| timeout-minutes: 5 | |
| run: | | |
| try { lemonade unload } catch {} | |
| try { lms unload --all } catch {} | |
| # Only unload ollama models if the server is already running. Running any | |
| # ollama command otherwise auto-starts the server, which inherits this | |
| # step's stdout handle and hangs the job (the pipeline never sees EOF). | |
| if (Get-Process ollama -ErrorAction SilentlyContinue) { | |
| try { | |
| $models = & ollama ps 2>$null | Select-Object -Skip 1 | ForEach-Object { ($_ -split '\s+')[0] } | Where-Object { $_ } | |
| foreach ($m in $models) { & ollama stop $m 2>$null | Out-Null } | |
| } catch {} | |
| } | |
| exit 0 | |
| - name: Extract and run tests (Windows) | |
| if: matrix.platform == 'windows' | |
| id: run_tests_windows | |
| shell: powershell | |
| run: | | |
| python .github/scripts/run_playbook_tests.py --playbook "${{ matrix.playbook }}" --platform windows --device "${{ matrix.arch }}" | |
| - name: Extract and run tests (Linux) | |
| if: matrix.platform == 'linux' | |
| id: run_tests_linux | |
| shell: bash | |
| run: | | |
| python .github/scripts/run_playbook_tests.py --playbook "${{ matrix.playbook }}" --platform linux --device "${{ matrix.arch }}" | |
| - name: Upload test artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-results-${{ matrix.playbook }}-${{ matrix.platform }}-${{ matrix.arch }} | |
| path: test-results/ | |
| if-no-files-found: ignore | |
| # Open / update a tracking issue for every failed test, but ONLY on the | |
| # main branch. PRs (whose base is main) intentionally don't trigger this: | |
| # those failures are usually the PR author's responsibility and would | |
| # otherwise spam the tracker. `failure()` only fires when a previous step | |
| # in this job failed, so passing matrix entries are skipped automatically. | |
| # | |
| # We invoke python directly (no shell-specific line continuations) so the | |
| # same step works on both Windows and Linux self-hosted runners. | |
| - name: Create failure issues (Linux) | |
| if: failure() && matrix.platform == 'linux' && github.event_name != 'pull_request' && github.ref == 'refs/heads/main' | |
| shell: bash | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| RUNNER_NAME: ${{ runner.name }} | |
| run: | | |
| python .github/scripts/create_failure_issues.py \ | |
| --repo "${{ github.repository }}" \ | |
| --playbook "${{ matrix.playbook }}" \ | |
| --workflow-file "test-playbooks.yml" \ | |
| --runner-labels "self-hosted,Linux,${{ matrix.arch }}" \ | |
| --runner-name "${{ runner.name }}" \ | |
| --run-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ | |
| --sha "${{ github.sha }}" | |
| - name: Create failure issues (Windows) | |
| if: failure() && matrix.platform == 'windows' && github.event_name != 'pull_request' && github.ref == 'refs/heads/main' | |
| shell: powershell | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| RUNNER_NAME: ${{ runner.name }} | |
| run: | | |
| python .github/scripts/create_failure_issues.py ` | |
| --repo "${{ github.repository }}" ` | |
| --playbook "${{ matrix.playbook }}" ` | |
| --workflow-file "test-playbooks.yml" ` | |
| --runner-labels "self-hosted,Windows,${{ matrix.arch }}" ` | |
| --runner-name "${{ runner.name }}" ` | |
| --run-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" ` | |
| --sha "${{ github.sha }}" | |
| - name: Test Summary (Windows) | |
| if: always() && matrix.platform == 'windows' | |
| shell: powershell | |
| run: | | |
| $summary = @" | |
| ### Test Results for ``${{ matrix.playbook }}`` (${{ matrix.platform }}/${{ matrix.arch }}) | |
| See the test artifacts for detailed logs. | |
| "@ | |
| Add-Content -Path $env:GITHUB_STEP_SUMMARY -Value $summary | |
| - name: Test Summary (Linux) | |
| if: always() && matrix.platform == 'linux' | |
| shell: bash | |
| run: | | |
| cat >> $GITHUB_STEP_SUMMARY << 'EOF' | |
| ### Test Results for `${{ matrix.playbook }}` (${{ matrix.platform }}/${{ matrix.arch }}) | |
| See the test artifacts for detailed logs. | |
| EOF | |
| test-gate: | |
| name: Test Gate (Required Tests) | |
| needs: [detect-changes, test-playbooks] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check required tests | |
| run: | | |
| echo "test-playbooks result: ${{ needs.test-playbooks.result }}" | |
| if [ "${{ needs.test-playbooks.result }}" = "failure" ]; then | |
| echo "::error::One or more required tests failed" | |
| exit 1 | |
| fi | |
| echo "All required tests passed (or no tests were needed)" |