Fix msbuild skill spec conformance violations #469
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: evaluation | |
| on: | |
| pull_request: | |
| paths: | |
| - 'plugins/**' | |
| - 'tests/**' | |
| - '.github/workflows/**' | |
| - 'eng/skill-validator/**' | |
| schedule: | |
| - cron: '0 */3 * * *' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| jobs: | |
| discover: | |
| # Skip fork PRs (handled by evaluation-fork-pr.yml) and scheduled runs on forks. | |
| # Without this guard, forks that have activated Actions will run the evaluation | |
| # schedule indefinitely, consuming the fork owner's Actions minutes. | |
| if: >- | |
| (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && | |
| (github.event_name != 'schedule' || github.repository == 'dotnet/skills') | |
| runs-on: ubuntu-latest | |
| outputs: | |
| entries: ${{ steps.find.outputs.entries }} | |
| has_entries: ${{ steps.find.outputs.has_entries }} | |
| is_infra: ${{ steps.find.outputs.is_infra }} | |
| plugins: ${{ steps.find.outputs.plugins }} | |
| has_plugins: ${{ steps.find.outputs.has_plugins }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Find skills to evaluate | |
| id: find | |
| run: | | |
| $entries = @() | |
| $plugins = @() | |
| if ("${{ github.event_name }}" -eq "pull_request") { | |
| # PR: detect individual changed skills | |
| $base = "${{ github.event.pull_request.base.sha }}" | |
| $head = "${{ github.event.pull_request.head.sha }}" | |
| $changedFiles = git diff --name-only --diff-filter=ACMR $base $head | |
| # Check if any changed files are in infrastructure paths (.github/workflows/ or eng/skill-validator/) | |
| $hasInfraChanges = $changedFiles | | |
| Where-Object { $_ -match '^(\.github/workflows/|eng/skill-validator/)' } | | |
| Select-Object -First 1 | |
| if ($hasInfraChanges) { | |
| # Infra changes can affect any evaluation — evaluate all plugins | |
| Write-Host "Infrastructure changes detected, evaluating all plugins" | |
| echo "is_infra=true" >> $env:GITHUB_OUTPUT | |
| $plugins = @(Get-ChildItem -Path "plugins" -Directory | | |
| Where-Object { (Test-Path (Join-Path $_.FullName "skills")) -and (Test-Path (Join-Path "tests" $_.Name)) } | | |
| Select-Object -ExpandProperty Name) | |
| $entries = @($plugins | ForEach-Object { | |
| @{ name = $_; plugin = $_; skills_path = "plugins/$_/skills" } | |
| }) | |
| } else { | |
| # Extract unique plugin/skill pairs from changed files under plugins/*/skills/*/ or tests/*/*/ | |
| $changedPairs = @($changedFiles | | |
| Where-Object { $_ -match '^(?:plugins/([^/]+)/skills|tests/([^/]+))/([^/]+)/' } | | |
| ForEach-Object { | |
| $p = if ($Matches[1]) { $Matches[1] } else { $Matches[2] } | |
| "$p/$($Matches[3])" | |
| } | | |
| Sort-Object -Unique) | |
| # Filter to skills that have a SKILL.md and a tests directory | |
| $entries = @($changedPairs | ForEach-Object { | |
| $parts = $_ -split '/' | |
| $plugin = $parts[0] | |
| $skill = $parts[1] | |
| $skillMd = Join-Path "plugins" $plugin "skills" $skill "SKILL.md" | |
| $testsDir = Join-Path "tests" $plugin | |
| if ((Test-Path $skillMd) -and (Test-Path $testsDir)) { | |
| @{ | |
| name = "$plugin--$skill" | |
| plugin = $plugin | |
| skills_path = "plugins/$plugin/skills/$skill" | |
| } | |
| } | |
| } | Where-Object { $_ }) | |
| $plugins = @($entries | ForEach-Object { $_.plugin } | Sort-Object -Unique) | |
| } | |
| } else { | |
| # Schedule: evaluate all plugins with skills and tests | |
| $plugins = @(Get-ChildItem -Path "plugins" -Directory | | |
| Where-Object { (Test-Path (Join-Path $_.FullName "skills")) -and (Test-Path (Join-Path "tests" $_.Name)) } | | |
| Select-Object -ExpandProperty Name) | |
| $entries = @($plugins | ForEach-Object { | |
| @{ name = $_; plugin = $_; skills_path = "plugins/$_/skills" } | |
| }) | |
| } | |
| # Output entries for evaluate matrix | |
| if (-not $entries -or $entries.Count -eq 0) { | |
| Write-Host "No entries to evaluate" | |
| echo "entries=[]" >> $env:GITHUB_OUTPUT | |
| echo "has_entries=false" >> $env:GITHUB_OUTPUT | |
| } else { | |
| $json = $entries | ConvertTo-Json -Compress -AsArray | |
| Write-Host "Entries to evaluate: $json" | |
| echo "entries=$json" >> $env:GITHUB_OUTPUT | |
| echo "has_entries=true" >> $env:GITHUB_OUTPUT | |
| } | |
| # Output plugins for publish-benchmark | |
| if (-not $plugins -or $plugins.Count -eq 0) { | |
| echo "plugins=[]" >> $env:GITHUB_OUTPUT | |
| echo "has_plugins=false" >> $env:GITHUB_OUTPUT | |
| } else { | |
| $cjson = $plugins | ConvertTo-Json -Compress -AsArray | |
| echo "plugins=$cjson" >> $env:GITHUB_OUTPUT | |
| echo "has_plugins=true" >> $env:GITHUB_OUTPUT | |
| } | |
| shell: pwsh | |
| run-evaluation: | |
| needs: discover | |
| if: needs.discover.outputs.has_entries == 'true' | |
| uses: ./.github/workflows/evaluation-run.yml | |
| with: | |
| entries: ${{ needs.discover.outputs.entries }} | |
| runs: ${{ needs.discover.outputs.is_infra == 'true' && '2' || (github.ref == 'refs/heads/main' && '5' || '3') }} | |
| pr-number: ${{ github.event.pull_request.number || '' }} | |
| secrets: | |
| COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} | |
| COPILOT_GITHUB_TOKEN_2: ${{ secrets.COPILOT_GITHUB_TOKEN_2 }} | |
| COPILOT_GITHUB_TOKEN_3: ${{ secrets.COPILOT_GITHUB_TOKEN_3 }} | |
| COPILOT_GITHUB_TOKEN_4: ${{ secrets.COPILOT_GITHUB_TOKEN_4 }} | |
| COPILOT_GITHUB_TOKEN_5: ${{ secrets.COPILOT_GITHUB_TOKEN_5 }} | |
| COPILOT_GITHUB_TOKEN_6: ${{ secrets.COPILOT_GITHUB_TOKEN_6 }} | |
| COPILOT_GITHUB_TOKEN_7: ${{ secrets.COPILOT_GITHUB_TOKEN_7 }} | |
| COPILOT_GITHUB_TOKEN_8: ${{ secrets.COPILOT_GITHUB_TOKEN_8 }} | |
| publish-benchmark: | |
| needs: [discover, run-evaluation] | |
| if: github.ref == 'refs/heads/main' && needs.discover.outputs.has_plugins == 'true' | |
| concurrency: | |
| group: publish-benchmark-deploy | |
| cancel-in-progress: false | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| - name: Download evaluation artifacts | |
| uses: actions/download-artifact@v8 | |
| with: | |
| pattern: skill-validator-results-* | |
| path: all-results/ | |
| merge-multiple: false | |
| - name: Fetch existing benchmark data | |
| run: | | |
| git fetch origin gh-pages:gh-pages 2>/dev/null || true | |
| mkdir -p /tmp/gh-pages/data | |
| git checkout gh-pages -- data/ 2>/dev/null && cp -r data/* /tmp/gh-pages/data/ && git checkout HEAD -- . || true | |
| - name: Generate Benchmark Data | |
| run: | | |
| $sha = "${{ github.sha }}" | |
| $commitMsg = git log -1 --format='%s' $sha | |
| $commitTimestamp = git log -1 --format='%aI' $sha | |
| $commitAuthor = git log -1 --format='%an' $sha | |
| $commitJson = @{ | |
| id = $sha | |
| message = $commitMsg | |
| timestamp = $commitTimestamp | |
| url = "https://github.com/${{ github.repository }}/commit/$sha" | |
| author = @{ name = $commitAuthor; username = "${{ github.actor }}" } | |
| } | ConvertTo-Json -Compress | |
| # Process each plugin's results | |
| $plugins = '${{ needs.discover.outputs.plugins }}' | ConvertFrom-Json | |
| foreach ($plugin in $plugins) { | |
| $artifactDir = "all-results/skill-validator-results-$plugin" | |
| $runDir = Get-ChildItem -Path $artifactDir -Directory -ErrorAction SilentlyContinue | | |
| Where-Object { $_.Name -match '^\d{8}-\d{6}$' } | | |
| Sort-Object Name -Descending | | |
| Select-Object -First 1 | |
| if (-not $runDir) { | |
| Write-Warning "No run results found for $plugin, skipping benchmark generation" | |
| continue | |
| } | |
| $resultsFile = Join-Path $runDir.FullName "results.json" | |
| Write-Host "`n=== Generating benchmark data for: $plugin ===" | |
| $existingFile = "/tmp/gh-pages/data/$plugin.json" | |
| $params = @{ | |
| ResultsFile = $resultsFile | |
| PluginName = $plugin | |
| OutputDir = "/tmp/gh-pages/data" | |
| CommitJson = $commitJson | |
| } | |
| if ((Test-Path $existingFile) -and (Get-Content $existingFile -Raw -ErrorAction SilentlyContinue)) { | |
| $params.ExistingDataFile = $existingFile | |
| } | |
| & ./eng/dashboard/generate-benchmark-data.ps1 @params | |
| } | |
| # Generate components.json manifest | |
| $plugins = Get-ChildItem -Path "/tmp/gh-pages/data" -Filter "*.json" -File -ErrorAction SilentlyContinue | | |
| Where-Object { $_.Name -ne "components.json" } | | |
| ForEach-Object { $_.BaseName } | |
| @($plugins) | ConvertTo-Json -AsArray | Out-File -FilePath "/tmp/gh-pages/data/components.json" -Encoding utf8 | |
| shell: pwsh | |
| - name: Deploy to GitHub Pages | |
| run: | | |
| cd /tmp | |
| REPO_URL="https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git" | |
| if git ls-remote --exit-code --heads "$REPO_URL" gh-pages > /dev/null 2>&1; then | |
| git clone --branch gh-pages --single-branch "$REPO_URL" deploy | |
| else | |
| mkdir deploy && cd deploy && git init && git checkout -b gh-pages | |
| git remote add origin "$REPO_URL" | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| cd /tmp | |
| fi | |
| cd /tmp/deploy | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| mkdir -p data | |
| cp /tmp/gh-pages/data/*.json data/ | |
| cp ${{ github.workspace }}/eng/dashboard/dashboard.html index.html | |
| cp ${{ github.workspace }}/eng/dashboard/dashboard.js dashboard.js | |
| git add . | |
| git diff --cached --quiet && echo "No changes to deploy" && exit 0 | |
| git commit -m "Update benchmark data" | |
| git push origin gh-pages |