test-suite.yml

# When /test-suite is commented on a PR, checks out the PR, builds clang and
# then the test-suite in several configurations. It then checks out the base of
# the PR, builds clang and the test-suite again, and then uploads the diff of
# the codegen.

name: Diff test-suite codegen

on:
  issue_comment:
    types:
      - created

permissions:
  contents: read

jobs:
  permission-check:
    if: github.event.issue.pull_request && startswith(github.event.comment.body, '/test-suite')
    runs-on: ubuntu-26.04
    environment:
      name: main-branch-only
      deployment: false
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
          sparse-checkout: |
            .github/workflows/

      - name: Check Permissions
        uses: ./.github/workflows/require-team-membership
        with:
          team-slug: llvm-committers
          LLVM_TOKEN_GENERATOR_CLIENT_ID: ${{ secrets.LLVM_TOKEN_GENERATOR_CLIENT_ID }}
          LLVM_TOKEN_GENERATOR_PRIVATE_KEY: ${{ secrets.LLVM_TOKEN_GENERATOR_PRIVATE_KEY }}

  react-to-comment:
    needs:
      - permission-check
    runs-on: ubuntu-26.04
    permissions:
      pull-requests: write
    steps:
      - name: Thumbs up comment
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          script: |
            github.rest.reactions.createForIssueComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              comment_id: context.payload.comment.id,
              content: '+1'
            })

  test-suite:
    name: Build test-suite and diff
    runs-on: ubuntu-26.04
    container:
      image: ghcr.io/llvm/ci-ubuntu-26.04:latest@sha256:7ac659feb90ac48dd8676e4eb305b9aa5b31fc8b329b7e39379c38af06efc6e1
    needs:
      - permission-check
    env:
      # llvm-nm/llvm-ar/llvm-ranlib are needed for the microbenchmarks
      NINJA_TARGETS: clang lld llvm-size llvm-nm llvm-ar llvm-ranlib
    steps:
      - name: Get pull request
        id: get-pr
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          script: |
            const { data: pr } = await github.rest.pulls.get({
              owner: context.repo.owner,
              repo: context.repo.repo,
              pull_number: context.payload.issue.number
            })
            return pr
      - name: Check pull request is mergeable
        if: ${{ !fromJSON(steps.get-pr.outputs.result).mergeable }}
        run: |
          cat << EOF > comments
          [{"body": "Unable to diff test-suite with PR, PR isn't mergeable"}]
          EOF
          exit 1
      - name: Checkout pull request
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          ref: ${{ fromJSON(steps.get-pr.outputs.result).merge_commit_sha }}
          # test merge commits are stored in the base repository, not the head
          repository: ${{ github.repository }}
          fetch-depth: 2
          path: llvm-project
          persist-credentials: false
      - name: Checkout llvm/llvm-test-suite
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          repository: llvm/llvm-test-suite
          path: llvm-test-suite
          persist-credentials: false
      - name: Checkout FFmpeg
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          repository: FFmpeg/FFmpeg
          ref: db69d06eeeab4f46da15030a80d539efb4503ca8 # n7.1.1
          path: llvm-test-suite/test-suite-externals/ffmpeg
          persist-credentials: false
      - name: Setup environment variables
        run: |
          echo "HEAD_SHA=$(git rev-parse HEAD)" >> $GITHUB_ENV
          echo "BASE_SHA=$(git rev-parse HEAD^)" >> $GITHUB_ENV
          SCRIPTS_DIR="$GITHUB_WORKSPACE/llvm-project/.github/workflows/test-suite"
          echo "SCRIPTS_DIR=$SCRIPTS_DIR" >> $GITHUB_ENV
          echo "$SCRIPTS_DIR" >> $GITHUB_PATH
          echo "$GITHUB_WORKSPACE/llvm-project/build/bin" >> $GITHUB_PATH
        working-directory: llvm-project
      - name: Install system dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y tcl libc6-dev-arm64-cross libc6-dev-riscv64-cross libgcc-14-dev-arm64-cross libgcc-14-dev-riscv64-cross libstdc++-14-dev-arm64-cross libstdc++-14-dev-riscv64-cross python3-pandas python3-tabulate
      - name: Configure Clang
        run: cmake -B build -C $SCRIPTS_DIR/llvm.cmake llvm -GNinja
        working-directory: llvm-project
      - name: Build Clang @ head
        run: ninja -C build $NINJA_TARGETS
        working-directory: llvm-project
      - name: Configure and build test-suite @ head
        run: |
          configure-and-build.sh rva23u64-O3-head $SCRIPTS_DIR/riscv64.cmake
          configure-and-build.sh armv9-a-O3-head $SCRIPTS_DIR/aarch64.cmake
          configure-and-build.sh x86-64-v3-O3-head $SCRIPTS_DIR/x86_64.cmake
        working-directory: llvm-test-suite
      - name: Build Clang @ base
        run: git checkout $BASE_SHA && ninja -C build $NINJA_TARGETS
        working-directory: llvm-project
      - name: Configure and build test-suite @ base
        run: |
          configure-and-build.sh rva23u64-O3-base $SCRIPTS_DIR/riscv64.cmake
          configure-and-build.sh armv9-a-O3-base $SCRIPTS_DIR/aarch64.cmake
          configure-and-build.sh x86-64-v3-O3-base $SCRIPTS_DIR/x86_64.cmake
        working-directory: llvm-test-suite
      - name: Compute diffs
        run: |
          mkdir diffs
          ./utils/tdiff.py -a build.rva23u64-O3-base -b build.rva23u64-O3-head -s all > diffs/rva23u64-O3.diff || true
          ./utils/tdiff.py -a build.armv9-a-O3-base -b build.armv9-a-O3-head -s all > diffs/armv9-a-O3.diff || true
          ./utils/tdiff.py -a build.x86-64-v3-O3-base -b build.x86-64-v3-O3-head -s all > diffs/x86-64-v3-O3.diff || true
        working-directory: llvm-test-suite
      - name: Upload diffs
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        id: upload-diffs
        with:
          name: diffs
          path: llvm-test-suite/diffs
      - name: Upload results
        id: upload-results
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: results
          path: llvm-test-suite/results*.json
      - name: Compare results
        run: |
          for ARCH in rva23u64-O3 armv9-a-O3 x86-64-v3-O3
          do
            ./utils/compare.py results.$ARCH-base.json vs results.$ARCH-head.json -m size..text --only-changed --lhs-name base --rhs-name head --format=md > ../compare-$ARCH.md || true
          done
        working-directory: llvm-test-suite
      - name: Create comment
        env:
          ARTIFACT_URL: ${{ steps.upload-diffs.outputs.artifact-url }}
          RESULTS_URL: ${{ steps.upload-results.outputs.artifact-url }}
        run: |
          cat << EOF > comment-body
          test-suite diff from $BASE_SHA...$HEAD_SHA: $ARTIFACT_URL

          Statistics: $RESULTS_URL

          ## riscv64-linux-gnu \`-march=rva23u64 -O3\`
          $(cat compare-rva23u64-O3.md)
          ## aarch64-linux-gnu \`-march=armv9-a -O3\`
          $(cat compare-armv9-a-O3.md)
          ## x86_64-linux-gnu \`-march=x86-64-v3 -O3\`
          $(cat compare-x86-64-v3-O3.md)
          EOF
          # write escaped markdown to format expected by issue-write.yml
          python3 -c 'import json; print(json.dumps([{"body": open("comment-body").read()}]))' > comments
      - name: Create comment on failure
        if: failure()
        env:
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        run: |
          cat << EOF > comments
          [{"body" : "Failed to get test-suite diff from $BASE_SHA...$HEAD_SHA: $RUN_URL"}]
          EOF
      - name: Save PR number
        if: always()
        env:
          PR_NUMBER: ${{ fromJSON(steps.get-pr.outputs.result).number }}
        run: echo $PR_NUMBER > pr_number
      - name: Upload comment and PR number
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        if: always()
        with:
          name: workflow-args
          path: |
            comments
            pr_number