trunk.yml

name: trunk

on:
  push:
    branches:
      - main
      - release/*
    tags:
      - ciflow/trunk/*
  pull_request:
    paths:
      - .ci/docker/ci_commit_pins/pytorch.txt
      - .ci/scripts/**
      - zephyr/**
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }}
  cancel-in-progress: true

jobs:
  docker-image:
    name: Resolve CI docker image
    uses: ./.github/workflows/_docker-image.yml

  # Emits the list of changed files for the current PR or push commit.
  # On PR: PR diff. On push: diff against `github.event.before`.
  # On events without a diff base (workflow_dispatch, tag creation,
  # initial push), emits '*' — note that `contains('*', 'path')` is
  # false (literal substring match, not glob), so path-filtered jobs
  # rely on run-decision's is-full-run output for those events.
  changed-files:
    name: Get changed files
    uses: ./.github/workflows/_get-changed-files.yml
    with:
      # Opt in to push-event diff so path-filtered jobs can skip pushes
      # that don't touch their relevant paths. Without this, push events
      # emit '*' and `contains('*', 'path')` is always false.
      include-push-diff: true

  run-decision:
    name: CI run decision
    uses: ./.github/workflows/_ci-run-decision.yml

  test-models-macos-cpu:
    name: test-models-macos-cpu
    needs: run-decision
    if: |
      github.event_name == 'pull_request' ||
      needs.run-decision.outputs.is-full-run == 'true'
    uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
    strategy:
      matrix:
        # Mac runners are expensive and limited, and non reliable.
        # Do some basic testing for macos jobs, and rely mostly on
        # test-models-linux-aarch64 job instead.
        model: [emformer_join, ic4, llama2, mobilebert, mv3, resnet50, vit, w2l]
        backend: [xnnpack-quantization-delegation]
        include:
          - model: efficient_sam
            backend: portable
          - model: llama
            backend: portable
          - model: llama3_2_vision_encoder
            backend: portable
          - model: mv3
            backend: portable
      fail-fast: false
    with:
      default-packages: ""
      runner: macos-m1-stable
      python-version: '3.11'
      submodules: 'recursive'
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      timeout: 90
      script: |
        MODEL_NAME=${{ matrix.model }}
        BUILD_TOOL=cmake
        BACKEND=${{ matrix.backend }}

        bash .ci/scripts/setup-conda.sh
        # Setup MacOS dependencies as there is no Docker support on MacOS atm
        PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh --build-tool "${BUILD_TOOL}"
        # Build and test executorch
        PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}"

  test-arm-backend-zephyr:
    needs: docker-image
    name: test-arm-backend-zephyr
    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
    permissions:
      id-token: write
      contents: read
    strategy:
      matrix:
        include:
          - { readme: zephyr/samples/hello-executorch/README.md, target: ethos-u55 }
          - { readme: zephyr/samples/hello-executorch/README.md, target: cortex-m55 }
          - { readme: zephyr/samples/hello-executorch/README.md, target: ethos-u85 }
          - { readme: zephyr/samples/mv2-ethosu/README.md, target: ethos-u55 }
          - { readme: zephyr/samples/mv2-ethosu/README.md, target: ethos-u85 }
      fail-fast: false
    with:
      runner: mt-l-x86iavx512-8-64
      docker-image: ${{ needs.docker-image.outputs.docker-registry }}/ci-image:executorch-ubuntu-22.04-zephyr-sdk-${{ needs.docker-image.outputs.ci-docker-hash }}
      submodules: 'recursive'
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      timeout: 120
      script: |
        #!/bin/bash

        set -euo pipefail

        # Zephyr requires a Python 3.12 venv; keep this independent of the
        # base image's default ExecuTorch Python environment.
        python3.12 -m venv zephyr_test_venv
        source zephyr_test_venv/bin/activate

        # The Zephyr SDK host tools carry their own runtime. The CI Docker image
        # sets a global LD_LIBRARY_PATH for other jobs, which can make SDK tools
        # pick host libs.
        unset LD_LIBRARY_PATH

        # Test zephyr backend
        .ci/scripts/test_zephyr.sh \
          --targets "${TARGET_LIST:-${{ matrix.target }}}" \
          --zephyr-samples-readme-path "${{ matrix.readme }}"

  test-models-linux-aarch64:
    needs: docker-image
    name: test-models-linux-aarch64
    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
    permissions:
      id-token: write
      contents: read
    strategy:
      matrix:
        model: [linear, add, add_mul, ic3, ic4, mv2, mv3, resnet18, resnet50, vit, w2l, mobilebert, emformer_join, emformer_transcribe]
        backend: [portable, xnnpack-quantization-delegation]
        runner: [mt-l-arm64g4-16-62]
        include:
          - model: lstm
            backend: portable
            runner: mt-l-arm64g4-16-62
          - model: mul
            backend: portable
            runner: mt-l-arm64g4-16-62
          - model: softmax
            backend: portable
            runner: mt-l-arm64g4-16-62
          - model: phi_4_mini
            backend: portable
            runner: mt-l-arm64g4-16-62
          - model: qwen2_5_1_5b
            backend: portable
            runner: mt-l-arm64g4-16-62
          - model: llama3_2_vision_encoder
            backend: portable
            runner: mt-l-arm64g4-16-62
      fail-fast: false
    with:
      runner: ${{ matrix.runner }}
      docker-image: ${{ needs.docker-image.outputs.docker-registry }}/ci-image:executorch-ubuntu-22.04-gcc11-aarch64-${{ needs.docker-image.outputs.ci-docker-hash }}
      submodules: 'recursive'
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      timeout: 90
      script: |
        # The generic Linux job chooses to use base env, not the one setup by the image
        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
        conda activate "${CONDA_ENV}"

        MODEL_NAME=${{ matrix.model }}
        BUILD_TOOL="cmake"
        BACKEND=${{ matrix.backend }}

        PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}"
        # Build and test ExecuTorch
        PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}"

  test-custom-ops-macos:
    name: test-custom-ops-macos
    needs: run-decision
    if: |
      github.event_name == 'pull_request' ||
      needs.run-decision.outputs.is-full-run == 'true'
    uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
    strategy:
      matrix:
        include:
          - build-tool: cmake
      fail-fast: false
    with:
      default-packages: ""
      runner: macos-m1-stable
      python-version: '3.11'
      submodules: 'recursive'
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      script: |
        BUILD_TOOL=${{ matrix.build-tool }}

        bash .ci/scripts/setup-conda.sh
        # Setup MacOS dependencies as there is no Docker support on MacOS atm
        PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh --build-tool "${BUILD_TOOL}"
        # Build and test custom ops
        PYTHON_EXECUTABLE=python ${CONDA_RUN} bash examples/portable/custom_ops/test_custom_ops.sh "${BUILD_TOOL}"

  test-selective-build-macos:
    name: test-selective-build-macos
    needs: run-decision
    if: |
      github.event_name == 'pull_request' ||
      needs.run-decision.outputs.is-full-run == 'true'
    uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
    strategy:
      matrix:
        include:
          - build-tool: cmake
      fail-fast: false
    with:
      default-packages: ""
      runner: macos-m1-stable
      python-version: '3.11'
      submodules: 'recursive'
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      script: |
        BUILD_TOOL=${{ matrix.build-tool }}

        bash .ci/scripts/setup-conda.sh
        # Setup MacOS dependencies as there is no Docker support on MacOS atm
        PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh --build-tool "${BUILD_TOOL}"
        # Build and test selective build
        PYTHON_EXECUTABLE=python ${CONDA_RUN} bash examples/selective_build/test_selective_build.sh "${BUILD_TOOL}"

  test-demo-backend-delegation:
    needs: docker-image
    name: test-demo-backend-delegation
    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
    permissions:
      id-token: write
      contents: read
    strategy:
      matrix:
        include:
          - build-tool: buck2
          - build-tool: cmake
      fail-fast: false
    with:
      runner: mt-l-x86iavx512-8-64
      docker-image: ${{ needs.docker-image.outputs.docker-registry }}/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }}
      submodules: 'recursive'
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      script: |
        # The generic Linux job chooses to use base env, not the one setup by the image
        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
        conda activate "${CONDA_ENV}"

        BUILD_TOOL=${{ matrix.build-tool }}
        PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}"
        # Test selective build
        PYTHON_EXECUTABLE=python bash examples/portable/scripts/test_demo_backend_delegation.sh "${BUILD_TOOL}"

  test-arm-backend-ethos-u:
    needs: docker-image
    name: test-arm-backend-ethos-u
    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
    permissions:
      id-token: write
      contents: read
    strategy:
      matrix:
        include:
          - test_arm_backend: test_pytest_ops_ethos_u55
          - test_arm_backend: test_pytest_models_ethos_u55
          - test_arm_backend: test_run_ethos_u55
          - test_arm_backend: test_minimal_classic_ml_ethos_u55
          - test_arm_backend: test_pytest_ops_ethos_u85
          - test_arm_backend: test_pytest_models_ethos_u85
          - test_arm_backend: test_run_ethos_u85
          - test_arm_backend: test_smaller_stories_llama_tosa
          - test_arm_backend: test_model_smollm2_135M_ethos_u85
          - test_arm_backend: test_runtime_ethos_u
          - test_arm_backend: test_ootb_tests_ethos_u
          - test_arm_backend: test_ootb_tests_tosa
          - test_arm_backend: test_deit_e2e_ethos_u
          - test_arm_backend: test_person_detection_e2e_cortex_m
      fail-fast: false
    with:
      runner: mt-l-x86iavx512-8-64
      docker-image: ${{ needs.docker-image.outputs.docker-registry }}/ci-image:executorch-ubuntu-22.04-arm-sdk-${{ needs.docker-image.outputs.ci-docker-hash }}
      submodules: 'recursive'
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      timeout: 120
      script: |
        # The generic Linux job chooses to use base env, not the one setup by the image
        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
        conda activate "${CONDA_ENV}"

        source .ci/scripts/utils.sh
        install_executorch "--use-pt-pinned-commit"

        .ci/scripts/setup-arm-baremetal-tools.sh

        # Increase number of files user can monitor to bypass buck failures.
        # Hopefully this is high enough for this setup.
        # Node-level, so an unprivileged pod cannot change it.
        sudo sysctl fs.inotify.max_user_watches=1048576 2>/dev/null || true

        ARM_TEST=${{ matrix.test_arm_backend }}

        # Output test report on pytest runs so that github can surface failing tests.
        if [[ -n "${RUNNER_TEST_RESULTS_DIR:-}" ]]; then
              export PYTEST_ADDOPTS="--junit-xml=${RUNNER_TEST_RESULTS_DIR}/${ARM_TEST}.xml ${PYTEST_ADDOPTS:-}"
        fi

        # Test test_arm_backend.sh with test
        backends/arm/test/test_arm_backend.sh "${ARM_TEST}"

  test-arm-backend-vkml:
    needs: docker-image
    name: test-arm-backend-vkml
    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
    permissions:
      id-token: write
      contents: read
    strategy:
      matrix:
        include:
          - test_arm_backend: test_pytest_ops_vkml
          - test_arm_backend: test_pytest_models_vkml
          - test_arm_backend: test_run_vkml
          - test_arm_backend: test_ootb_tests_vgf
          - test_arm_backend: test_runtime_vgf
          - test_arm_backend: test_smaller_stories_llama_vkml
      fail-fast: false
    with:
      runner: mt-l-x86iavx512-8-64
      docker-image: ${{ needs.docker-image.outputs.docker-registry }}/ci-image:executorch-ubuntu-24.04-arm-sdk-${{ needs.docker-image.outputs.ci-docker-hash }}
      submodules: 'recursive'
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      timeout: 120
      script: |
        # The generic Linux job chooses to use base env, not the one setup by the image
        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
        conda activate "${CONDA_ENV}"
        source .ci/scripts/utils.sh
        install_executorch "--use-pt-pinned-commit"

        .ci/scripts/setup-arm-baremetal-tools.sh --disable-ethos-u-deps --enable-mlsdk-deps --install-mlsdk-deps-with-pip

        # Increase number of files user can monitor to bypass buck failures.
        # Hopefully this is high enough for this setup.
        # Node-level, so an unprivileged pod cannot change it.
        sudo sysctl fs.inotify.max_user_watches=1048576 2>/dev/null || true

        ARM_TEST=${{ matrix.test_arm_backend }}

        # Output test report on pytest runs so that github can surface failing tests.
        if [[ -n "${RUNNER_TEST_RESULTS_DIR:-}" ]]; then
          export PYTEST_ADDOPTS="--junit-xml=${RUNNER_TEST_RESULTS_DIR}/${ARM_TEST}.xml ${PYTEST_ADDOPTS:-}"
        fi

        backends/arm/test/test_arm_backend.sh "${ARM_TEST}"

  test-coreml-delegate:
    needs: [changed-files, run-decision]
    # Path-filtered: see _ci-run-decision.yml for the sampling policy.
    if: |
      contains(needs.changed-files.outputs.changed-files, 'backends/apple/coreml') ||
      contains(needs.changed-files.outputs.changed-files, 'examples/apple/coreml') ||
      contains(needs.changed-files.outputs.changed-files, '.ci/scripts/setup-macos.sh') ||
      contains(needs.changed-files.outputs.changed-files, '.ci/scripts/setup-conda.sh') ||
      contains(needs.changed-files.outputs.changed-files, '.ci/docker/ci_commit_pins/pytorch.txt') ||
      contains(needs.changed-files.outputs.changed-files, 'torch_pin.py') ||
      contains(needs.changed-files.outputs.changed-files, '.github/workflows/trunk.yml') ||
      needs.run-decision.outputs.is-full-run == 'true'
    name: test-coreml-delegate
    uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
    with:
      default-packages: ""
      runner: macos-15-xlarge
      python-version: '3.11'
      submodules: 'recursive'
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      timeout: 90
      script: |
        set -eux
        # Pinned PyTorch's MPS build requires the Metal 4 SDK.
        export DEVELOPER_DIR=/Applications/Xcode_26.0.1.app/Contents/Developer
        xcodebuild -version

        BUILD_TOOL=cmake

        bash .ci/scripts/setup-conda.sh
        # Setup MacOS dependencies as there is no Docker support on MacOS atm
        GITHUB_RUNNER=1 PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh --build-tool "${BUILD_TOOL}"
        # Build and test coreml delegate
        PYTHON_EXECUTABLE=python ${CONDA_RUN} bash backends/apple/coreml/scripts/build_all.sh

  test-static-llama-ane:
    needs: [changed-files, run-decision]
    if: |
      contains(needs.changed-files.outputs.changed-files, 'backends/apple/coreml') ||
      contains(needs.changed-files.outputs.changed-files, 'examples/apple/coreml') ||
      contains(needs.changed-files.outputs.changed-files, 'examples/models/llama') ||
      contains(needs.changed-files.outputs.changed-files, 'extension/llm/export') ||
      contains(needs.changed-files.outputs.changed-files, 'extension/llm/tokenizers') ||
      contains(needs.changed-files.outputs.changed-files, '.ci/scripts/test_ane_static_llama.sh') ||
      contains(needs.changed-files.outputs.changed-files, '.ci/scripts/utils.sh') ||
      contains(needs.changed-files.outputs.changed-files, '.github/workflows/trunk.yml') ||
      needs.run-decision.outputs.is-full-run == 'true'
    name: test-static-llama-ane
    uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
    with:
      default-packages: ""
      runner: macos-m1-stable
      python-version: '3.11'
      submodules: 'recursive'
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      script: |
        set -eux
        bash .ci/scripts/setup-conda.sh
        eval "$(conda shell.bash hook)"

        # Install requirements
        ${CONDA_RUN} sh install_requirements.sh
        ${CONDA_RUN} sh backends/apple/coreml/scripts/install_requirements.sh
        ${CONDA_RUN} python install_executorch.py
        ${CONDA_RUN} sh examples/models/llama/install_requirements.sh

        # Test ANE llama
        ${CONDA_RUN} sh .ci/scripts/test_ane_static_llama.sh

  test-llama-torchao-lowbit:
    name: test-llama-torchao-lowbit
    needs: run-decision
    if: |
      github.event_name == 'pull_request' ||
      needs.run-decision.outputs.is-full-run == 'true'
    uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
    with:
      default-packages: ""
      runner: macos-m1-stable
      python-version: '3.11'
      submodules: 'recursive'
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      script: |
        set -eux
        bash .ci/scripts/setup-conda.sh
        eval "$(conda shell.bash hook)"

        # Install requirements
        ${CONDA_RUN} EXECUTORCH_BUILD_KERNELS_TORCHAO=1 python install_executorch.py
        ${CONDA_RUN} sh examples/models/llama/install_requirements.sh

        # Run test
        ${CONDA_RUN} sh .ci/scripts/test_llama_torchao_lowbit.sh

  test-llama-runner-linux:
    needs: docker-image
    # Test Both linux x86 and linux aarch64
    name: test-llama-runner-linux
    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
    permissions:
      id-token: write
      contents: read
    strategy:
      matrix:
        dtype: [fp32]
        mode: [portable, xnnpack+custom]
        runner: [mt-l-x86iavx512-8-64, mt-l-arm64g4-16-62]
        docker-image: [executorch-ubuntu-22.04-clang12, executorch-ubuntu-22.04-gcc11-aarch64]
        include:
          - dtype: bf16
            mode: portable
            runner: mt-l-x86iavx512-8-64
            docker-image: executorch-ubuntu-22.04-clang12
          - dtype: bf16
            mode: portable
            runner: mt-l-arm64g4-16-62
            docker-image: executorch-ubuntu-22.04-gcc11-aarch64
          - dtype: bf16
            mode: custom
            runner: mt-l-arm64g4-16-62
            docker-image: executorch-ubuntu-22.04-gcc11-aarch64
        # Excluding specific runner + docker image combinations that don't make sense:
        #   - Excluding the ARM64 gcc image on the x86 runner
        #   - Excluding the x86 clang image on the ARM64 runner
        exclude:
          - runner: mt-l-x86iavx512-8-64
            docker-image: executorch-ubuntu-22.04-gcc11-aarch64
          - runner: mt-l-arm64g4-16-62
            docker-image: executorch-ubuntu-22.04-clang12
      fail-fast: false
    with:
      runner: ${{ matrix.runner }}
      docker-image: ${{ needs.docker-image.outputs.docker-registry }}/ci-image:${{ matrix.docker-image }}-${{ needs.docker-image.outputs.ci-docker-hash }}
      submodules: 'recursive'
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      timeout: 900
      script: |
        # The generic Linux job chooses to use base env, not the one setup by the image
        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
        conda activate "${CONDA_ENV}"

        DTYPE=${{ matrix.dtype }}
        BUILD_TOOL="cmake"
        MODE=${{ matrix.mode }}
        ARTIFACTS_DIR_NAME="artifacts-to-be-uploaded/${DTYPE}-${MODE}"
        ARTIFACTS_DIR_NAME="${ARTIFACTS_DIR_NAME/+/-}"

        # Setup executorch
        PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}"
        # Install requirements for export_llama
        PYTHON_EXECUTABLE=python bash examples/models/llama/install_requirements.sh
        # Test llama2
        PYTHON_EXECUTABLE=python bash .ci/scripts/test_llama.sh -model stories110M -build_tool "${BUILD_TOOL}" -dtype "${DTYPE}" -mode "${MODE}" -upload "${ARTIFACTS_DIR_NAME}"

  test-llama-runner-macos:
    needs: [changed-files, run-decision]
    # Whole-job gate (matrix cells can't be individually if'd):
    # coreml / xnnpack+custom+quantize_kv.
    if: |
      contains(needs.changed-files.outputs.changed-files, 'backends/apple/coreml') ||
      contains(needs.changed-files.outputs.changed-files, 'backends/xnnpack') ||
      contains(needs.changed-files.outputs.changed-files, 'examples/models/llama') ||
      contains(needs.changed-files.outputs.changed-files, 'extension/llm/export') ||
      contains(needs.changed-files.outputs.changed-files, 'extension/llm/tokenizers') ||
      contains(needs.changed-files.outputs.changed-files, 'extension/llm/runner') ||
      contains(needs.changed-files.outputs.changed-files, 'extension/llm/custom_ops') ||
      contains(needs.changed-files.outputs.changed-files, 'extension/llm/sampler') ||
      contains(needs.changed-files.outputs.changed-files, '.ci/scripts/test_llama.sh') ||
      contains(needs.changed-files.outputs.changed-files, '.ci/scripts/setup-macos.sh') ||
      contains(needs.changed-files.outputs.changed-files, '.github/workflows/trunk.yml') ||
      needs.run-decision.outputs.is-full-run == 'true'
    name: test-llama-runner-mac
    uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
    strategy:
      matrix:
        dtype: [fp32]
        mode: [coreml, xnnpack+custom+quantize_kv]
      fail-fast: false
    with:
      default-packages: ""
      runner: macos-m1-stable
      python-version: '3.11'
      submodules: 'recursive'
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      timeout: 900
      script: |

        DTYPE=${{ matrix.dtype }}
        MODE=${{ matrix.mode }}

        bash .ci/scripts/setup-conda.sh

        # Setup executorch
        PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh --build-tool cmake

        if [[ "${MODE}" == "coreml" ]]; then
          # Install coreml delegate
          PYTHON_EXECUTABLE=python ${CONDA_RUN} bash backends/apple/coreml/scripts/install_requirements.sh
          echo "Finishing installing coreml."
        fi

        # Install requirements for export_llama
        PYTHON_EXECUTABLE=python ${CONDA_RUN} bash examples/models/llama/install_requirements.sh
        # Test llama2
        PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/test_llama.sh -model stories110M -build_tool cmake -dtype "${DTYPE}" -mode "${MODE}"

  test-torchao-huggingface-checkpoints:
    needs: docker-image
    name: test-torchao-huggingface-checkpoints
    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
    permissions:
      id-token: write
      contents: read
    strategy:
      matrix:
        model: [qwen3_4b, phi_4_mini, lfm2_5_1_2b]
        runner: [mt-l-x86iavx512-8-64]
        docker-image: [executorch-ubuntu-22.04-clang12]
        backend: [xnnpack]
        include:
          - model: qwen3_4b
            runner: mt-l-arm64g4-16-62
            docker-image: executorch-ubuntu-22.04-gcc11-aarch64
            backend: torchao
          - model: phi_4_mini
            runner: mt-l-arm64g4-16-62
            docker-image: executorch-ubuntu-22.04-gcc11-aarch64
            backend: torchao
          - model: lfm2_5_1_2b
            runner: mt-l-arm64g4-16-62
            docker-image: executorch-ubuntu-22.04-gcc11-aarch64
            backend: torchao
      fail-fast: false
    with:
      runner: ${{ matrix.runner }}
      docker-image: ${{ needs.docker-image.outputs.docker-registry }}/ci-image:${{ matrix.docker-image }}-${{ needs.docker-image.outputs.ci-docker-hash }}
      submodules: 'recursive'
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      timeout: 900
      script: |
        # The generic Linux job chooses to use base env, not the one setup by the image
        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
        conda activate "${CONDA_ENV}"

        PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool cmake

        if [[ "${{ matrix.backend }}" == "torchao" ]]; then
          BUILD_TORCHAO_EXPERIMENTAL=1 TORCHAO_BUILD_CPU_AARCH64=1 TORCHAO_BUILD_KLEIDIAI=1 TORCHAO_ENABLE_ARM_NEON_DOT=1 TORCHAO_PARALLEL_BACKEND=OPENMP pip install --no-build-isolation third-party/ao
        fi

        pip install -U "huggingface_hub[cli]<1.0"

        bash .ci/scripts/test_torchao_huggingface_checkpoints.sh ${{ matrix.model }} --test_with_runner ${{ matrix.backend == 'torchao' && '--use_torchao_kernels' || '' }}

  test-multimodal-macos:
    needs: run-decision
    if: |
      !github.event.pull_request.head.repo.fork &&
      (
        github.event_name == 'pull_request' ||
        needs.run-decision.outputs.is-full-run == 'true'
      )
    name: test-multimodal-macos
    uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
    permissions:
      id-token: write
      contents: read
    secrets: inherit
    strategy:
      fail-fast: false
      matrix:
        model: ["gemma3-4b"] # llava gives segfault so not covering.
    with:
      default-packages: ""
      secrets-env: EXECUTORCH_HF_TOKEN
      runner: macos-15-xlarge
      python-version: '3.11'
      submodules: 'recursive'
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      timeout: 90
      script: |
        echo "::group::Set up ExecuTorch"
        bash .ci/scripts/setup-conda.sh
        eval "$(conda shell.bash hook)"

        # Install requirements
        ${CONDA_RUN} python install_executorch.py
        echo "::endgroup::"

        echo "::group::Set up Huggingface"
        ${CONDA_RUN} pip install -U "huggingface_hub[cli]<1.0" accelerate
        ${CONDA_RUN} huggingface-cli login --token $SECRET_EXECUTORCH_HF_TOKEN
        OPTIMUM_ET_VERSION=$(cat .ci/docker/ci_commit_pins/optimum-executorch.txt)
        ${CONDA_RUN} pip install git+https://github.com/huggingface/optimum-executorch.git@${OPTIMUM_ET_VERSION}
        ${CONDA_RUN} pip list
        echo "::endgroup::"

        echo "::group::Test ${{ matrix.model }}"
        ${CONDA_RUN} python .ci/scripts/test_huggingface_optimum_model.py --model ${{ matrix.model }} --quantize --recipe xnnpack
        echo "::endgroup::"

  test-qnn-model:
    needs: docker-image
    name: test-qnn-model
    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
    permissions:
      id-token: write
      contents: read
    strategy:
      matrix:
        dtype: [fp32]
        model: [dl3, mv3, mv2, ic4, ic3, vit, mb, w2l, conv_former]
      fail-fast: false
    with:
      runner: mt-l-x86iavx512-8-64
      docker-image: ${{ needs.docker-image.outputs.docker-registry }}/ci-image:executorch-ubuntu-22.04-qnn-sdk-${{ needs.docker-image.outputs.ci-docker-hash }}
      submodules: 'recursive'
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      timeout: 900
      script: |
        # The generic Linux job chooses to use base env, not the one setup by the image
        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
        conda activate "${CONDA_ENV}"
        PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool cmake
        PYTHON_EXECUTABLE=python bash .ci/scripts/setup-qnn-deps.sh
        PYTHON_EXECUTABLE=python bash .ci/scripts/build-qnn-sdk.sh
        PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh ${{ matrix.model }} "cmake" "qnn"

  test-qnn-optimum-model:
    needs: docker-image
    name: test-qnn-optimum-model
    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
    permissions:
      id-token: write
      contents: read
    strategy:
      matrix:
        dtype: [fp32]
        model: [cvt, dit, efficientnet, focalnet, mobilevit_v1, mobilevit_v2, pvt, swin, albert, bert, distilbert, roberta] # eurobert requires transfomer >= 4.48.0, skip for now
      fail-fast: false
    with:
      runner: mt-l-x86iavx512-8-64
      docker-image: ${{ needs.docker-image.outputs.docker-registry }}/ci-image:executorch-ubuntu-22.04-qnn-sdk-${{ needs.docker-image.outputs.ci-docker-hash }}
      submodules: 'recursive'
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      timeout: 900
      script: |
        # The generic Linux job chooses to use base env, not the one setup by the image
        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
        conda activate "${CONDA_ENV}"
        PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool cmake
        PYTHON_EXECUTABLE=python bash .ci/scripts/setup-qnn-deps.sh
        PYTHON_EXECUTABLE=python bash .ci/scripts/build-qnn-sdk.sh
        PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh ${{ matrix.model }} "cmake" "qnn"

  test-models-macos-coreml:
    needs: [changed-files, run-decision]
    if: |
      contains(needs.changed-files.outputs.changed-files, 'backends/apple/coreml') ||
      contains(needs.changed-files.outputs.changed-files, 'examples/apple/coreml') ||
      contains(needs.changed-files.outputs.changed-files, 'examples/models') ||
      contains(needs.changed-files.outputs.changed-files, '.ci/scripts/test_model.sh') ||
      contains(needs.changed-files.outputs.changed-files, '.ci/scripts/setup-macos.sh') ||
      contains(needs.changed-files.outputs.changed-files, '.github/workflows/trunk.yml') ||
      needs.run-decision.outputs.is-full-run == 'true'
    name: test-models-macos-coreml
    uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
    strategy:
      matrix:
        model: [dl3, edsr, efficient_sam, emformer_join, emformer_transcribe, ic3, ic4, mobilebert, mv2, mv3, resnet50, vit, w2l]
      fail-fast: false
    with:
      default-packages: ""
      runner: macos-m1-stable
      python-version: '3.11'
      submodules: 'recursive'
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      timeout: 90
      script: |
        MODEL_NAME=${{ matrix.model }}
        BUILD_TOOL=cmake
        BACKEND="coreml-pybind"


        # Set model specific overrides
        if [[ "${MODEL_NAME}" == "mobilebert" ]]; then
          # See https://github.com/pytorch/executorch/issues/12907
          # mobilebert has nan output on FP16, and high MSE on fp32, so we disable runtime test now
          BACKEND="coreml"
        fi

        if [[ "${MODEL_NAME}" == "efficient_sam" ]]; then
          # See https://github.com/pytorch/executorch/issues/12906
          # efficient_sam fails to run on CoreML
          BACKEND="coreml"
        fi

        bash .ci/scripts/setup-conda.sh

        # Setup MacOS dependencies as there is no Docker support on MacOS atm
        PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh --build-tool "${BUILD_TOOL}"
        PYTHON_EXECUTABLE=python ${CONDA_RUN} bash backends/apple/coreml/scripts/install_requirements.sh
        echo "Finishing installing coreml."

        PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}"

  test-huggingface-transformers-xnnpack:
    needs: docker-image
    # NB: Don't run this on fork PRs because they won't have access to the secret and would fail anyway
    if: ${{ !github.event.pull_request.head.repo.fork }}
    name: test-huggingface-transformers-xnnpack
    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
    permissions:
      id-token: write
      contents: read
    secrets: inherit
    strategy:
      matrix:
        config: [
          # XNNPack.
          llama3.2-1b|xnnpack|--quantize,
          qwen3-0.6b|xnnpack|--quantize,
          qwen3-1.7b|xnnpack|--quantize,
          gemma3-1b|xnnpack|--quantize,
          # phi4-mini|xnnpack|--quantize, transformers v5.0.0rc0 introduces a data-dependent branching in transformers/modeling_rope_utils.py:61
          smollm2-135m|xnnpack|--quantize,
          smollm3-3b|xnnpack|--quantize
        ]
      fail-fast: false
    with:
      secrets-env: EXECUTORCH_HF_TOKEN
      runner: mt-l-x86iavx512-8-64
      docker-image: ${{ needs.docker-image.outputs.docker-registry }}/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }}
      submodules: 'recursive'
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      timeout: 90
      upload-artifact: profiling-artifacts-${{ strategy.job-index }}
      script: |
        set -eux
        IFS='|' read -r MODEL RECIPE QUANTIZE <<< "${{ matrix.config }}"
        echo "Model: $MODEL"
        echo "Recipe: $RECIPE"
        echo "Quantize: $QUANTIZE"

        # The generic Linux job chooses to use base env, not the one setup by the image
        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
        conda activate "${CONDA_ENV}"

        echo "::group::Setup ExecuTorch"
        PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "cmake"
        echo "::endgroup::"

        echo "::group::Setup Huggingface"
        pip install -U "huggingface_hub[cli]<1.0" accelerate
        huggingface-cli login --token $SECRET_EXECUTORCH_HF_TOKEN
        OPTIMUM_ET_VERSION=$(cat .ci/docker/ci_commit_pins/optimum-executorch.txt)
        pip install git+https://github.com/huggingface/optimum-executorch.git@${OPTIMUM_ET_VERSION}
        echo "::endgroup::"

        echo "::group::Test MODEL: $MODEL RECIPE: $RECIPE QUANTIZE: $QUANTIZE"
        export OUTPUT_DIR="$(pwd)/${MODEL}_${RECIPE}_${QUANTIZE}"
        python .ci/scripts/test_huggingface_optimum_model.py --model "$MODEL" --recipe "$RECIPE" $QUANTIZE --model_dir "$OUTPUT_DIR"
        echo "::endgroup::"

        # Build executor_runner with ETdump enabled
        PYTHON_EXECUTABLE=python cmake -DPYTHON_EXECUTABLE=python \
          -DCMAKE_INSTALL_PREFIX=cmake-out \
          -DEXECUTORCH_ENABLE_LOGGING=1 \
          -DCMAKE_BUILD_TYPE=Release \
          -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
          -DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
          -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
          -DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \
          -DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
          -DEXECUTORCH_BUILD_XNNPACK=ON \
          -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \
          -DEXECUTORCH_BUILD_KERNELS_OPTIMIZED=ON \
          -DEXECUTORCH_BUILD_KERNELS_LLM=ON \
          -DEXECUTORCH_BUILD_DEVTOOLS=ON \
          -DEXECUTORCH_ENABLE_EVENT_TRACER=ON \
          -Bcmake-out .
        cmake --build cmake-out -j16 --target install --config Release

        echo "::group::Generate artifacts for performance profiling"
        ./cmake-out/executor_runner \
          --model_path ${OUTPUT_DIR}/model.pte \
          --etdump_path ${OUTPUT_DIR}/etdump.etdp

        export TSV_PATH=artifacts-to-be-uploaded/${MODEL}_op_prof.tsv
        mkdir -p $(dirname "$TSV_PATH")
        python3 -m devtools.inspector.inspector_cli \
          --etdump_path ${OUTPUT_DIR}/etdump.etdp \
          --tsv_path ${TSV_PATH}
        echo "::endgroup::"

  test-huggingface-transformers-macos:
    needs: [changed-files, run-decision]
    # NB: Don't run this on fork PRs because they won't have access to the secret and would fail anyway
    if: |
      !github.event.pull_request.head.repo.fork &&
      (
        contains(needs.changed-files.outputs.changed-files, 'backends/apple/coreml') ||
        contains(needs.changed-files.outputs.changed-files, 'extension/llm/runner') ||
        contains(needs.changed-files.outputs.changed-files, '.ci/scripts/test_huggingface_optimum_model.py') ||
        contains(needs.changed-files.outputs.changed-files, '.ci/docker/ci_commit_pins/optimum-executorch.txt') ||
        contains(needs.changed-files.outputs.changed-files, 'install_executorch.py') ||
        contains(needs.changed-files.outputs.changed-files, 'install_requirements.py') ||
        contains(needs.changed-files.outputs.changed-files, '.github/workflows/trunk.yml') ||
        needs.run-decision.outputs.is-full-run == 'true'
      )
    name: test-huggingface-transformers-macos
    uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
    permissions:
      id-token: write
      contents: read
    secrets: inherit
    # Models below selected based on https://huggingface.co/models?pipeline_tag=text-generation&num_parameters=min:0,max:3B&sort=trending.
    strategy:
      matrix:
        config: [
          # # XNNPack. (Skipping for now due to intermittent segmentation faults, see https://github.com/huggingface/optimum-executorch/issues/122.)
          # llama3.2-1b|xnnpack|--quantize,
          # qwen3-0.6b|xnnpack|--quantize,
          # qwen3-1.7b|xnnpack|--quantize,
          # gemma3-1b|xnnpack|--quantize,
          # phi4-mini|xnnpack|--quantize,
          # smollm2-135m|xnnpack|--quantize,
          # smollm3-3b|xnnpack|--quantize,
          # qwen3-1.7b|xnnpack|--quantize,
          # CoreML.
          llama3.2-1b|coreml_fp32_gpu|--quantize,
          qwen3-0.6b|coreml_fp32_gpu|--quantize,
          smollm2-135m|coreml_fp32_gpu|--quantize,
          olmo-1b|coreml_fp32_gpu|--quantize,
          bert|coreml_fp32_gpu|--quantize,
          distilbert|coreml_fp32_gpu|--quantize
        ]
      fail-fast: false
    with:
      default-packages: ""
      secrets-env: EXECUTORCH_HF_TOKEN
      runner: macos-15-xlarge
      python-version: '3.11'
      submodules: 'recursive'
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      timeout: 90
      script: |
        set -eux
        IFS='|' read -r MODEL RECIPE QUANTIZE <<< "${{ matrix.config }}"
        echo "Model: $MODEL"
        echo "Recipe: $RECIPE"
        echo "Quantize: $QUANTIZE"

        echo "::group::Set up ExecuTorch"
        bash .ci/scripts/setup-conda.sh
        eval "$(conda shell.bash hook)"

        # Install requirements
        ${CONDA_RUN} python install_executorch.py
        echo "::endgroup::"

        echo "::group::Set up Huggingface"
        ${CONDA_RUN} pip install -U "huggingface_hub[cli]<1.0" accelerate
        ${CONDA_RUN} huggingface-cli login --token $SECRET_EXECUTORCH_HF_TOKEN
        OPTIMUM_ET_VERSION=$(cat .ci/docker/ci_commit_pins/optimum-executorch.txt)
        ${CONDA_RUN} pip install git+https://github.com/huggingface/optimum-executorch.git@${OPTIMUM_ET_VERSION}
        ${CONDA_RUN} pip list
        echo "::endgroup::"

        # Run test
        ${CONDA_RUN} python .ci/scripts/test_huggingface_optimum_model.py --model ${MODEL} --recipe ${RECIPE} ${QUANTIZE}

  test-llama-runner-qnn-linux:
    needs: docker-image
    name: test-llama-runner-qnn-linux
    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
    permissions:
      id-token: write
      contents: read
    strategy:
      matrix:
        dtype: [fp32]
        pt2e_quantize: [qnn_16a16w, qnn_8a8w]
        mode: [qnn]
      fail-fast: false
    with:
      runner: mt-l-x86iavx512-8-64
      docker-image: ${{ needs.docker-image.outputs.docker-registry }}/ci-image:executorch-ubuntu-22.04-qnn-sdk-${{ needs.docker-image.outputs.ci-docker-hash }}
      submodules: 'recursive'
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      timeout: 900
      script: |
        # The generic Linux job chooses to use base env, not the one setup by the image
        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
        conda activate "${CONDA_ENV}"

        BUILD_TOOL="cmake"
        DTYPE=${{ matrix.dtype }}
        MODE=${{ matrix.mode }}
        PT2E_QUANTIZE=${{ matrix.pt2e_quantize }}

        ./install_requirements.sh --use-pt-pinned-commit
        PYTHON_EXECUTABLE=python bash .ci/scripts/setup-qnn-deps.sh
        PYTHON_EXECUTABLE=python bash .ci/scripts/build-qnn-sdk.sh

        # Setup executorch
        PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}"
        # Install requirements for export_llama
        PYTHON_EXECUTABLE=python bash examples/models/llama/install_requirements.sh
        # Test llama2
        PYTHON_EXECUTABLE=python bash .ci/scripts/test_llama.sh -model stories110M -build_tool "${BUILD_TOOL}" -mode "${MODE}" -dtype "${DTYPE}" -pt2e_quantize "${PT2E_QUANTIZE}"

  # this is for filtering out the qnn changes such that qnn jobs only triggered when the specific files are changed
  changes:
    runs-on: ubuntu-latest
    outputs:
      qnn: ${{ steps.filter.outputs.qnn }}
    steps:
      - uses: actions/checkout@v4
      - uses: dorny/paths-filter@v3
        id: filter
        with:
          filters: |
            qnn:
              - 'backends/qualcomm/**'
              - 'examples/qualcomm/**'
              - 'examples/models/llama/**'

  unittest-release:
    uses: ./.github/workflows/_unittest.yml
    permissions:
      actions: read
      id-token: write
      contents: read
    with:
      upload-test-run-reports: true
      build-mode: Release
      build-tool: cmake
      docker-image: ci-image:executorch-ubuntu-22.04-clang12
      linux-timeout: 120

  llm-server:
    needs: [changed-files, run-decision]
    if: |
      github.event_name != 'pull_request' && (
        contains(needs.changed-files.outputs.changed-files, 'examples/llm_server') ||
        contains(needs.changed-files.outputs.changed-files, 'examples/models/muse-glimmer/serving') ||
        contains(needs.changed-files.outputs.changed-files, 'examples/models/muse-glimmer/tests') ||
        contains(needs.changed-files.outputs.changed-files, 'examples/models/muse_glimmer') ||
        contains(needs.changed-files.outputs.changed-files, 'extension/llm/batching') ||
        contains(needs.changed-files.outputs.changed-files, 'extension/llm/serving') ||
        contains(needs.changed-files.outputs.changed-files, 'extension/llm/runner') ||
        contains(needs.changed-files.outputs.changed-files, 'extension/llm/tokenizers') ||
        contains(needs.changed-files.outputs.changed-files, '.github/workflows/_get-changed-files.yml') ||
        contains(needs.changed-files.outputs.changed-files, '.github/workflows/_ci-run-decision.yml') ||
        contains(needs.changed-files.outputs.changed-files, '.github/workflows/_llm_server.yml') ||
        contains(needs.changed-files.outputs.changed-files, '.github/workflows/pull.yml') ||
        contains(needs.changed-files.outputs.changed-files, '.github/workflows/trunk.yml') ||
        needs.run-decision.outputs.is-full-run == 'true'
      )
    uses: ./.github/workflows/_llm_server.yml
    permissions:
      id-token: write
      contents: read

  test-models-windows:
    needs: run-decision
    if: |
      github.event_name == 'pull_request' ||
      needs.run-decision.outputs.is-full-run == 'true'
    uses: pytorch/test-infra/.github/workflows/windows_job.yml@main
    strategy:
      fail-fast: false
      matrix:
        model: [mv3, resnet50, vit, mobilebert, emformer_transcribe]
        backend: [portable, xnnpack-q8]
    with:
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      timeout: 60
      script: |
        git config --global http.sslBackend openssl
        git submodule update --init --recursive
        conda init powershell

        powershell -Command "& {
          Set-PSDebug -Trace 1
          \$ErrorActionPreference = 'Stop'
          \$PSNativeCommandUseErrorActionPreference = \$true

          .ci/scripts/setup-windows.ps1

          .ci/scripts/test_model.ps1 -modelName ${{ matrix.model }} -backend ${{ matrix.backend }}
        }"

  test-cortex-m-e2e:
    name: test-cortex-m-e2e
    permissions:
      id-token: write
      contents: read
    uses: ./.github/workflows/_test_cortex_m_e2e.yml
    with:
      models: '["mv2", "mv3"]'
      targets: '["cortex-m55", "cortex-m7", "cortex-m0plus"]'

  test-cortex-m-ops:
    name: test-cortex-m-ops
    permissions:
      id-token: write
      contents: read
    uses: ./.github/workflows/_test_cortex_m_ops.yml
    with:
      targets: '["cortex-m7", "cortex-m0plus"]'

  test-samsung-quantmodels-linux:
    needs: docker-image
    name: test-samsung-quantmodels-linux
    # Skip this job if the pull request is from a fork (secrets are not available)
    if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name != 'pull_request'
    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
    permissions:
      id-token: write
      contents: read
    secrets: inherit
    with:
      secrets-env: SAMSUNG_AI_LITECORE_KEY
      runner: mt-l-x86iavx512-8-64
      docker-image: ${{ needs.docker-image.outputs.docker-registry }}/ci-image:executorch-ubuntu-22.04-clang12-android-${{ needs.docker-image.outputs.ci-docker-hash }}
      submodules: 'recursive'
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      timeout: 180
      script: |
        set -ex

        # The generic Linux job chooses to use base env, not the one setup by the image
        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
        conda activate "${CONDA_ENV}"

        # Setup python
        PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "cmake"

        # Setup Samsung SDK (AI Lite Core) and install enn backend
        export SAMSUNG_AI_LITECORE_KEY=$SECRET_SAMSUNG_AI_LITECORE_KEY

        trap 'bash .ci/scripts/cleanup-samsung-linux-deps.sh' EXIT TERM INT
        source .ci/scripts/setup-samsung-linux-deps.sh --skip-device-connect

        # Test quant models
        model_scripts="deeplab_v3 edsr inception_v3 inception_v4 mobilenet_v2 mobilenet_v3 resnet18 resnet50 vit wav2letter"
        for m_script in $model_scripts; do
          python -m executorch.examples.samsung.scripts.${m_script} -c e9955 -p A8W8
        done

  test-samsung-models-linux:
    needs: docker-image
    name: test-samsung-models-linux
    # Skip this job if the pull request is from a fork (secrets are not available)
    if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name != 'pull_request'
    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
    permissions:
      id-token: write
      contents: read
    secrets: inherit
    with:
      secrets-env: SAMSUNG_AI_LITECORE_KEY
      runner: mt-l-x86iavx512-8-64
      docker-image: ${{ needs.docker-image.outputs.docker-registry }}/ci-image:executorch-ubuntu-22.04-clang12-android-${{ needs.docker-image.outputs.ci-docker-hash }}
      submodules: 'recursive'
      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
      timeout: 360
      script: |
        set -ex

        # The generic Linux job chooses to use base env, not the one setup by the image
        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
        conda activate "${CONDA_ENV}"

        # Setup python
        PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "cmake"

        # Setup Samsung SDK (AI Lite Core) and install enn backend
        export SAMSUNG_AI_LITECORE_KEY=$SECRET_SAMSUNG_AI_LITECORE_KEY
        trap 'bash .ci/scripts/cleanup-samsung-linux-deps.sh' EXIT TERM INT
        source .ci/scripts/setup-samsung-linux-deps.sh

        # Graph-level pass unit tests (no device needed)
        python -m unittest discover -s backends/samsung/test/passes -p "test_*.py"

        # Check if device was reserved
        if [[ "${DEVICE_ACQUIRED:-0}" != "1" ]]; then
          echo "::warning::Skipping tests - no Samsung device available"
          exit 0
        fi

        # Test models
        # TODO: Switch to run_tests.py once CI device provisioning is stable
        # python -m executorch.backends.samsung.test.utils.run_tests --chipset E9955
        # accelerate is for test_mobilebert_qat, whose TrainingArguments needs it;
        # transformers reports the same error when it is absent and when it is too old.
        pip install evaluate 'accelerate>=1.1.0'
        pip install --upgrade-strategy only-if-needed \
          --extra-index-url https://download.pytorch.org/whl/cpu \
          -r examples/models/yolo26/requirements.txt
        python -m unittest discover -s backends/samsung/test/models -p "test_*.py"