Skip to content
Core AI

Import torch only when the Python runtime needs it, warn before outputs follow inputs, and upload bfloat16 without torch #1155

Sign in to view logs

Import torch only when the Python runtime needs it, warn before outputs follow inputs, and upload bfloat16 without torch

Import torch only when the Python runtime needs it, warn before outputs follow inputs, and upload bfloat16 without torch #1155

Workflow file for this run

.github/workflows/coreai.yml at c3eb82b
name: Core AI
on:
push:
branches:
- main
- release/*
tags:
- ciflow/coreai/*
pull_request:
paths:
- .github/workflows/coreai.yml
- .clang-format
- CMakeLists.txt
- backends/apple/coreai/**
- backends/test/**
- extension/module/**
- extension/pybindings/**
- extension/tensor/**
- setup.py
- runtime/**
- tools/cmake/**
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
cancel-in-progress: true
permissions:
contents: read
jobs:
run-decision:
name: CI run decision
uses: ./.github/workflows/_ci-run-decision.yml
test-coreai:
needs: run-decision
if: |
github.event_name == 'pull_request' ||
needs.run-decision.outputs.is-full-run == 'true'
uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
with:
default-packages: ""
job-name: test-coreai
# GitHub-hosted macOS 27 with Xcode 27 (public preview).
runner: xcode-27
# coreai-opt supports Python < 3.14.
python-version: "3.12"
submodules: recursive
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
timeout: 90
script: |
set -eux
echo "::group::Toolchain"
sw_vers
xcodebuild -version
xcrun --show-sdk-version
echo "::endgroup::"
echo "::group::Install the Metal Toolchain"
# coreai-build ships in the Metal Toolchain; without it the AOT compile
# tests skip.
xcodebuild -downloadComponent MetalToolchain
xcrun --find coreai-build
echo "::endgroup::"
echo "::group::Install ExecuTorch and Core AI requirements"
${CONDA_RUN} python install_executorch.py --editable > /dev/null
${CONDA_RUN} pip install -r backends/apple/coreai/requirements.txt "cmake>=3.31,<4" ninja clang-format==18.1.3 pytest-xdist
${CONDA_RUN} pip list
echo "::endgroup::"
echo "::group::Check Core AI formatting"
${CONDA_RUN} python - <<'PY'
from pathlib import Path
import subprocess
import yaml
root = yaml.safe_load(Path(".clang-format").read_text())
coreai = yaml.safe_load(Path("backends/apple/coreai/.clang-format").read_text())
root.pop("Language")
assert coreai == root, "Sync the Core AI clang-format configuration with the root style (without Language)"
sources = sorted(Path("backends/apple/coreai").rglob("*.mm"))
subprocess.run(["clang-format", "--dry-run", "--Werror", *map(str, sources)], check=True)
PY
echo "::endgroup::"
echo "::group::Run Core AI export tests"
${CONDA_RUN} python -m pytest backends/apple/coreai -n auto
echo "::endgroup::"
echo "::group::Build and run Core AI runtime tests"
${CONDA_RUN} cmake -S . -B cmake-out-coreai -G Ninja \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_OSX_DEPLOYMENT_TARGET=27.0 \
-DEXECUTORCH_BUILD_COREAI=ON \
-DEXECUTORCH_BUILD_TESTS=ON \
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
-DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
-DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \
-DEXECUTORCH_BUILD_PORTABLE_OPS=ON
${CONDA_RUN} cmake --build cmake-out-coreai --target backends/apple/coreai/all -j$(sysctl -n hw.ncpu)
COREAI_STRESS_THREADS=50 COREAI_STRESS_ITERATIONS=100 \
${CONDA_RUN} ctest --test-dir cmake-out-coreai/backends/apple/coreai --output-on-failure --no-tests=error
echo "::endgroup::"
backend-tester:
needs: run-decision
if: |
github.event_name == 'pull_request' ||
needs.run-decision.outputs.is-full-run == 'true'
strategy:
fail-fast: false
max-parallel: 8
matrix:
suite: [models, operators]
flow: [coreai_fp32, coreai_fp16, coreai_w8a8, coreai_w8]
uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
with:
default-packages: ""
job-name: test-${{ matrix.flow }}-${{ matrix.suite }}
runner: xcode-27
python-version: "3.12"
submodules: recursive
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
timeout: 120
script: |
set -euxo pipefail
echo "::group::Install the Core AI toolchain and Python runtime"
xcodebuild -version
xcodebuild -downloadComponent MetalToolchain
xcrun --find coreai-build
export CMAKE_GENERATOR=Ninja
export CMAKE_ARGS="-DEXECUTORCH_BUILD_COREAI=ON -DEXECUTORCH_BUILD_COREML=OFF -DEXECUTORCH_BUILD_MLX=OFF -DCMAKE_OSX_DEPLOYMENT_TARGET=27.0"
${CONDA_RUN} pip install "cmake>=3.31,<4" ninja
${CONDA_RUN} python install_executorch.py --editable > /dev/null
${CONDA_RUN} pip install -r backends/apple/coreai/requirements.txt pytest-xdist pytest-json-report pytest-timeout
${CONDA_RUN} pip list
${CONDA_RUN} python - <<'PY'
from executorch.extension.pybindings.portable_lib import _get_registered_backend_names, _is_available
assert "CoreAIBackend" in _get_registered_backend_names()
assert _is_available("CoreAIBackend")
PY
echo "::endgroup::"
echo "::group::Run shared backend suite (${{ matrix.flow }} / ${{ matrix.suite }})"
export GOLDEN_ARTIFACTS_DIR="${RUNNER_TEMP}/coreai-${{ matrix.flow }}-${{ matrix.suite }}"
# Allowed failures for gating suites; unlisted suites are informational.
case "${{ matrix.flow }}/${{ matrix.suite }}" in
coreai_fp32/operators | coreai_fp16/operators | coreai_w8/operators | \
coreai_fp32/models | coreai_fp16/models | coreai_w8/models) MAX_FAILED=0 ;;
# Known failures; lower this as they're fixed.
coreai_w8a8/operators) MAX_FAILED=23 ;;
# 7 fail locally (accuracy). On the xcode-27 runners, 19 w8a8
# dynamic_shapes models also fail execution with Core AI
# inferenceFailed(-1) (3 overlap the local 7), while their static
# variants and fp32 dynamic_shapes pass. TODO: report to Apple and
# lower back to 7.
coreai_w8a8/models) MAX_FAILED=23 ;;
*) MAX_FAILED="" ;;
esac
FAIL_FAST=""
if [[ "${MAX_FAILED}" == "0" ]]; then FAIL_FAST="-x"; fi
status=0
${CONDA_RUN} python -m pytest -c /dev/null backends/test/suite/${{ matrix.suite }}/ \
-m flow_${{ matrix.flow }} -n auto ${FAIL_FAST} \
--json-report --json-report-file=coreai-results.json \
--junitxml=coreai-results.xml 2>&1 | tee coreai-pytest.log || status=$?
echo "::endgroup::"
if [[ -z "${MAX_FAILED}" ]]; then
if [[ ${status} -ne 0 ]]; then
echo "::warning::${{ matrix.flow }} / ${{ matrix.suite }} is not gating; pytest exited with ${status}"
fi
exit 0
fi
# 1 means tests failed (checked against MAX_FAILED below); anything else
# is a pytest crash, interrupt or usage error.
if [[ ${status} -ne 0 && ${status} -ne 1 ]]; then exit ${status}; fi
echo "::group::Check ${{ matrix.flow }} / ${{ matrix.suite }} results"
export MAX_FAILED
${CONDA_RUN} python - <<'PY'
import json
import os
summary = json.load(open("coreai-results.json"))["summary"]
max_failed = int(os.environ["MAX_FAILED"])
print(summary)
failed = summary.get("failed", 0)
assert failed <= max_failed, f"{failed} tests failed; at most {max_failed} allowed"
assert summary.get("error", 0) == 0, "tests errored"
assert summary.get("passed", 0) > 0, "no tests passed"
PY
echo "::endgroup::"