Merge branch 'main' into delete-exir-capture

Author: krisnaparahita

.github/workflows/_llm_server.yml

@@ -44,7 +44,9 @@ jobs:
 
         python -m pytest -q \
           examples/llm_server/python/tests \
-          examples/models/muse-glimmer/tests/test_serve.py
+          examples/models/muse-glimmer/tests/test_serve.py \
+          examples/models/muse-glimmer/tests/test_serve_batching.py \
+          examples/models/muse-glimmer/tests/test_stream_parser.py
 
         cmake -S . -B cmake-out \
           -DCMAKE_BUILD_TYPE=Release \

.github/workflows/_test_arduino_library.yml

@@ -47,6 +47,9 @@ jobs:
         cd examples/arduino
         ./build_arduino_library.sh
 
+        test "$(sed -n 's/^version=//p' arduino_lib/ExecuTorch/library.properties)" \
+          = "$(cat ../../version.txt)"
+
         # A model exported against a different ExecuTorch commit than the one
         # that built the library loads fine, resolves every operator, and then
         # fails inside Method::execute. Catch that here rather than on a board.

.github/workflows/cuda.yml

@@ -535,8 +535,15 @@ jobs:
         # Build executor_runner (needed by CUDA backend e2e tests)
         cmake --build cmake-out --target executor_runner
 
-        # Run CUDA backend Python tests
-        python -m pytest backends/cuda/tests backends/cuda/passes/tests -v -o "addopts="
+        # Run CUDA backend Python tests. The quantized GEMM kernel suites run
+        # in unittest-cuda-quantized-gemm; together they do not fit this
+        # job's timeout.
+        python -m pytest backends/cuda/tests backends/cuda/passes/tests -v -o "addopts=" \
+          --ignore=backends/cuda/tests/test_int4_quantized_gemm.py \
+          --ignore=backends/cuda/tests/test_int5_quantized_gemm.py \
+          --ignore=backends/cuda/tests/test_int6_quantized_gemm.py \
+          --ignore=backends/cuda/tests/test_int8_quantized_gemm.py \
+          --ignore=backends/cuda/tests/test_cuda_graph_timing.py
 
         # Run Qwen 3.5 MoE tests (quantize roundtrip + TurboQuant KV cache + sampler)
         python -m pytest examples/models/qwen3_5_moe/test_quantize_roundtrip.py examples/models/qwen3_5_moe/test_turboquant.py examples/models/qwen3_5_moe/test_sampler.py -v -o "addopts="
@@ -545,6 +552,45 @@ jobs:
         pip install gguf
         python -m pytest examples/models/gemma4_31b/tests/ --ignore=examples/models/gemma4_31b/tests/test_mlx_pipeline.py -v -o "addopts="
 
+  unittest-cuda-quantized-gemm:
+    name: unittest-cuda-quantized-gemm (${{ matrix.shard }})
+    needs: [changed-files, run-decision]
+    if: |
+      contains(needs.changed-files.outputs.changed-files, 'backends/cuda') ||
+      contains(needs.changed-files.outputs.changed-files, 'backends/aoti') ||
+      contains(needs.changed-files.outputs.changed-files, '.github/workflows/cuda.yml') ||
+      needs.run-decision.outputs.is-full-run == 'true'
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - shard: int4
+            tests: backends/cuda/tests/test_int4_quantized_gemm.py
+          - shard: int5-int6-int8-graph-timing
+            tests: >-
+              backends/cuda/tests/test_int5_quantized_gemm.py
+              backends/cuda/tests/test_int6_quantized_gemm.py
+              backends/cuda/tests/test_int8_quantized_gemm.py
+              backends/cuda/tests/test_cuda_graph_timing.py
+    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
+    permissions:
+      id-token: write
+      contents: read
+    with:
+      timeout: 90
+      runner: mt-l-x86aavx2-11-41-a10g
+      gpu-arch-type: cuda
+      gpu-arch-version: "13.0"
+      submodules: recursive
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      script: |
+        set -eux
+        bash ./install_executorch.sh
+        conda install -y -c conda-forge 'libstdcxx-ng>=12'
+        export LD_LIBRARY_PATH=/opt/conda/lib:$LD_LIBRARY_PATH
+
+        python -m pytest ${{ matrix.tests }} -v -o "addopts=" --durations=20
+
   unittest-cuda-runtime:
     name: unittest-cuda-runtime
     needs: [changed-files, run-decision]

.github/workflows/mlx.yml

@@ -10,11 +10,16 @@ on:
   pull_request:
     paths:
       - .github/workflows/mlx.yml
+      - Makefile
       - backends/mlx/**
       - extension/llm/batching/**
       - extension/llm/cache/**
       - extension/llm/export/**
+      - extension/llm/runner/**
+      - extension/llm/sampler/**
+      - extension/llm/serving/**
       - extension/audio/**
+      - examples/llm_server/cpp/**
       - examples/models/gemma4_31b/**
       - examples/models/muse-glimmer/**
       - examples/models/nemotron3_diarization/**
@@ -128,6 +133,8 @@ jobs:
           backends/mlx/test/test_sample.py \
           examples/models/gemma4_31b/tests/test_mlx_pipeline.py \
           examples/models/muse-glimmer/tests/test_mlx_pipeline.py \
+          examples/models/muse-glimmer/tests/test_mlx_offgraph.py \
+          examples/models/muse-glimmer/tests/test_checkpoint_context.py \
           -v
         echo "::endgroup::"
 
@@ -309,9 +316,8 @@ jobs:
         echo "Success: built $RUNNER"
         echo "::endgroup::"
 
-  # Build-only: the Muse Glimmer GGUF checkpoints are far too large to export in
-  # CI, so this covers the compile / link / metallib wiring of the runners --
-  # which is what MLX backend changes are most likely to break.
+  # No full-model export: Muse Glimmer GGUF checkpoints are too large for CI.
+  # Cover runner compile/link/metallib wiring and asset-independent preparation.
   test-mlx-muse-glimmer-build:
     needs: run-decision
     if: |
@@ -355,6 +361,19 @@ jobs:
         echo "Success: built solo_runner, dflash_runner and muse_glimmer_worker"
         echo "::endgroup::"
 
+        echo "::group::Build and test Muse Glimmer batching with MLX"
+        ${CONDA_RUN} make muse-glimmer-batching-mlx
+        BATCHING_BIN_DIR="${PWD}/cmake-out/examples/models/muse-glimmer-batching"
+        ${CONDA_RUN} cmake -S "${PWD}/examples/models/muse-glimmer" -B "${BATCHING_BIN_DIR}" \
+          -DMUSE_GLIMMER_BUILD_BATCHING_TESTS=ON
+        ${CONDA_RUN} cmake --build "${BATCHING_BIN_DIR}" --target muse_glimmer_batching_test --parallel
+        ${CONDA_RUN} ctest --test-dir "${BATCHING_BIN_DIR}" \
+          -R '^muse_glimmer_batching$' --output-on-failure --no-tests=error
+        test -x "${BATCHING_BIN_DIR}/muse_glimmer_batching_runner"
+        test -x "${BATCHING_BIN_DIR}/muse_glimmer_batching_worker"
+        test -f "${BATCHING_BIN_DIR}/mlx.metallib"
+        echo "::endgroup::"
+
   test-mlx-nemotron3-diarization-build:
     needs: run-decision
     if: |

.github/workflows/pull.yml

@@ -74,7 +74,8 @@ jobs:
       contains(needs.changed-files.outputs.changed-files, 'examples/arduino/') ||
       contains(needs.changed-files.outputs.changed-files, 'kernels/portable/') ||
       contains(needs.changed-files.outputs.changed-files, 'runtime/') ||
-      contains(needs.changed-files.outputs.changed-files, 'schema/')
+      contains(needs.changed-files.outputs.changed-files, 'schema/') ||
+      contains(needs.changed-files.outputs.changed-files, 'version.txt')
     name: test-arduino-library
     uses: ./.github/workflows/_test_arduino_library.yml
     permissions:

.github/workflows/rocm.yml

@@ -175,7 +175,10 @@ jobs:
       id-token: write
       contents: read
     with:
-      timeout: 90
+      # Since #23603 the quantized GEMM kernels compile and run here instead of
+      # failing fast; the job takes ~80 min, ~37 of them Int4AutotunePickTest's
+      # CPU-saturated AOTI compile.
+      timeout: 150
       no-sudo: true
       runner: linux.rocm.gpu.gfx950.1
       gpu-arch-type: rocm

.github/workflows/vulkan.yml

@@ -110,7 +110,7 @@ jobs:
         ./cmake-out/backends/vulkan/test/custom_ops/test_q8ta_clone
         ./cmake-out/backends/vulkan/test/custom_ops/test_q8ta_binary
 
-        PYTHON_EXECUTABLE=python bash backends/vulkan/test/scripts/test_op.sh --build
+        PYTHON_EXECUTABLE=python bash backends/vulkan/test/scripts/test_op.sh --build vulkan_sdpa_av_test
 
         # Run e2e testing for selected operators.
         python -m unittest backends/vulkan/test/test_vulkan_delegate.py -k "*pt2e*"

Makefile

@@ -139,6 +139,7 @@ help:
 	@echo "  gemma4_31b-mlx      - Build Gemma 4 31B runner and worker with MLX backend"
 	@echo "  muse-glimmer-cuda   - Build Muse Glimmer runners and worker with CUDA backend"
 	@echo "  muse-glimmer-mlx    - Build Muse Glimmer runners and worker with MLX backend"
+	@echo "  muse-glimmer-batching-mlx - Build batching Muse Glimmer runner and worker with MLX"
 	@echo "  qwen3_5_moe-cuda    - Build Qwen3.5 MoE runner with CUDA backend"
 	@echo "  qwen3_5_moe-metal   - Build Qwen3.5 MoE runner with Metal backend"
 	@echo "  qwen3_5_moe-mlx     - Build Qwen3.5 MoE runner with MLX backend"
@@ -513,6 +514,24 @@ muse-glimmer-mlx:
 	@echo "  DFlash runner: cmake-out/examples/models/muse-glimmer/dflash_runner"
 	@echo "  Worker:        cmake-out/examples/models/muse-glimmer/muse_glimmer_worker"
 
+.PHONY: muse-glimmer-batching-mlx
+muse-glimmer-batching-mlx:
+	@echo "==> Building and installing ExecuTorch with MLX..."
+	cmake --workflow --preset mlx-release
+	@echo "==> Configuring batching Muse Glimmer runners with MLX..."
+	cd examples/models/muse-glimmer && cmake --preset muse-glimmer-mlx \
+		-B "$(CURDIR)/cmake-out/examples/models/muse-glimmer-batching" \
+		-DEXECUTORCH_BUILD_CUDA=OFF \
+		-DMUSE_GLIMMER_BUILD_BATCHING=ON \
+		-DMUSE_GLIMMER_BUILD_BATCHING_TESTS=OFF
+	cmake --build "$(CURDIR)/cmake-out/examples/models/muse-glimmer-batching" \
+		--target muse_glimmer_batching_runner muse_glimmer_batching_worker \
+		--parallel "$$(sysctl -n hw.ncpu)"
+	@echo ""
+	@echo "Build complete!"
+	@echo "  Runner:        $(CURDIR)/cmake-out/examples/models/muse-glimmer-batching/muse_glimmer_batching_runner"
+	@echo "  Worker:        $(CURDIR)/cmake-out/examples/models/muse-glimmer-batching/muse_glimmer_batching_worker"
+
 gemma4_31b-cuda:
 	@echo "==> Building and installing ExecuTorch with CUDA..."
 	cmake --workflow --preset llm-release-cuda

backends/arm/BUCK

@@ -108,6 +108,9 @@ fbcode_target(
         "vgf/check_env.py",
         "vgf/compile_spec.py",
         "vgf/model_converter.py",
+        "vgf/quantization_quality.py",
+        "vgf/quantization_assessment.py",
+        "vgf/quantization_visualization.py",
         "vgf/partitioner.py",
         "vgf/shaders/__init__.py",
         "vgf/shaders/grid_sampler.py",
@@ -132,6 +135,9 @@ fbcode_target(
         "//executorch/backends/arm/tosa/dialect:lib",
         "//executorch/backends/arm/tosa:specification",
         "//executorch/backends/arm/tosa:partitioner",
+        "//executorch/devtools/inspector/numerical_comparator:mse_numerical_comparator",
+        "//executorch/devtools/inspector/numerical_comparator:snr_numerical_comparator",
+        "//executorch/devtools/visualization:visualization",
         "//executorch/exir:lib",
     ],
 )

backends/arm/cmsis_pack/README.md

@@ -64,7 +64,7 @@ cmake \
   -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \
   -DEXECUTORCH_BUILD_FLATC=ON \
   -Bcmake-out-arm .
-cmake --build cmake-out-arm --config Release -j$(nproc)
+cmake --build cmake-out-arm --config Release
 
 # 2. Build the pack. --output-dir is where the .pack archive lands
 #    (created if absent); each invocation rewrites this directory.
@@ -75,6 +75,8 @@ backends/arm/cmsis_pack/scripts/build_pack.sh \
   --output-dir pack-output
 ```
 
+Adding `-j<N>` to `cmake --build` runs `<N>` jobs in parallel and tends to speed up the build significantly.
+
 The resulting `.pack` file is a zip archive installable via `cpackget add <file>.pack`.
 
 ## Local testing