From ffcb2924acc6df29c215891c5810713ab99fcd5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5ns=20Nilsson?= Date: Tue, 9 Sep 2025 10:18:24 +0200 Subject: [PATCH 1/9] Arm backend: Enable pytest VKML op unit testing in ciflow/trunk Change-Id: I58fa82ba8c55103c934a1a0b309d6e7c1327d35e --- .github/workflows/trunk.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/trunk.yml b/.github/workflows/trunk.yml index 04ecc44e966..05a4845b36a 100644 --- a/.github/workflows/trunk.yml +++ b/.github/workflows/trunk.yml @@ -281,6 +281,7 @@ jobs: include: - test_arm_baremetal: test_pytest_ops_ethosu_fvp - test_arm_baremetal: test_pytest_models_ethosu_fvp + - test_arm_baremetal: test_pytest_ops_vkml - test_arm_baremetal: test_run_ethosu_fvp - test_arm_baremetal: test_models_tosa - test_arm_baremetal: test_models_ethos-u55 @@ -301,7 +302,7 @@ jobs: source .ci/scripts/utils.sh install_executorch "--use-pt-pinned-commit" - .ci/scripts/setup-arm-baremetal-tools.sh + .ci/scripts/setup-arm-baremetal-tools.sh --enable-mlsdk-deps # Increase number of files user can monitor to bypass buck failures. # Hopefully this is high enough for this setup. From d403007dbc6983cd564fe14470f8e68cf4dbac66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5ns=20Nilsson?= Date: Fri, 7 Nov 2025 08:36:16 +0100 Subject: [PATCH 2/9] Use pip option for MLSDK setup in CI --- .github/workflows/trunk.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/trunk.yml b/.github/workflows/trunk.yml index fd29257d6d5..786fa3f4814 100644 --- a/.github/workflows/trunk.yml +++ b/.github/workflows/trunk.yml @@ -307,7 +307,7 @@ jobs: source .ci/scripts/utils.sh install_executorch "--use-pt-pinned-commit" - .ci/scripts/setup-arm-baremetal-tools.sh --enable-mlsdk-deps + .ci/scripts/setup-arm-baremetal-tools.sh --enable-mlsdk-deps --install-mlsdk-deps-with-pip # Increase number of files user can monitor to bypass buck failures. # Hopefully this is high enough for this setup. From 10fac6eac5469ab487525d31e267334f87669151 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5ns=20Nilsson?= Date: Fri, 7 Nov 2025 13:39:28 +0100 Subject: [PATCH 3/9] Add VGF flows to enable test runner for VKML in CI --- .ci/scripts/test_backend.sh | 4 ++++ .github/workflows/test-backend-arm.yml | 2 +- backends/test/suite/flow.py | 4 ++++ backends/test/suite/flows/arm.py | 10 ++++++++++ 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/.ci/scripts/test_backend.sh b/.ci/scripts/test_backend.sh index a48cc9ec41a..79d2111c39f 100755 --- a/.ci/scripts/test_backend.sh +++ b/.ci/scripts/test_backend.sh @@ -66,6 +66,10 @@ if [[ "$FLOW" == *arm* ]]; then backends/arm/scripts/build_executorch.sh backends/arm/test/setup_testing.sh fi + if [[ "$FLOW" == *vgf* ]]; then + # Prepare a test runner binary for VKML runtime + backends/arm/test/setup_testing_vkml.sh + fi fi if [[ $IS_MACOS -eq 1 ]]; then diff --git a/.github/workflows/test-backend-arm.yml b/.github/workflows/test-backend-arm.yml index 22e3d524f6b..638d5a2079f 100644 --- a/.github/workflows/test-backend-arm.yml +++ b/.github/workflows/test-backend-arm.yml @@ -26,7 +26,7 @@ jobs: uses: ./.github/workflows/_test_backend.yml with: backend: arm - flows: '["arm_tosa_fp", "arm_tosa_int", "arm_ethos_u55", "arm_ethos_u85"]' + flows: '["arm_tosa_fp", "arm_tosa_int", "arm_ethos_u55", "arm_ethos_u85", "arm_vgf_fp", "arm_vgf_int"]' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 120 run-linux: true diff --git a/backends/test/suite/flow.py b/backends/test/suite/flow.py index 29394951bd7..f3c9ee75083 100644 --- a/backends/test/suite/flow.py +++ b/backends/test/suite/flow.py @@ -132,6 +132,8 @@ def all_flows() -> dict[str, TestFlow]: ARM_ETHOS_U85_FLOW, ARM_TOSA_FP_FLOW, ARM_TOSA_INT_FLOW, + ARM_VGF_FP_FLOW, + ARM_VGF_INT_FLOW, ) flows += [ @@ -139,6 +141,8 @@ def all_flows() -> dict[str, TestFlow]: ARM_TOSA_INT_FLOW, ARM_ETHOS_U55_FLOW, ARM_ETHOS_U85_FLOW, + ARM_VGF_FP_FLOW, + ARM_VGF_INT_FLOW, ] except Exception as e: logger.info(f"Skipping ARM flow registration: {e}") diff --git a/backends/test/suite/flows/arm.py b/backends/test/suite/flows/arm.py index db3bb2cfd9e..1dfb894a281 100644 --- a/backends/test/suite/flows/arm.py +++ b/backends/test/suite/flows/arm.py @@ -78,3 +78,13 @@ def create_quantize_stage() -> Quantize: lambda: common.get_u85_compile_spec(), quantize=True, ) +ARM_VGF_FP_FLOW = _create_arm_flow( + "arm_vgf_fp", + lambda: common.get_vgf_compile_spec(tosa_spec="TOSA-1.0+FP"), + quantize=False, +) +ARM_VGF_INT_FLOW = _create_arm_flow( + "arm_vgf_int", + lambda: common.get_vgf_compile_spec(tosa_spec="TOSA-1.0+INT"), + quantize=True, +) From c939c5b6741628b2e1efda94d3701858e9249d30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5ns=20Nilsson?= Date: Fri, 7 Nov 2025 14:36:48 +0100 Subject: [PATCH 4/9] Install Vulkan-SDK for MLSDK deps in CI --- .ci/scripts/test_backend.sh | 7 +++++++ .github/workflows/trunk.yml | 3 +++ 2 files changed, 10 insertions(+) diff --git a/.ci/scripts/test_backend.sh b/.ci/scripts/test_backend.sh index 79d2111c39f..cb39d040bc2 100755 --- a/.ci/scripts/test_backend.sh +++ b/.ci/scripts/test_backend.sh @@ -57,6 +57,12 @@ if [[ "$FLOW" == *vulkan* ]]; then fi if [[ "$FLOW" == *arm* ]]; then + + if [[ "$FLOW" == *vgf* ]]; then + # Setup Vulkan SDK which is required for VGF backend + source .ci/scripts/setup-vulkan-linux-deps.sh + fi + # Setup ARM deps. .ci/scripts/setup-arm-baremetal-tools.sh source examples/arm/ethos-u-scratch/setup_path.sh @@ -66,6 +72,7 @@ if [[ "$FLOW" == *arm* ]]; then backends/arm/scripts/build_executorch.sh backends/arm/test/setup_testing.sh fi + if [[ "$FLOW" == *vgf* ]]; then # Prepare a test runner binary for VKML runtime backends/arm/test/setup_testing_vkml.sh diff --git a/.github/workflows/trunk.yml b/.github/workflows/trunk.yml index 786fa3f4814..243638e3cbf 100644 --- a/.github/workflows/trunk.yml +++ b/.github/workflows/trunk.yml @@ -307,6 +307,9 @@ jobs: source .ci/scripts/utils.sh install_executorch "--use-pt-pinned-commit" + # Install Vulkan-SDK for MLSDK deps + source .ci/scripts/setup-vulkan-linux-deps.sh + .ci/scripts/setup-arm-baremetal-tools.sh --enable-mlsdk-deps --install-mlsdk-deps-with-pip # Increase number of files user can monitor to bypass buck failures. From 6875b8d8b7e0d1ddfa7ef077755a69559f09236e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5ns=20Nilsson?= Date: Fri, 7 Nov 2025 15:05:51 +0100 Subject: [PATCH 5/9] Add missing mlsd-deps option to test-backend-linux vgf flow --- .ci/scripts/test_backend.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.ci/scripts/test_backend.sh b/.ci/scripts/test_backend.sh index cb39d040bc2..faf2d3caa9d 100755 --- a/.ci/scripts/test_backend.sh +++ b/.ci/scripts/test_backend.sh @@ -58,13 +58,14 @@ fi if [[ "$FLOW" == *arm* ]]; then + # Setup ARM deps. if [[ "$FLOW" == *vgf* ]]; then - # Setup Vulkan SDK which is required for VGF backend + # Vulkan-SDK required for VGF backend source .ci/scripts/setup-vulkan-linux-deps.sh + .ci/scripts/setup-arm-baremetal-tools.sh --enable-mlsdk-deps --install-mlsdk-deps-with-pip + else + .ci/scripts/setup-arm-baremetal-tools.sh fi - - # Setup ARM deps. - .ci/scripts/setup-arm-baremetal-tools.sh source examples/arm/ethos-u-scratch/setup_path.sh if [[ "$FLOW" == *ethos_u* ]]; then From b213bbdd2b69c460450582bac0148e849ad8fc5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5ns=20Nilsson?= Date: Fri, 7 Nov 2025 19:52:19 +0100 Subject: [PATCH 6/9] Do not use portable ops for vgf flows --- backends/test/suite/flows/arm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backends/test/suite/flows/arm.py b/backends/test/suite/flows/arm.py index 1dfb894a281..29ef504d50c 100644 --- a/backends/test/suite/flows/arm.py +++ b/backends/test/suite/flows/arm.py @@ -82,9 +82,11 @@ def create_quantize_stage() -> Quantize: "arm_vgf_fp", lambda: common.get_vgf_compile_spec(tosa_spec="TOSA-1.0+FP"), quantize=False, + use_portable_ops=False, ) ARM_VGF_INT_FLOW = _create_arm_flow( "arm_vgf_int", lambda: common.get_vgf_compile_spec(tosa_spec="TOSA-1.0+INT"), quantize=True, + use_portable_ops=False, ) From f818ec4b368c09d26d9e02b5cb9aa9bff76d1ead Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5ns=20Nilsson?= Date: Sat, 8 Nov 2025 09:12:12 +0100 Subject: [PATCH 7/9] Test with linux.g5.4xlarge.nvidia.gpu runner if VGF test pass Note this is for testing, and not meant for final PR. --- .github/workflows/trunk.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/trunk.yml b/.github/workflows/trunk.yml index 243638e3cbf..ec97124334e 100644 --- a/.github/workflows/trunk.yml +++ b/.github/workflows/trunk.yml @@ -294,7 +294,7 @@ jobs: - test_arm_baremetal: test_model_smollm2-135M fail-fast: false with: - runner: linux.2xlarge.memory + runner: linux.g5.4xlarge.nvidia.gpu docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} From f0baf5e53897d5d21107d7d57e484db9fd61c6fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5ns=20Nilsson?= Date: Sat, 8 Nov 2025 11:07:00 +0100 Subject: [PATCH 8/9] Do not isntall swiftshader --- .github/workflows/trunk.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/trunk.yml b/.github/workflows/trunk.yml index ec97124334e..8c1914a9ca4 100644 --- a/.github/workflows/trunk.yml +++ b/.github/workflows/trunk.yml @@ -307,9 +307,6 @@ jobs: source .ci/scripts/utils.sh install_executorch "--use-pt-pinned-commit" - # Install Vulkan-SDK for MLSDK deps - source .ci/scripts/setup-vulkan-linux-deps.sh - .ci/scripts/setup-arm-baremetal-tools.sh --enable-mlsdk-deps --install-mlsdk-deps-with-pip # Increase number of files user can monitor to bypass buck failures. From ced6a62e0f47b8b38c466e14a9c01d8de8d4ade1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5ns=20Nilsson?= Date: Sat, 8 Nov 2025 13:21:30 +0100 Subject: [PATCH 9/9] Revert linux.g5.4xlarge.nvidia.gpu runner --- .ci/scripts/test_backend.sh | 2 -- .github/workflows/trunk.yml | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.ci/scripts/test_backend.sh b/.ci/scripts/test_backend.sh index faf2d3caa9d..e959a2f074a 100755 --- a/.ci/scripts/test_backend.sh +++ b/.ci/scripts/test_backend.sh @@ -60,8 +60,6 @@ if [[ "$FLOW" == *arm* ]]; then # Setup ARM deps. if [[ "$FLOW" == *vgf* ]]; then - # Vulkan-SDK required for VGF backend - source .ci/scripts/setup-vulkan-linux-deps.sh .ci/scripts/setup-arm-baremetal-tools.sh --enable-mlsdk-deps --install-mlsdk-deps-with-pip else .ci/scripts/setup-arm-baremetal-tools.sh diff --git a/.github/workflows/trunk.yml b/.github/workflows/trunk.yml index 8c1914a9ca4..786fa3f4814 100644 --- a/.github/workflows/trunk.yml +++ b/.github/workflows/trunk.yml @@ -294,7 +294,7 @@ jobs: - test_arm_baremetal: test_model_smollm2-135M fail-fast: false with: - runner: linux.g5.4xlarge.nvidia.gpu + runner: linux.2xlarge.memory docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}