Skip to content

Commit 04a2d72

Browse files
authored
Arm backend: Add QAT support to test framework and model tests (#22741)
### Summary Adds an `is_qat` option to the shared TOSA and VGF test pipelines so quantization-aware training follows the existing export and lowering flow while PTQ remains the default. Adds QAT smoke coverage for the current FP32 NSS and NFRU models through TOSA and VGF. ### Test plan Gerrit patch set 13 passed CI (`Verified +1`) and received `Code-Review +2` and `IP-Review +1`. Also checked the patch with `git diff --check`. cc @digantdesai @freddan80 @per @zingo @oscarandersson8218 @mansnils @Sebastian-Larsson @robell @rascani
1 parent 4798a67 commit 04a2d72

4 files changed

Lines changed: 151 additions & 6 deletions

File tree

‎backends/arm/MODELS.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,8 @@
2020
- Some popular torch.nn.modules models (NN modules)
2121
- Some popular torch ops (Torch Functions)
2222
- T5 (T5 for conditional generation)
23-
- Neural Super Sampler (NSS)
23+
- Neural Frame Rate Upscaling (NFRU), including QAT smoke coverage
24+
- Neural Super Sampling (NSS), including QAT smoke coverage
2425
- Phi-3
2526
- ResNet 18
2627
- ResNet-8
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
# Copyright 2026 Arm Limited and/or its affiliates.
2+
#
3+
# This source code is licensed under the BSD-style license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
import os
7+
8+
os.environ.setdefault("HF_HUB_DISABLE_XET", "1")
9+
10+
import pytest
11+
import torch
12+
13+
from executorch.backends.arm.test import common
14+
from executorch.backends.arm.test.tester.test_pipeline import (
15+
TosaPipelineINT,
16+
VgfPipeline,
17+
)
18+
from huggingface_hub import hf_hub_download
19+
from ng_model_gym.usecases.nfru.model.nfru_v1_nn import ( # type: ignore[import-not-found,import-untyped]
20+
NFRUAutoEncoder,
21+
)
22+
23+
24+
_RELEASE_REFS = (
25+
os.environ.get("GITHUB_REF", ""),
26+
os.environ.get("GITHUB_REF_NAME", ""),
27+
os.environ.get("GITHUB_BASE_REF", ""),
28+
)
29+
_IS_FROZEN_RELEASE = any(
30+
ref.removeprefix("refs/heads/").startswith("release/") for ref in _RELEASE_REFS
31+
)
32+
pytestmark = pytest.mark.skipif(
33+
_IS_FROZEN_RELEASE,
34+
reason="NFRU tests depend on resources fetched from main.",
35+
)
36+
37+
_NFRU_QAT_INPUTS = (torch.ones((1, 16, 64, 64)),)
38+
39+
40+
def nfru() -> NFRUAutoEncoder:
41+
weights = hf_hub_download( # nosec B615
42+
repo_id="Arm/neural-frame-rate-upscaling",
43+
filename="nfru_v1_fp32.pt",
44+
revision="main",
45+
cache_dir=os.environ.get("RUNNER_TEMP"),
46+
)
47+
checkpoint = torch.load(
48+
weights,
49+
map_location=torch.device("cpu"),
50+
weights_only=True,
51+
)["model_state_dict"]
52+
prefix = "network.auto_encoder."
53+
assert all(key.startswith(prefix) for key in checkpoint)
54+
55+
model = NFRUAutoEncoder()
56+
model.load_state_dict(
57+
{key.removeprefix(prefix): value for key, value in checkpoint.items()},
58+
strict=True,
59+
)
60+
return model
61+
62+
63+
def test_nfru_qat_tosa_INT() -> None:
64+
pipeline = TosaPipelineINT(
65+
nfru(),
66+
_NFRU_QAT_INPUTS,
67+
aten_op=[],
68+
exir_op=[],
69+
per_channel_quantization=False,
70+
use_to_edge_transform_and_lower=True,
71+
is_qat=True,
72+
frobenius_threshold=None,
73+
cosine_threshold=None,
74+
)
75+
pipeline.run()
76+
77+
78+
@common.SkipIfNoModelConverter
79+
def test_nfru_qat_vgf_INT() -> None:
80+
pipeline = VgfPipeline(
81+
nfru(),
82+
_NFRU_QAT_INPUTS,
83+
aten_op=[],
84+
exir_op=[],
85+
run_on_vulkan_runtime=True,
86+
quantize=True,
87+
per_channel_quantization=False,
88+
use_to_edge_transform_and_lower=True,
89+
is_qat=True,
90+
tosa_version="TOSA-1.0+INT",
91+
)
92+
pipeline.run()

‎backends/arm/test/models/test_nss.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,3 +212,38 @@ def test_nss_vgf_INT(use_real_data):
212212
if use_real_data:
213213
_set_nss_calibration_samples(pipeline)
214214
pipeline.run()
215+
216+
217+
def test_nss_qat_tosa_INT() -> None:
218+
pipeline = TosaPipelineINT[input_t](
219+
nss().eval(),
220+
example_inputs(),
221+
aten_op=[],
222+
exir_op=[],
223+
use_to_edge_transform_and_lower=True,
224+
is_qat=True,
225+
frobenius_threshold=None,
226+
cosine_threshold=None,
227+
qtol=12,
228+
)
229+
_set_nss_calibration_samples(pipeline)
230+
pipeline.run()
231+
232+
233+
@common.SkipIfNoModelConverter
234+
def test_nss_qat_vgf_INT() -> None:
235+
pipeline = VgfPipeline[input_t](
236+
nss().eval(),
237+
example_inputs(),
238+
aten_op=[],
239+
exir_op=[],
240+
symmetric_io_quantization=True,
241+
use_to_edge_transform_and_lower=True,
242+
run_on_vulkan_runtime=True,
243+
quantize=True,
244+
is_qat=True,
245+
tosa_version="TOSA-1.0+INT",
246+
qtol=12,
247+
)
248+
_set_nss_calibration_samples(pipeline)
249+
pipeline.run()

‎backends/arm/test/tester/test_pipeline.py‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -435,6 +435,7 @@ class TosaPipelineINT(TOSAPipeline, Generic[T]):
435435
run_on_tosa_ref_model: Set to true to test the tosa file on the TOSA reference model.
436436
symmetric_io_quantization: Whether to use symmetric I/O quantization.
437437
per_channel_quantization: Whether to use per-channel quantization.
438+
is_qat: Whether to prepare the model for quantization-aware training instead of PTQ.
438439
use_to_edge_transform_and_lower: Selects betweeen two possible ways of lowering the module.
439440
custom_path : Path to dump intermediate artifacts such as tosa and pte to.
440441
tosa_debug_mode: Optional debug mode for TOSA compilation.
@@ -475,6 +476,7 @@ def __init__(
475476
tosa_extensions: Optional[List[str]] = None,
476477
epsilon: float = 2**-16,
477478
fold_quantize: bool = True,
479+
is_qat: bool = False,
478480
):
479481
if tosa_extensions is None:
480482
tosa_extensions = []
@@ -492,16 +494,22 @@ def __init__(
492494
# choose 16A8W quantization config when int16 extension is requested
493495
if "int16" in tosa_extensions:
494496
quantization_config = get_symmetric_a16w8_quantization_config(
495-
is_per_channel=per_channel_quantization, epsilon=epsilon
497+
is_per_channel=per_channel_quantization,
498+
is_qat=is_qat,
499+
epsilon=epsilon,
496500
)
497501
else:
498502
quantization_config = get_symmetric_quantization_config(
499-
is_per_channel=per_channel_quantization
503+
is_per_channel=per_channel_quantization,
504+
is_qat=is_qat,
500505
)
501506
if symmetric_io_quantization:
502507
quantizer.set_io(quantization_config)
503508
quant_stage = Quantize(
504-
quantizer, quantization_config, fold_quantize=fold_quantize
509+
quantizer,
510+
quantization_config,
511+
is_qat=is_qat,
512+
fold_quantize=fold_quantize,
505513
)
506514

507515
super().__init__(
@@ -1217,6 +1225,7 @@ class VgfPipeline(BasePipeline, Generic[T]):
12171225
run_on_vulkan_runtime: Whether to test VGF output on VKML runtime.
12181226
12191227
vgf_compiler_flags: Optional compiler flags.
1228+
is_qat: Whether to prepare the model for quantization-aware training instead of PTQ.
12201229
12211230
tosa_version: A string for identifying the TOSA version.
12221231
tosa_spec: Optional override for the TOSA specification.
@@ -1255,7 +1264,11 @@ def __init__(
12551264
fold_quantize: bool = True,
12561265
preserve_io_quantization: bool = False,
12571266
n_expected_delegates: int = 1,
1267+
is_qat: bool = False,
12581268
):
1269+
if is_qat and not quantize:
1270+
raise ValueError("QAT requires quantize=True.")
1271+
12591272
if tosa_spec is None:
12601273
if tosa_version is None:
12611274
tosa_version = str(VgfCompileSpec().tosa_spec)
@@ -1301,12 +1314,16 @@ def __init__(
13011314
if quantize:
13021315
quantizer = VgfQuantizer(compile_spec)
13031316
quantization_config = get_symmetric_quantization_config(
1304-
is_per_channel=per_channel_quantization
1317+
is_per_channel=per_channel_quantization,
1318+
is_qat=is_qat,
13051319
)
13061320
if symmetric_io_quantization:
13071321
quantizer.set_io(quantization_config)
13081322
quant_stage = Quantize(
1309-
quantizer, quantization_config, fold_quantize=fold_quantize
1323+
quantizer,
1324+
quantization_config,
1325+
is_qat=is_qat,
1326+
fold_quantize=fold_quantize,
13101327
)
13111328

13121329
self.add_stage(self.tester.quantize, quant_stage, pos=0)

0 commit comments

Comments
 (0)