NXP backend: use channels-first for reference testing model (pytorch#23308)

### Summary

Refactor the testing suite to enable reference testing in
channels-first.

PR is merged.

### Test plan

tests can be manually run using `pytest -c /dev/null
backends/nxp/tests/`

cc @robert-kalmar @JakeStevens @digantdesai @rascani @roman-janik-nxp

Author: novak-vaclav

backends/nxp/tests/generic_tests/test_debug_results.py

@@ -37,12 +37,15 @@ def test_nsys_test_debug_results__single_input(caplog, request):
 
     graph_verifier = BaseGraphVerifier(1, [])
 
+    # use different reference model dim order so transposed dataset is created
+    ref_input_spec = ModelInputSpec(input_shape, dim_order=torch.channels_last)
     lower_run_compare(
         model,
         input_shape,
         graph_verifier,
         request,
         remove_quant_io_ops=True,
+        reference_input_spec=[ref_input_spec],
     )
 
     test_name = get_test_name(request)
@@ -70,10 +73,16 @@ def test_nsys_test_debug_results__single_input(caplog, request):
 
     # Check text tensor variants
     assert os.path.isfile(
-        os.path.join(OUTPUTS_DIR, test_name, "dataset", "calibration", "0000.txt")
+        os.path.join(OUTPUTS_DIR, test_name, "model_dataset", "calibration", "0000.txt")
     )
     assert os.path.isfile(
-        os.path.join(OUTPUTS_DIR, test_name, "dataset_quant", "0000.txt")
+        os.path.join(OUTPUTS_DIR, test_name, "model_dataset_quant", "0000.txt")
+    )
+    assert os.path.isfile(
+        os.path.join(OUTPUTS_DIR, test_name, "ref_dataset", "calibration", "0000.bin")
+    )
+    assert os.path.isfile(
+        os.path.join(OUTPUTS_DIR, test_name, "ref_dataset_quant", "0000.bin")
     )
     assert os.path.isfile(
         os.path.join(OUTPUTS_DIR, test_name, "results_cpu", "0000.bin", "0000.txt")
@@ -140,7 +149,7 @@ def test_nsys_test_debug_results__multiple_input(self, caplog, request):
         # Check text tensor variants
         assert os.path.isfile(
             os.path.join(
-                OUTPUTS_DIR, test_name, "dataset", "calibration", "0000", "00.txt"
+                OUTPUTS_DIR, test_name, "model_dataset", "calibration", "0000", "00.txt"
             )
         )
         assert os.path.isfile(

backends/nxp/tests/generic_tests/test_quantized_input_data.py

@@ -12,7 +12,7 @@
 from executorch.backends.nxp.tests.nsys_testing import (
     lower_run_compare,
     OUTPUTS_DIR,
-    ReferenceModel,
+    ReferenceRunner,
 )
 from executorch.backends.nxp.tests.simple_models import AvgPool2dModule, MulTensorModule
 
@@ -34,7 +34,7 @@ def test__single_quantized_inputs(mocker, request):
     )
 
     test_name = nsys_testing.get_test_name(request)
-    assert (OUTPUTS_DIR / test_name / "dataset_quant" / "0000.bin").exists()
+    assert (OUTPUTS_DIR / test_name / "model_dataset_quant" / "0000.bin").exists()
 
     # Check outputs are in quantized int8 format
     output_tensor_spec = output_tensor_spec_spy.spy_return
@@ -54,12 +54,12 @@ def test__single_quantized_inputs_edge_python_reference(mocker, request):
         [input_spec],
         graph_verifier,
         request,
-        reference_model=ReferenceModel.QUANTIZED_EDGE_PYTHON,
+        reference_runner=ReferenceRunner.QUANTIZED_EDGE_PYTHON,
         remove_quant_io_ops=True,
     )
 
     test_name = nsys_testing.get_test_name(request)
-    assert (OUTPUTS_DIR / test_name / "dataset_quant" / "0000.bin").exists()
+    assert (OUTPUTS_DIR / test_name / "model_dataset_quant" / "0000.bin").exists()
 
     # Check outputs are in quantized int8 format
     output_tensor_spec = output_tensor_spec_spy.spy_return
@@ -83,7 +83,9 @@ def test__multiple_quantized_inputs(mocker, request):
     )
 
     test_name = nsys_testing.get_test_name(request)
-    assert (OUTPUTS_DIR / test_name / "dataset_quant" / "0000" / "00.bin").exists()
+    assert (
+        OUTPUTS_DIR / test_name / "model_dataset_quant" / "0000" / "00.bin"
+    ).exists()
 
     # Check outputs are in quantized int8 format
     output_tensor_spec = output_tensor_spec_spy.spy_return
@@ -103,12 +105,14 @@ def test__multiple_quantized_inputs_edge_python_reference(mocker, request):
         [x_input_spec, x_input_spec],
         graph_verifier,
         request,
-        reference_model=ReferenceModel.QUANTIZED_EDGE_PYTHON,
+        reference_runner=ReferenceRunner.QUANTIZED_EDGE_PYTHON,
         remove_quant_io_ops=True,
     )
 
     test_name = nsys_testing.get_test_name(request)
-    assert (OUTPUTS_DIR / test_name / "dataset_quant" / "0000" / "00.bin").exists()
+    assert (
+        OUTPUTS_DIR / test_name / "model_dataset_quant" / "0000" / "00.bin"
+    ).exists()
 
     # Check outputs are in quantized int8 format
     output_tensor_spec = output_tensor_spec_spy.spy_return

backends/nxp/tests/ir/converter/node_converter/test_clamp_converter.py

@@ -89,7 +89,7 @@ def test__qat(self, mocker, request, min, max, use_qat):
 
         lower_run_compare(
             model=model,
-            input_spec=[x_input_spec],
+            model_input_spec=[x_input_spec],
             request=request,
             dlg_model_verifier=graph_verifier,
         )
@@ -132,7 +132,7 @@ def test_convert_clamp__full_pipeline(self, mocker, request, min, max):
 
         lower_run_compare(
             model=model,
-            input_spec=[x_input_spec],
+            model_input_spec=[x_input_spec],
             dlg_model_verifier=graph_verifier,
             request=request,
         )

backends/nxp/tests/ir/converter/node_converter/test_conv_converter.py

@@ -20,7 +20,7 @@
 from executorch.backends.nxp.tests.nsys_testing import (
     AllCloseOutputComparator,
     lower_run_compare,
-    ReferenceModel,
+    ReferenceRunner,
 )
 from executorch.backends.nxp.tests.simple_models import (
     Conv2dModule,
@@ -43,7 +43,7 @@ def assert_delegated_and_correct(
     use_qat=False,
     exp_delegated_ops=None,
     exp_non_delegated_ops=None,
-    et_ref_model=ReferenceModel.QUANTIZED_EXECUTORCH_CPP,
+    et_ref_model=ReferenceRunner.QUANTIZED_EXECUTORCH_CPP,
 ):
     if exp_delegated_ops is None:
         exp_delegated_ops = {Convolution: 1}
@@ -70,7 +70,7 @@ def assert_delegated_and_correct(
         comparator,
         use_qat=use_qat,
         remove_quant_io_ops=remove_quant_io_ops,
-        reference_model=et_ref_model,
+        reference_runner=et_ref_model,
     )
 
 
@@ -198,9 +198,9 @@ def test__tr_unusual(
         # Running `conv_transpose2d` with `output_channels = 1` produces errors in Executorch. The issue has been reported:
         # https://github.com/pytorch/executorch/issues/20804
         ref_model = (
-            ReferenceModel.QUANTIZED_EXECUTORCH_CPP
+            ReferenceRunner.QUANTIZED_EXECUTORCH_CPP
             if use_et_ref_model
-            else ReferenceModel.QUANTIZED_EDGE_PYTHON
+            else ReferenceRunner.QUANTIZED_EDGE_PYTHON
         )
 
         assert_delegated_and_correct(

backends/nxp/tests/models/test_cifarnet.py

@@ -17,7 +17,10 @@
     NumericalStatsOutputComparator,
 )
 
-from executorch.backends.nxp.tests.nsys_testing import lower_run_compare, ReferenceModel
+from executorch.backends.nxp.tests.nsys_testing import (
+    lower_run_compare,
+    ReferenceRunner,
+)
 from executorch.examples.nxp.experimental.cifar_net.cifar_net import (
     CifarNet,
     store_test_data,
@@ -69,10 +72,10 @@ def test_cifarnet(mocker, request, cifar_test_files, channels_last):
         # Run the channels last reference in PyTorch as the ExecuTorch CPU model contains incorrectly
         #  lowered channels last convolution weights, which cause incorrect inference results. The issue
         #  is caused by ExecuTorch (not NXP). https://github.com/pytorch/executorch/issues/16464
-        reference_model=(
-            ReferenceModel.QUANTIZED_EDGE_PYTHON
+        reference_runner=(
+            ReferenceRunner.QUANTIZED_EDGE_PYTHON
             if channels_last
-            else ReferenceModel.QUANTIZED_EXECUTORCH_CPP
+            else ReferenceRunner.QUANTIZED_EXECUTORCH_CPP
         ),
     )
 

backends/nxp/tests/models/test_mlperf_tiny_anomaly_detection.py

@@ -24,8 +24,8 @@
 )
 
 BOUNDS_MSE = {
-    "PTQ": 1.4e-08,
-    "QAT": 5.205e-06,
+    "PTQ": 1.0e-8,
+    "QAT": 1.0e-6,
 }
 
 
@@ -53,22 +53,26 @@ def test_mlperf_tiny_anomaly_detection_mse_cpu_vs_npu(
         dataset, num_examples=num_samples, idx_to_label=labels
     )
 
-    input_spec = ModelInputSpec(anomaly_detection.input_shape)
+    model_input_spec = ModelInputSpec(anomaly_detection.input_shape)
     quant_type_key = "QAT" if use_qat else "PTQ"
 
     mse = BOUNDS_MSE[quant_type_key]
     comparator = NumericalStatsOutputComparator(max_mse_error=mse)
     model_verifier = BaseGraphVerifier(1, [])
     train_fn = anomaly_detection.train_model_fn if use_qat else None
 
+    ref_input_spec = ModelInputSpec(anomaly_detection.input_shape)
+    ref_input_spec.dim_order = torch.contiguous_format
+
     lower_run_compare(
         model,
-        [input_spec],
+        [model_input_spec],
         model_verifier,
         request,
         dataset_creator=dataset_creator,
         output_comparator=comparator,
         mocker=mocker,
+        reference_input_spec=[ref_input_spec],
         use_qat=use_qat,
         train_fn=train_fn,
     )

backends/nxp/tests/models/test_mlperf_tiny_image_classification.py

@@ -19,15 +19,15 @@
 from executorch.backends.nxp.tests.model_output_comparator import (
     NumericalStatsOutputComparator,
 )
-from executorch.backends.nxp.tests.nsys_testing import lower_run_compare, ReferenceModel
+from executorch.backends.nxp.tests.nsys_testing import lower_run_compare
 from executorch.backends.nxp.tests.use_qat import *  # noqa F403
 from executorch.examples.nxp.models.mlperf_tiny.image_classification.mlperf_tiny_image_classification import (
     MLPerfTinyImageClassification,
 )
 
 BOUNDS_MSE = {
-    "PTQ": {"channels-last": 1.859e-05, "channels-first": 7.432e-09},
-    "QAT": {"channels-last": 2.751e-07, "channels-first": 5.205e-06},
+    "PTQ": {"channels-last": 1.0e-9, "channels-first": 1.0e-9},
+    "QAT": {"channels-last": 1.0e-8, "channels-first": 1.0e-7},
 }
 
 
@@ -54,10 +54,10 @@ def test_mlperf_tiny_classification_mse_cpu_vs_npu(
         dataset, num_examples=num_samples, idx_to_label=labels
     )
 
-    input_spec = ModelInputSpec(img_classification.input_shape)
+    model_input_spec = ModelInputSpec(img_classification.input_shape)
     if channels_last:
         model.to(memory_format=torch.channels_last)
-        input_spec.dim_order = torch.channels_last
+        model_input_spec.dim_order = torch.channels_last
 
     quant_type_key = "QAT" if use_qat else "PTQ"
     format_key = "channels-last" if channels_last else "channels-first"
@@ -73,22 +73,18 @@ def test_mlperf_tiny_classification_mse_cpu_vs_npu(
         else None
     )
 
-    # Portable constant_pad_nd does not support channels-last tensors.
-    ref_model = (
-        ReferenceModel.QUANTIZED_EDGE_PYTHON
-        if channels_last
-        else ReferenceModel.QUANTIZED_EXECUTORCH_CPP
-    )
+    ref_input_spec = ModelInputSpec(img_classification.input_shape)
+    ref_input_spec.dim_order = torch.contiguous_format
 
     lower_run_compare(
         model,
-        [input_spec],
+        [model_input_spec],
         model_verifier,
         request,
         dataset_creator=dataset_creator,
         output_comparator=comparator,
-        reference_model=ref_model,
         mocker=mocker,
+        reference_input_spec=[ref_input_spec],
         use_qat=use_qat,
         train_fn=train_fn,
     )

backends/nxp/tests/models/test_mlperf_tiny_keyword_spotting.py

@@ -19,20 +19,20 @@
 from executorch.backends.nxp.tests.model_output_comparator import (
     NumericalStatsOutputComparator,
 )
-from executorch.backends.nxp.tests.nsys_testing import lower_run_compare, ReferenceModel
+from executorch.backends.nxp.tests.nsys_testing import lower_run_compare
 from executorch.backends.nxp.tests.use_qat import *  # noqa F403
 from executorch.examples.nxp.models.mlperf_tiny.keyword_spotting.mlperf_tiny_keyword_spotting import (
     MLPerfTinyKeywordSpotting,
 )
 
 BOUNDS_MSE = {
     "PTQ": {
-        "channels-last": 3.5e-4,
-        "channels-first": 2.0e-6,
+        "channels-last": 1.0e-15,
+        "channels-first": 1.0e-6,
     },
     "QAT": {
-        "channels-last": 3.5e-4,
-        "channels-first": 5.0e-5,
+        "channels-last": 0.0,
+        "channels-first": 1.0e-4,
     },
 }
 
@@ -58,10 +58,10 @@ def test_mlperf_tiny_kws_mse_cpu_vs_npu(mocker, request, channels_last, use_qat)
         dataset, num_examples=num_samples, idx_to_label=labels
     )
 
-    input_spec = ModelInputSpec(kws.input_shape)
+    model_input_spec = ModelInputSpec(kws.input_shape)
     if channels_last:
         model.to(memory_format=torch.channels_last)
-        input_spec.dim_order = torch.channels_last
+        model_input_spec.dim_order = torch.channels_last
 
     quant_type_key = "QAT" if use_qat else "PTQ"
     format_key = "channels-last" if channels_last else "channels-first"
@@ -74,15 +74,18 @@ def test_mlperf_tiny_kws_mse_cpu_vs_npu(mocker, request, channels_last, use_qat)
         partial(kws.train_model_fn, channels_last=channels_last) if use_qat else None
     )
 
+    ref_input_spec = ModelInputSpec(kws.input_shape)
+    ref_input_spec.dim_order = torch.contiguous_format
+
     lower_run_compare(
         model,
-        [input_spec],
+        [model_input_spec],
         model_verifier,
         request,
         dataset_creator=dataset_creator,
         output_comparator=comparator,
         mocker=mocker,
-        reference_model=ReferenceModel.QUANTIZED_EXECUTORCH_CPP,
+        reference_input_spec=[ref_input_spec],
         use_qat=use_qat,
         train_fn=train_fn,
     )