Skip to content

Commit 2dca183

Browse files
Erik-LundellErik-Lundell
authored
Arm backend: Adjust tests (pytorch#19289)
- Removing passing xfails - Increase tolerance of some flaky tests - Remote frobenius/cosinue testing of some tests with bool output
1 parent 8e653a6 commit 2dca183

6 files changed

Lines changed: 11 additions & 43 deletions

File tree

‎backends/arm/test/models/Gemma3n/test_gemma3nModel.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -695,7 +695,7 @@ class Gemma3nTestCase:
695695
),
696696
"rms_norm": Gemma3nTestCase(model_cls=RMSNormModel),
697697
"altup": Gemma3nTestCase(model_cls=AltUpModel),
698-
"attention": Gemma3nTestCase(model_cls=AttentionModel, atol=0.04),
698+
"attention": Gemma3nTestCase(model_cls=AttentionModel, atol=0.07),
699699
"decoder_layer": Gemma3nTestCase(
700700
model_cls=DecoderLayerModel,
701701
call_delegates=0,

‎backends/arm/test/ops/test_conv2d.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -541,7 +541,7 @@ def test_convolution_2d_tosa_INT_a8w4(test_data):
541541
aten_op,
542542
exir_op,
543543
tosa_extensions=["int4"],
544-
frobenius_threshold=0.4,
544+
frobenius_threshold=0.45,
545545
)
546546
pipeline.quantizer.set_global(
547547
get_symmetric_a8w4_quantization_config(is_per_channel=per_channel_quantization)

‎backends/arm/test/ops/test_group_norm.py‎

Lines changed: 1 addition & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright 2025 Arm Limited and/or its affiliates.
1+
# Copyright 2025-2026 Arm Limited and/or its affiliates.
22
#
33
# This source code is licensed under the BSD-style license found in the
44
# LICENSE file in the root directory of this source tree.
@@ -77,13 +77,6 @@ def test_native_group_norm_tosa_FP(test_data):
7777
@common.parametrize(
7878
"test_data",
7979
test_data_suite,
80-
xfails={
81-
"randn_1_12_8_6_groups_12": "MLETORCH-925: Fix numerical issue for aten.native_group_norm",
82-
"rand_6_8_10_12_groups_1": "MLETORCH-925: Fix numerical issue for aten.native_group_norm",
83-
"rand_6_8_10_12_groups_4_no_affine": "MLETORCH-925: Fix numerical issue for aten.native_group_norm",
84-
"rand_6_8_10_12_groups_8": "MLETORCH-925: Fix numerical issue for aten.native_group_norm",
85-
},
86-
strict=False,
8780
)
8881
def test_native_group_norm_tosa_INT(test_data):
8982
aten_op = "torch.ops.aten.sub.Tensor" # 'sub' op arbitrarily chosen to confirm groupnorm was decomposed
@@ -101,16 +94,6 @@ def test_native_group_norm_tosa_INT(test_data):
10194
@common.parametrize(
10295
"test_data",
10396
test_data_suite,
104-
xfails={
105-
"rand_4_6_8_groups_2_eps_no_affine": "MLETORCH-925: Fix numerical issue for aten.native_group_norm",
106-
"rand_4_6_groups_1": "MLETORCH-925: Fix numerical issue for aten.native_group_norm",
107-
"rand_4_6_groups_2": "MLETORCH-925: Fix numerical issue for aten.native_group_norm",
108-
"randn_1_12_8_6_groups_12": "MLETORCH-925: Fix numerical issue for aten.native_group_norm",
109-
"rand_6_8_10_12_groups_1": "MLETORCH-925: Fix numerical issue for aten.native_group_norm",
110-
"rand_6_8_10_12_groups_4_no_affine": "MLETORCH-925: Fix numerical issue for aten.native_group_norm",
111-
"rand_6_8_10_12_groups_8": "MLETORCH-925: Fix numerical issue for aten.native_group_norm",
112-
},
113-
strict=False,
11497
)
11598
@common.XfailIfNoCorstone300
11699
def test_native_group_norm_u55_INT(test_data):
@@ -127,13 +110,6 @@ def test_native_group_norm_u55_INT(test_data):
127110
@common.parametrize(
128111
"test_data",
129112
test_data_suite,
130-
xfails={
131-
"randn_1_12_8_6_groups_12": "MLETORCH-925: Fix numerical issue for aten.native_group_norm",
132-
"rand_6_8_10_12_groups_1": "MLETORCH-925: Fix numerical issue for aten.native_group_norm",
133-
"rand_6_8_10_12_groups_4_no_affine": "MLETORCH-925: Fix numerical issue for aten.native_group_norm",
134-
"rand_6_8_10_12_groups_8": "MLETORCH-925: Fix numerical issue for aten.native_group_norm",
135-
},
136-
strict=False,
137113
)
138114
@common.XfailIfNoCorstone320
139115
def test_native_group_norm_u85_INT(test_data):
@@ -150,13 +126,6 @@ def test_native_group_norm_u85_INT(test_data):
150126
@common.parametrize(
151127
"test_data",
152128
test_data_suite,
153-
xfails={
154-
"randn_1_12_8_6_groups_12": "MLETORCH-925: Fix numerical issue",
155-
"rand_6_8_10_12_groups_1": "MLETORCH-925: Fix numerical issue",
156-
"rand_6_8_10_12_groups_4_no_affine": "MLETORCH-925: Fix numerical issue",
157-
"rand_6_8_10_12_groups_8": "MLETORCH-925: Fix numerical issue",
158-
},
159-
strict=False,
160129
)
161130
@common.SkipIfNoModelConverter
162131
def test_native_group_norm_vgf_no_quant(test_data):
@@ -176,13 +145,6 @@ def test_native_group_norm_vgf_no_quant(test_data):
176145
@common.parametrize(
177146
"test_data",
178147
test_data_suite,
179-
xfails={
180-
"randn_1_12_8_6_groups_12": "MLETORCH-925: Fix numerical issue",
181-
"rand_6_8_10_12_groups_1": "MLETORCH-925: Fix numerical issue",
182-
"rand_6_8_10_12_groups_4_no_affine": "MLETORCH-925: Fix numerical issue",
183-
"rand_6_8_10_12_groups_8": "MLETORCH-925: Fix numerical issue",
184-
},
185-
strict=False,
186148
)
187149
@common.SkipIfNoModelConverter
188150
def test_native_group_norm_vgf_quant(test_data):

‎backends/arm/test/ops/test_gt.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,8 @@ def test_gt_tensor_tosa_INT_a16w8(test_module):
179179
Greater.aten_op_tensor,
180180
Greater.exir_op,
181181
tosa_extensions=["int16"],
182+
frobenius_threshold=None, # Quantized comparisons with small diffs can be inaccurate, leading to large errors in unlucky cases.
183+
cosine_threshold=None,
182184
)
183185
pipeline.run()
184186

@@ -191,6 +193,8 @@ def test_gt_scalar_tosa_INT_a16w8(test_module):
191193
Greater.aten_op_tensor,
192194
Greater.exir_op,
193195
tosa_extensions=["int16"],
196+
frobenius_threshold=None, # Quantized comparisons with small diffs can be inaccurate, leading to large errors in unlucky cases.
197+
cosine_threshold=None,
194198
)
195199
pipeline.run()
196200

‎backends/arm/test/ops/test_lt.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,8 @@ def test_lt_tensor_tosa_INT_a16w8(test_module):
155155
LessThan.aten_op_tensor,
156156
LessThan.exir_op,
157157
tosa_extensions=["int16"],
158+
frobenius_threshold=None, # Quantized comparisons with small diffs can be inaccurate, leading to large errors in unlucky cases.
159+
cosine_threshold=None,
158160
)
159161
pipeline.run()
160162

@@ -167,6 +169,8 @@ def test_lt_scalar_tosa_INT_a16w8(test_module):
167169
LessThan.aten_op_tensor,
168170
LessThan.exir_op,
169171
tosa_extensions=["int16"],
172+
frobenius_threshold=None, # Quantized comparisons with small diffs can be inaccurate, leading to large errors in unlucky cases.
173+
cosine_threshold=None,
170174
)
171175
pipeline.run()
172176

‎backends/arm/test/ops/test_max_pool1d.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,6 @@ def forward(self, x):
7575

7676

7777
@common.parametrize("test_data", test_data_suite_all)
78-
@pytest.mark.xfail(reason="MaxPool1D not yet supported", strict=False)
7978
def test_max_pool2d_tosa_FP_decomposed(test_data: Callable):
8079
"""Test max_pool1d with TOSA FP pipeline."""
8180
test_data, model_params = test_data()
@@ -134,7 +133,6 @@ def test_max_pool2d_u85_INT_decomposed(test_data: Callable):
134133

135134
# VGF tests
136135
@common.parametrize("test_data", test_data_suite_all)
137-
@pytest.mark.xfail(reason="MaxPool1D not yet supported", strict=False)
138136
@common.SkipIfNoModelConverter
139137
def test_max_pool2d_vgf_no_quant(test_data: Callable):
140138
"""Test max_pool1d with VGF pipeline (non-quantized)."""

0 commit comments

Comments
 (0)