Skip to content

Commit 8176dcf

Browse files
Jiseong-ohChen03ZhaoSamsung
Jiseong-oh
and
Chen03ZhaoSamsung
committed
Add framework for quantization tc
- Add Quantization TestCases Co-authored-by: chen03.zhao@samsung.com <chen03.zhao@samsung.com> Signed-off-by: jiseong.oh <jiseong.oh@samsung.com>
1 parent f9f988d commit 8176dcf

13 files changed

+328
-14
lines changed

‎backends/samsung/test/models/test_deeplab_v3.py‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,20 @@
1-
# Copyright (c) Samsung Electronics Co. LTD
1+
# Copyright (c) 2025 Samsung Electronics Co. LTD
22
# All rights reserved
33
#
44
# Licensed under the BSD License (the "License"); you may not use this file
55
# except in compliance with the License. See the license file in the root
66
# directory of this source tree for more details.
7+
import os
78
import unittest
89

910
from executorch.backends.samsung.serialization.compile_options import (
1011
gen_samsung_backend_compile_spec,
1112
)
1213
from executorch.backends.samsung.test.tester import SamsungTester
14+
from executorch.backends.samsung.test.utils.datasets import (
15+
get_quant_test_data_segmentation,
16+
)
17+
from executorch.backends.samsung.test.utils.quant_checkers import CheckerConfig
1318
from executorch.backends.samsung.test.utils.utils import TestConfig
1419
from executorch.examples.models.deeplab_v3 import DeepLabV3ResNet50Model
1520

@@ -27,3 +32,26 @@ def test_dl3_fp16(self):
2732
.to_executorch()
2833
.run_method_and_compare_outputs(inputs=example_input, atol=0.009)
2934
)
35+
36+
def test_dl3_a8w8(self):
37+
model = DeepLabV3ResNet50Model().get_eager_model()
38+
example_input, cali, testdata = get_quant_test_data_segmentation(
39+
os.path.join(os.environ["DATASET_PATH"], "VOC_image")
40+
)
41+
checker_config = CheckerConfig(
42+
"segmentation",
43+
{
44+
"dataset": testdata,
45+
"threshold": 0.7,
46+
},
47+
)
48+
tester = SamsungTester(
49+
model, example_input, [gen_samsung_backend_compile_spec(TestConfig.chipset)]
50+
)
51+
(
52+
tester.quantize(cali_dataset=cali, checker_config=checker_config)
53+
.export()
54+
.to_edge_transform_and_lower()
55+
.to_executorch()
56+
.run_method_and_compare_outputs(inputs=example_input, atol=1, rtol=1)
57+
)

‎backends/samsung/test/models/test_edsr.py‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) Samsung Electronics Co. LTD
1+
# Copyright (c) 2025 Samsung Electronics Co. LTD
22
# All rights reserved
33
#
44
# Licensed under the BSD License (the "License"); you may not use this file
@@ -12,6 +12,10 @@
1212
gen_samsung_backend_compile_spec,
1313
)
1414
from executorch.backends.samsung.test.tester import SamsungTester
15+
from executorch.backends.samsung.test.utils.datasets import (
16+
get_quant_test_data_super_resolution,
17+
)
18+
from executorch.backends.samsung.test.utils.quant_checkers import CheckerConfig
1519
from executorch.backends.samsung.test.utils.utils import TestConfig
1620
from executorch.examples.models.edsr import EdsrModel
1721

@@ -29,3 +33,22 @@ def test_edsr_fp16(self):
2933
.to_executorch()
3034
.run_method_and_compare_outputs(inputs=example_input, atol=0.02)
3135
)
36+
37+
def test_edsr_a8w8(self):
38+
example_input, cali, testdata = get_quant_test_data_super_resolution(
39+
os.path.join(os.environ["DATASET_PATH"]), "B100"
40+
)
41+
model = EdsrModel().get_eager_model()
42+
checker_config = CheckerConfig(
43+
"super_resolution", {"dataset": testdata, "threshold": 0.7}
44+
)
45+
tester = SamsungTester(
46+
model, example_input, [gen_samsung_backend_compile_spec(TestConfig.chipset)]
47+
)
48+
(
49+
tester.quantize(cali_dataset=cali, checker_config=checker_config)
50+
.export()
51+
.to_edge_transform_and_lower()
52+
.to_executorch()
53+
.run_method_and_compare_outputs(inputs=example_input, atol=1, rtol=1)
54+
)

‎backends/samsung/test/models/test_inception_v3.py‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
1-
# Copyright (c) Samsung Electronics Co. LTD
1+
# Copyright (c) 2025 Samsung Electronics Co. LTD
22
# All rights reserved
33
#
44
# Licensed under the BSD License (the "License"); you may not use this file
55
# except in compliance with the License. See the license file in the root
66
# directory of this source tree for more details.
77

88

9+
import os
910
import unittest
1011

1112
import torch
@@ -14,6 +15,8 @@
1415
gen_samsung_backend_compile_spec,
1516
)
1617
from executorch.backends.samsung.test.tester import SamsungTester
18+
from executorch.backends.samsung.test.utils.datasets import get_quant_test_data_classify
19+
from executorch.backends.samsung.test.utils.quant_checkers import CheckerConfig
1720
from executorch.backends.samsung.test.utils.utils import TestConfig
1821
from executorch.examples.models.inception_v3 import InceptionV3Model
1922

@@ -32,3 +35,25 @@ def test_inception_v3_fp16(self):
3235
.to_executorch()
3336
.run_method_and_compare_outputs(inputs=example_input, atol=0.02, rtol=0.02)
3437
)
38+
39+
def test_inception_v3_a8w8(self):
40+
example_input, cali, testdata = get_quant_test_data_classify(
41+
os.path.join(os.environ["DATASET_PATH"], "imagenet_ptq_subset")
42+
)
43+
checker_config = CheckerConfig(
44+
"classifier",
45+
{
46+
"dataset": testdata,
47+
},
48+
)
49+
model = InceptionV3Model().get_eager_model()
50+
tester = SamsungTester(
51+
model, example_input, [gen_samsung_backend_compile_spec(TestConfig.chipset)]
52+
)
53+
(
54+
tester.quantize(cali_dataset=cali, checker_config=checker_config)
55+
.export()
56+
.to_edge_transform_and_lower()
57+
.to_executorch()
58+
.run_method_and_compare_outputs(inputs=example_input, atol=1, rtol=1)
59+
)

‎backends/samsung/test/models/test_inception_v4.py‎

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) Samsung Electronics Co. LTD
1+
# Copyright (c) 2025 Samsung Electronics Co. LTD
22
# All rights reserved
33
#
44
# Licensed under the BSD License (the "License"); you may not use this file
@@ -13,8 +13,11 @@
1313
gen_samsung_backend_compile_spec,
1414
)
1515
from executorch.backends.samsung.test.tester import SamsungTester
16+
from executorch.backends.samsung.test.utils.datasets import get_quant_test_data_classify
17+
from executorch.backends.samsung.test.utils.quant_checkers import CheckerConfig
1618
from executorch.backends.samsung.test.utils.utils import TestConfig
1719
from executorch.examples.models.inception_v4 import InceptionV4Model
20+
from torchvision import transforms
1821

1922

2023
def patch_iv4(weight_path: str):
@@ -66,3 +69,36 @@ def test_inception_v4_fp16(self):
6669
.to_executorch()
6770
.run_method_and_compare_outputs(inputs=example_input, atol=0.02, rtol=0.02)
6871
)
72+
73+
def test_inception_v4_a8w8(self):
74+
transform_compose = transforms.Compose(
75+
[
76+
transforms.Resize((299, 299)),
77+
transforms.ToTensor(),
78+
transforms.Normalize(
79+
mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]
80+
),
81+
transforms.Lambda(lambda x: x.unsqueeze(0)), # Add batch dim
82+
]
83+
)
84+
example_input, cali, testdata = get_quant_test_data_classify(
85+
os.path.join(os.environ["DATASET_PATH"], "imagenet_ptq_subset"),
86+
transform_compose=transform_compose,
87+
)
88+
checker_config = CheckerConfig(
89+
"classifier",
90+
{
91+
"dataset": testdata,
92+
},
93+
)
94+
model = InceptionV4Model().get_eager_model()
95+
tester = SamsungTester(
96+
model, example_input, [gen_samsung_backend_compile_spec(TestConfig.chipset)]
97+
)
98+
(
99+
tester.quantize(cali_dataset=cali, checker_config=checker_config)
100+
.export()
101+
.to_edge_transform_and_lower()
102+
.to_executorch()
103+
.run_method_and_compare_outputs(inputs=example_input, atol=1, rtol=1)
104+
)

‎backends/samsung/test/models/test_mobilebert_finetuning.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) Samsung Electronics Co. LTD
1+
# Copyright (c) 2025 Samsung Electronics Co. LTD
22
# All rights reserved
33
#
44
# Licensed under the BSD License (the "License"); you may not use this file

‎backends/samsung/test/models/test_mobilenet_v2.py‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,18 @@
1-
# Copyright (c) Samsung Electronics Co. LTD
1+
# Copyright (c) 2025 Samsung Electronics Co. LTD
22
# All rights reserved
33
#
44
# Licensed under the BSD License (the "License"); you may not use this file
55
# except in compliance with the License. See the license file in the root
66
# directory of this source tree for more details.
7+
import os
78
import unittest
89

910
from executorch.backends.samsung.serialization.compile_options import (
1011
gen_samsung_backend_compile_spec,
1112
)
1213
from executorch.backends.samsung.test.tester import SamsungTester
14+
from executorch.backends.samsung.test.utils.datasets import get_quant_test_data_classify
15+
from executorch.backends.samsung.test.utils.quant_checkers import CheckerConfig
1316
from executorch.backends.samsung.test.utils.utils import TestConfig
1417
from executorch.examples.models.mobilenet_v2 import MV2Model
1518

@@ -27,3 +30,25 @@ def test_mv2_fp16(self):
2730
.to_executorch()
2831
.run_method_and_compare_outputs(inputs=example_input, atol=0.02)
2932
)
33+
34+
def test_mv2_a8w8(self):
35+
example_input, cali, testdata = get_quant_test_data_classify(
36+
os.path.join(os.environ["DATASET_PATH"], "imagenet_ptq_subset")
37+
)
38+
checker_config = CheckerConfig(
39+
"classifier",
40+
{
41+
"dataset": testdata,
42+
},
43+
)
44+
model = MV2Model().get_eager_model()
45+
tester = SamsungTester(
46+
model, example_input, [gen_samsung_backend_compile_spec(TestConfig.chipset)]
47+
)
48+
(
49+
tester.quantize(cali_dataset=cali, checker_config=checker_config)
50+
.export()
51+
.to_edge_transform_and_lower()
52+
.to_executorch()
53+
.run_method_and_compare_outputs(inputs=example_input, atol=1, rtol=1)
54+
)

‎backends/samsung/test/models/test_mobilenet_v3.py‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
1-
# Copyright (c) Samsung Electronics Co. LTD
1+
# Copyright (c) 2025 Samsung Electronics Co. LTD
22
# All rights reserved
33
#
44
# Licensed under the BSD License (the "License"); you may not use this file
55
# except in compliance with the License. See the license file in the root
66
# directory of this source tree for more details.
77

88

9+
import os
910
import unittest
1011

1112
import torch
@@ -14,6 +15,8 @@
1415
gen_samsung_backend_compile_spec,
1516
)
1617
from executorch.backends.samsung.test.tester import SamsungTester
18+
from executorch.backends.samsung.test.utils.datasets import get_quant_test_data_classify
19+
from executorch.backends.samsung.test.utils.quant_checkers import CheckerConfig
1720
from executorch.backends.samsung.test.utils.utils import TestConfig
1821
from executorch.examples.models.mobilenet_v3 import MV3Model
1922

@@ -32,3 +35,26 @@ def test_mv3_fp16(self):
3235
.to_executorch()
3336
.run_method_and_compare_outputs(inputs=example_input, atol=0.07, rtol=0.07)
3437
)
38+
39+
def test_mv3_a8w8(self):
40+
example_input, cali, testdata = get_quant_test_data_classify(
41+
os.path.join(os.environ["DATASET_PATH"], "imagenet_ptq_subset")
42+
)
43+
checker_config = CheckerConfig(
44+
"classifier",
45+
{
46+
"dataset": testdata,
47+
"topktol": {1: 0.0, 2: 0.0},
48+
},
49+
)
50+
model = MV3Model().get_eager_model()
51+
tester = SamsungTester(
52+
model, example_input, [gen_samsung_backend_compile_spec(TestConfig.chipset)]
53+
)
54+
(
55+
tester.quantize(cali_dataset=cali, checker_config=checker_config)
56+
.export()
57+
.to_edge_transform_and_lower()
58+
.to_executorch()
59+
.run_method_and_compare_outputs(inputs=example_input, atol=3, rtol=3)
60+
)

‎backends/samsung/test/models/test_resnet18.py‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,20 @@
1-
# Copyright (c) Samsung Electronics Co. LTD
1+
# Copyright (c) 2025 Samsung Electronics Co. LTD
22
# All rights reserved
33
#
44
# Licensed under the BSD License (the "License"); you may not use this file
55
# except in compliance with the License. See the license file in the root
66
# directory of this source tree for more details.
77

88

9+
import os
910
import unittest
1011

1112
from executorch.backends.samsung.serialization.compile_options import (
1213
gen_samsung_backend_compile_spec,
1314
)
1415
from executorch.backends.samsung.test.tester import SamsungTester
16+
from executorch.backends.samsung.test.utils.datasets import get_quant_test_data_classify
17+
from executorch.backends.samsung.test.utils.quant_checkers import CheckerConfig
1518
from executorch.backends.samsung.test.utils.utils import TestConfig
1619
from executorch.examples.models.resnet import ResNet18Model
1720

@@ -29,3 +32,25 @@ def test_resnet18_fp16(self):
2932
.to_executorch()
3033
.run_method_and_compare_outputs(inputs=example_input, atol=0.02, rtol=0.02)
3134
)
35+
36+
def test_resnet18_a8w8(self):
37+
example_input, cali, testdata = get_quant_test_data_classify(
38+
os.path.join(os.environ["DATASET_PATH"], "imagenet_ptq_subset")
39+
)
40+
checker_config = CheckerConfig(
41+
"classifier",
42+
{
43+
"dataset": testdata,
44+
},
45+
)
46+
model = ResNet18Model().get_eager_model()
47+
tester = SamsungTester(
48+
model, example_input, [gen_samsung_backend_compile_spec(TestConfig.chipset)]
49+
)
50+
(
51+
tester.quantize(cali_dataset=cali, checker_config=checker_config)
52+
.export()
53+
.to_edge_transform_and_lower()
54+
.to_executorch()
55+
.run_method_and_compare_outputs(inputs=example_input, atol=1, rtol=1)
56+
)

‎backends/samsung/test/models/test_resnet50.py‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,20 @@
1-
# Copyright (c) Samsung Electronics Co. LTD
1+
# Copyright (c) 2025 Samsung Electronics Co. LTD
22
# All rights reserved
33
#
44
# Licensed under the BSD License (the "License"); you may not use this file
55
# except in compliance with the License. See the license file in the root
66
# directory of this source tree for more details.
77

88

9+
import os
910
import unittest
1011

1112
from executorch.backends.samsung.serialization.compile_options import (
1213
gen_samsung_backend_compile_spec,
1314
)
1415
from executorch.backends.samsung.test.tester import SamsungTester
16+
from executorch.backends.samsung.test.utils.datasets import get_quant_test_data_classify
17+
from executorch.backends.samsung.test.utils.quant_checkers import CheckerConfig
1518
from executorch.backends.samsung.test.utils.utils import TestConfig
1619
from executorch.examples.models.resnet import ResNet50Model
1720

@@ -29,3 +32,25 @@ def test_resnet50_fp16(self):
2932
.to_executorch()
3033
.run_method_and_compare_outputs(inputs=example_input, atol=0.02, rtol=0.02)
3134
)
35+
36+
def test_resnet50_a8w8(self):
37+
example_input, cali, testdata = get_quant_test_data_classify(
38+
os.path.join(os.environ["DATASET_PATH"], "imagenet_ptq_subset")
39+
)
40+
checker_config = CheckerConfig(
41+
"classifier",
42+
{
43+
"dataset": testdata,
44+
},
45+
)
46+
model = ResNet50Model().get_eager_model()
47+
tester = SamsungTester(
48+
model, example_input, [gen_samsung_backend_compile_spec(TestConfig.chipset)]
49+
)
50+
(
51+
tester.quantize(cali_dataset=cali, checker_config=checker_config)
52+
.export()
53+
.to_edge_transform_and_lower()
54+
.to_executorch()
55+
.run_method_and_compare_outputs(inputs=example_input, atol=1, rtol=1)
56+
)

0 commit comments

Comments
 (0)