Skip to content

Commit aad14d1

Browse files
authored
[5963347] Expose iterator interface for calibration data (#1075)
### What does this PR do? Expose the internal calibration_data_reader parameter to the public quantize function to allow providing an external CalibrationDataReader object. This enables an iterator style interface for large calibration sets. <!-- Details about the change. --> ### Usage ```python input_tensor = torch.randn(2, 16, 16) # Calibration data comes from a custom data reader, enabling iterator based reading functionality class ExampleCalibrationDataReader(CalibrationDataReader): def __init__(self, input_data): self.data_list = [{"input": input_data.numpy()}] self.iter = iter(self.data_list) def get_next(self): return next(self.iter, None) def get_first(self): return self.data_list[0] def rewind(self): self.iter = iter(self.data_list) calibration_reader = ExampleCalibrationDataReader(input_tensor) moq.quantize( onnx_path, quantize_mode="int8", high_precision_dtype=high_precision_dtype, calibration_data_reader=calibration_reader, ) ``` ### Testing Unit test added to exercise this interface. ### Before your PR is "*Ready for review*" - Is this change backward compatible?: ✅ - If you copied code from any other sources or added a new PIP dependency, did you follow guidance in `CONTRIBUTING.md`: N/A - Did you write any new necessary tests?: ✅ - Did you update [Changelog](https://github.com/NVIDIA/Model-Optimizer/blob/main/CHANGELOG.rst)?: ✅ ### Additional Information Resolves bug: 5963347 <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * ONNX quantization accepts an optional calibration-data reader, allowing users to supply custom calibration workflows while preserving existing behavior when none is provided. * **Documentation** * Changelog updated to document the new quantization workflow option. * **Tests** * Added tests validating both custom calibration-reader and precomputed calibration-data workflows. <!-- end of auto-generated comment: release notes by coderabbit.ai --> --------- Signed-off-by: dmoodie <dmoodie@nvidia.com>
1 parent 08e5f92 commit aad14d1

3 files changed

Lines changed: 64 additions & 10 deletions

File tree

CHANGELOG.rst

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
NVIDIA Model Optimizer Changelog
22
================================
3+
0.44 (2026-04-xx)
4+
5+
**New Features**
6+
- Added iterator interface using CalibrationDataReader in ONNX quantization workflow.
7+
38

49
0.44 (2026-05-xx)
510
^^^^^^^^^^^^^^^^^

modelopt/onnx/quantization/quantize.py

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
import onnx.onnx_cpp2py_export.checker as C
4444
import onnx_graphsurgeon as gs
4545
import onnxslim
46+
from onnxruntime.quantization.calibrate import CalibrationDataReader
4647

4748
from modelopt.onnx.logging_config import configure_logging, logger
4849
from modelopt.onnx.op_types import is_data_dependent_shape_op
@@ -319,6 +320,7 @@ def quantize(
319320
calibration_data: CalibrationDataType = None,
320321
calibration_method: str | None = None,
321322
calibration_cache_path: str | None = None,
323+
calibration_data_reader: CalibrationDataReader | None = None,
322324
calibration_shapes: str | None = None,
323325
calibration_eps: list[str] = ["cpu", "cuda:0", "trt"],
324326
override_shapes: str | None = None,
@@ -375,6 +377,8 @@ def quantize(
375377
and int4: {'awq_clip' (default), 'awq_lite', 'awq_full', 'rtn_dq'}.
376378
calibration_cache_path:
377379
Path to pre-calculated activation tensor ranges, also known as calibration cache.
380+
calibration_data_reader:
381+
Instance of a CalibrationDataReader object to provide calibration data.
378382
calibration_shapes:
379383
Input shapes used for calibration process.
380384
It should be provided as a string representing the shape of each input tensors for one calibration step.
@@ -585,13 +589,14 @@ def quantize(
585589
)
586590
trt_plugins = update_trt_ep_support(calibration_eps, has_dds_op, has_custom_op, trt_plugins) # type: ignore[arg-type]
587591

588-
# Use random scales if calibration data is not supplied
589-
if calibration_data is None:
590-
calibration_data_reader = RandomDataProvider(onnx_path, calibration_shapes)
591-
else:
592-
calibration_data_reader = CalibrationDataProvider(
593-
onnx_path, calibration_data, calibration_shapes
594-
)
592+
if calibration_data_reader is None:
593+
# Use random scales if calibration data is not supplied
594+
if calibration_data is None:
595+
calibration_data_reader = RandomDataProvider(onnx_path, calibration_shapes)
596+
else:
597+
calibration_data_reader = CalibrationDataProvider(
598+
onnx_path, calibration_data, calibration_shapes
599+
)
595600

596601
nodes_to_quantize = nodes_to_quantize or []
597602
nodes_to_exclude = nodes_to_exclude or []

tests/unit/onnx/quantization/test_quantize_int8.py

Lines changed: 47 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import pytest
2121
import torch
2222
from _test_utils.onnx.lib_test_models import SimpleMLP, export_as_onnx
23+
from onnxruntime.quantization.calibrate import CalibrationDataReader
2324

2425
import modelopt.onnx.quantization as moq
2526

@@ -34,14 +35,15 @@ def assert_nodes_are_quantized(nodes):
3435
return True
3536

3637

37-
@pytest.mark.parametrize("high_precision_dtype", ["fp32", "fp16", "bf16"])
38-
def test_int8(tmp_path, high_precision_dtype):
38+
def int8_test_helper(tmp_path, high_precision_dtype, **kwargs):
3939
model_torch = SimpleMLP()
4040
input_tensor = torch.randn(2, 16, 16)
4141

4242
onnx_path = os.path.join(tmp_path, "model.onnx")
4343
export_as_onnx(model_torch, input_tensor, onnx_filename=onnx_path)
44-
moq.quantize(onnx_path, quantize_mode="int8", high_precision_dtype=high_precision_dtype)
44+
moq.quantize(
45+
onnx_path, quantize_mode="int8", high_precision_dtype=high_precision_dtype, **kwargs
46+
)
4547

4648
# Output model should be produced in the same tmp_path
4749
output_onnx_path = onnx_path.replace(".onnx", ".quant.onnx")
@@ -55,3 +57,45 @@ def test_int8(tmp_path, high_precision_dtype):
5557
# Check that all MatMul nodes are quantized
5658
mm_nodes = [n for n in graph.nodes if n.op == "MatMul"]
5759
assert assert_nodes_are_quantized(mm_nodes)
60+
61+
62+
@pytest.mark.parametrize("high_precision_dtype", ["fp32", "fp16", "bf16"])
63+
def test_int8(tmp_path, high_precision_dtype):
64+
int8_test_helper(tmp_path, high_precision_dtype)
65+
66+
67+
@pytest.mark.parametrize("high_precision_dtype", ["fp32", "fp16", "bf16"])
68+
def test_int8_with_calibration_reader(tmp_path, high_precision_dtype):
69+
input_tensor = torch.randn(2, 16, 16)
70+
71+
# Calibration data comes from a custom data reader, enabling iterator based reading functionality
72+
class ExampleCalibrationDataReader(CalibrationDataReader):
73+
def __init__(self, input_data):
74+
self.data_list = [{"input": input_data.numpy()}]
75+
self.iter = iter(self.data_list)
76+
self.get_first_calls = 0
77+
self.get_next_calls = 0
78+
79+
def get_next(self):
80+
self.get_next_calls += 1
81+
return next(self.iter, None)
82+
83+
def get_first(self):
84+
self.get_first_calls += 1
85+
return self.data_list[0]
86+
87+
def rewind(self):
88+
self.iter = iter(self.data_list)
89+
90+
calibration_reader = ExampleCalibrationDataReader(input_tensor)
91+
int8_test_helper(tmp_path, high_precision_dtype, calibration_data_reader=calibration_reader)
92+
assert calibration_reader.get_first_calls > 0 or calibration_reader.get_next_calls > 0
93+
94+
95+
@pytest.mark.parametrize("high_precision_dtype", ["fp32", "fp16", "bf16"])
96+
def test_int8_with_calibration_data(tmp_path, high_precision_dtype):
97+
input_tensor = torch.randn(2, 16, 16)
98+
99+
# test pre-allocated calibration data pathway
100+
calibration_data = {"input": input_tensor.numpy()}
101+
int8_test_helper(tmp_path, high_precision_dtype, calibration_data=calibration_data)

0 commit comments

Comments
 (0)