From da9cb8969417bfa77ca6041c8d5af648f3e32063 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= Date: Sun, 21 Dec 2025 15:42:10 +0100 Subject: [PATCH 1/7] support custom-tracing --- onnx_diagnostic/torch_models/validate.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/onnx_diagnostic/torch_models/validate.py b/onnx_diagnostic/torch_models/validate.py index 748e312c..a0fd12cf 100644 --- a/onnx_diagnostic/torch_models/validate.py +++ b/onnx_diagnostic/torch_models/validate.py @@ -2330,6 +2330,7 @@ def call_torch_export_custom( "custom-dec", "custom-decall", "custom-fake", + "custom-tracing", } assert exporter in available, f"Unexpected value for exporter={exporter!r} in {available}" assert "model" in data, f"model is missing from data: {sorted(data)}" @@ -2342,11 +2343,16 @@ def call_torch_export_custom( f"Options strict cannot be specified in the exporter name {exporter!r} " f"and in the options {exporter_options}" ) + assert ("-tracing" not in exporter) or ("tracing" not in exporter_options), ( + f"Options tracing cannot be specified in the exporter name {exporter!r} " + f"and in the options {exporter_options}" + ) summary: Dict[str, Union[str, int, float]] = {} strict = "-strict" in exporter or exporter_options.pop("strict", False) args, kwargs = split_args_kwargs(data["inputs_export"]) ds = data.get("dynamic_shapes", None) fake = "-fake" in exporter or exporter_options.pop("fake", False) + tracing = "-tracing" in exporter or exporter_options.pop("tracing", False) if fake: from onnx_diagnostic.export.shape_helper import make_fake_with_dynamic_dimensions @@ -2370,6 +2376,7 @@ def call_torch_export_custom( summary["export_exporter"] = exporter summary["export_optimization"] = optimization or "" summary["export_strict"] = strict + summary["export_tracing"] = tracing summary["export_fake"] = fake summary["export_args"] = string_type(args, with_shape=True) summary["export_kwargs"] = string_type(kwargs, with_shape=True) @@ -2392,6 +2399,7 @@ def call_torch_export_custom( ) ) large_model = bool(exporter_options.pop("large_model", True)) + exporter_options.pop("tracing", False) return_optimize_report = bool(exporter_options.pop("return_optimize_report", True)) export_modules_as_functions = bool( exporter_options.pop("export_modules_as_functions", False) @@ -2405,6 +2413,7 @@ def call_torch_export_custom( summary["export_external_threshold"] = str(external_threshold) export_options = ExportOptions( + tracing=tracing, strict=strict, decomposition_table=decomposition_table, save_ep=( From f0a6ead9912484470e93b40dfda5aee5b7dc279e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= Date: Sun, 21 Dec 2025 18:15:52 +0100 Subject: [PATCH 2/7] enable fake tensor more --- .../test_validate_whole_models1.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/_unittests/ut_torch_models/test_validate_whole_models1.py b/_unittests/ut_torch_models/test_validate_whole_models1.py index e6cabc0b..c64c7fad 100644 --- a/_unittests/ut_torch_models/test_validate_whole_models1.py +++ b/_unittests/ut_torch_models/test_validate_whole_models1.py @@ -225,6 +225,21 @@ def test_n_validate_phi35_mini_instruct(self): self.assertIn("If", op_types) self.clean_dump() + @hide_stdout() + def test_o_validate_model_export_fake(self): + mid = "arnir0/Tiny-LLM" + summary, data = validate_model( + mid, + do_run=True, + verbose=10, + exporter="custom-fake", + dump_folder="dump_test/validate_model_export_fake", + patch=True, + ) + self.assertIsInstance(summary, dict) + self.assertIsInstance(data, dict) + self.clean_dump() + if __name__ == "__main__": unittest.main(verbosity=2) From 3d84e35b493e7ddeff0d27d9db1af1a0ffc2d685 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= Date: Mon, 22 Dec 2025 13:24:36 +0100 Subject: [PATCH 3/7] fix fake tensors --- .../test_validate_whole_models1.py | 1 + onnx_diagnostic/export/shape_helper.py | 1 + onnx_diagnostic/helpers/fake_tensor_helper.py | 16 ++++++++++++++-- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/_unittests/ut_torch_models/test_validate_whole_models1.py b/_unittests/ut_torch_models/test_validate_whole_models1.py index c64c7fad..49111fb6 100644 --- a/_unittests/ut_torch_models/test_validate_whole_models1.py +++ b/_unittests/ut_torch_models/test_validate_whole_models1.py @@ -226,6 +226,7 @@ def test_n_validate_phi35_mini_instruct(self): self.clean_dump() @hide_stdout() + @requires_transformers("4.57") def test_o_validate_model_export_fake(self): mid = "arnir0/Tiny-LLM" summary, data = validate_model( diff --git a/onnx_diagnostic/export/shape_helper.py b/onnx_diagnostic/export/shape_helper.py index 1388d325..9b96a0e6 100644 --- a/onnx_diagnostic/export/shape_helper.py +++ b/onnx_diagnostic/export/shape_helper.py @@ -210,6 +210,7 @@ def make_fake_with_dynamic_dimensions( This uses function :func:`onnx_diagnostic.helpers.fake_tensor_helper.make_fake`. Parameter ``existing`` is used to reused the same object when the dynamic dimension is given the same name as another one. + This function works with caches only if ``transformers>=4.57``. A simple tensor: diff --git a/onnx_diagnostic/helpers/fake_tensor_helper.py b/onnx_diagnostic/helpers/fake_tensor_helper.py index ef6c7904..c959f164 100644 --- a/onnx_diagnostic/helpers/fake_tensor_helper.py +++ b/onnx_diagnostic/helpers/fake_tensor_helper.py @@ -144,17 +144,18 @@ def make_fake_with_dynamic_dimensions(self, x: Any, dynamic_shapes: Any) -> Any: """ See :func:`onnx_diagnostic.export.shape_helper.make_fake_with_dynamic_dimensions`. + If caches are used, it requires ``transformers>=4.57``. """ if x is None: return None, None - if isinstance(x, (list, tuple)): + if type(x) in (list, tuple): return x.__class__( [ self.make_fake_with_dynamic_dimensions(i, dynamic_shapes=ds) for i, ds in zip(x, dynamic_shapes) ] ) - if isinstance(x, dict): + if type(x) is dict: return { k: self.make_fake_with_dynamic_dimensions(v, dynamic_shapes=dynamic_shapes[k]) for k, v in x.items() @@ -187,6 +188,17 @@ def make_fake_with_dynamic_dimensions(self, x: Any, dynamic_shapes: Any) -> Any: x.cross_attention_cache, dynamic_shapes=dynamic_shapes[1] ) return x + if x.__class__.__name__ == "BaseModelOutput": + assert ( + list(x.keys()) == ["last_hidden_state"] and x.last_hidden_state is not None + ), ( + f"Field 'last_hidden_state' is empty for {type(x)} or other fields " + f"{list(x.keys())} are used." + ) + x.last_hidden_state = self.make_fake_with_dynamic_dimensions( + x.last_hidden_state, dynamic_shapes=dynamic_shapes[0] + ) + return x if hasattr(x, "shape"): assert dynamic_shapes is None or isinstance(dynamic_shapes, dict), ( f"dynamic_shapes must be a dictionary at this stage but " From 21853a836654a99b6f96b277bfcae0f3b3ef9df8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= Date: Mon, 22 Dec 2025 16:57:10 +0100 Subject: [PATCH 4/7] fix --- .github/workflows/documentation.yml | 4 ++-- CHANGELOGS.rst | 2 ++ onnx_diagnostic/helpers/fake_tensor_helper.py | 7 ++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 3a606c6d..6b659970 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -109,9 +109,9 @@ jobs: grep ERROR doc.txt | grep -v 'l-plot-tiny-llm-export' exit 1 fi - if [[ $(grep WARNING doc.txt | grep -v 'l-plot-tiny-llm-export' | grep -v 'Inline emphasis start-string' | grep -v 'Definition list ends without a blank line' | grep -v 'Unexpected section title or transition' | grep -v 'Inline strong start-string' | grep -v 'MambaCache') ]]; then + if [[ $(grep WARNING doc.txt | grep -v 'l-plot-tiny-llm-export' | grep -v 'Inline emphasis start-string' | grep -v 'Definition list ends without a blank line' | grep -v 'Unexpected section title or transition' | grep -v 'Inline strong start-string' | grep -v 'MambaCache' | grep 'exported_program_dynamic.rst') ]]; then echo "Documentation produces warnings." - grep WARNING doc.txt | grep -v 'l-plot-tiny-llm-export' | grep -v 'Inline emphasis start-string' | grep -v 'Definition list ends without a blank line' | grep -v 'Unexpected section title or transition' | grep -v 'Inline strong start-string' | grep -v 'MambaCache' + grep WARNING doc.txt | grep -v 'l-plot-tiny-llm-export' | grep -v 'Inline emphasis start-string' | grep -v 'Definition list ends without a blank line' | grep -v 'Unexpected section title or transition' | grep -v 'Inline strong start-string' | grep -v 'MambaCache' | grep 'exported_program_dynamic.rst' exit 1 fi diff --git a/CHANGELOGS.rst b/CHANGELOGS.rst index c0a54803..30a1c835 100644 --- a/CHANGELOGS.rst +++ b/CHANGELOGS.rst @@ -4,6 +4,8 @@ Change Logs 0.8.8 +++++ +* :pr:`371`: fix make_fake_with_dynamic_dimensions + 0.8.7 +++++ diff --git a/onnx_diagnostic/helpers/fake_tensor_helper.py b/onnx_diagnostic/helpers/fake_tensor_helper.py index c959f164..c32e1830 100644 --- a/onnx_diagnostic/helpers/fake_tensor_helper.py +++ b/onnx_diagnostic/helpers/fake_tensor_helper.py @@ -209,9 +209,11 @@ def make_fake_with_dynamic_dimensions(self, x: Any, dynamic_shapes: Any) -> Any: for idim, dim in enumerate(x.shape): if dynamic_shapes is not None and idim in dynamic_shapes: s = dynamic_shapes[idim] + if s.__class__.__name__ == "Dim": + s = s.__name__ assert isinstance(s, str), ( f"Unexpected type {type(s)} in dynamic_shapes={dynamic_shapes} " - f"at index {idim}" + f"at index {idim}, self._mapping_str={self._mapping_str}" ) if s in self._mapping_str: dim = self._mapping_str[s] @@ -233,6 +235,9 @@ def make_fake_with_dynamic_dimensions(self, x: Any, dynamic_shapes: Any) -> Any: assert t.device == x.device, f"device mismatch {x.device} -> {t.device}" assert t.dtype == x.dtype, f"dtype mismatch {x.dtype} -> {t.dtype}" return t + if isinstance(x, (int, bool, float)): + # It is a constant, we don't change that. + return x from ..helpers import string_type raise TypeError( From ee96c16c4afaaedb5b8ac78f6d0d36736ae6c82d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= Date: Tue, 23 Dec 2025 10:39:43 +0100 Subject: [PATCH 5/7] fix --- .github/workflows/documentation.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 6b659970..bba1eb11 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -109,9 +109,9 @@ jobs: grep ERROR doc.txt | grep -v 'l-plot-tiny-llm-export' exit 1 fi - if [[ $(grep WARNING doc.txt | grep -v 'l-plot-tiny-llm-export' | grep -v 'Inline emphasis start-string' | grep -v 'Definition list ends without a blank line' | grep -v 'Unexpected section title or transition' | grep -v 'Inline strong start-string' | grep -v 'MambaCache' | grep 'exported_program_dynamic.rst') ]]; then + if [[ $(grep WARNING doc.txt | grep -v 'l-plot-tiny-llm-export' | grep -v 'Inline emphasis start-string' | grep -v 'Definition list ends without a blank line' | grep -v 'Unexpected section title or transition' | grep -v 'Inline strong start-string' | grep -v 'MambaCache' | grep 'Unexpected section title or transition.') ]]; then echo "Documentation produces warnings." - grep WARNING doc.txt | grep -v 'l-plot-tiny-llm-export' | grep -v 'Inline emphasis start-string' | grep -v 'Definition list ends without a blank line' | grep -v 'Unexpected section title or transition' | grep -v 'Inline strong start-string' | grep -v 'MambaCache' | grep 'exported_program_dynamic.rst' + grep WARNING doc.txt | grep -v 'l-plot-tiny-llm-export' | grep -v 'Inline emphasis start-string' | grep -v 'Definition list ends without a blank line' | grep -v 'Unexpected section title or transition' | grep -v 'Inline strong start-string' | grep -v 'MambaCache' | grep 'Unexpected section title or transition.' exit 1 fi From 30a37084c03b36e7da1313cd4809fbe580859c75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= Date: Tue, 23 Dec 2025 12:15:11 +0100 Subject: [PATCH 6/7] fix documentation --- .github/workflows/documentation.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index bba1eb11..fc226af0 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -104,14 +104,14 @@ jobs: - name: Check for errors and warnings run: | - if [[ $(grep ERROR doc.txt | grep -v 'l-plot-tiny-llm-export') ]]; then + if [[ $(grep ERROR doc.txt | grep -v 'l-plot-tiny-llm-export' | grep 'Unexpected section title or transition.') ]]; then echo "Documentation produces errors." - grep ERROR doc.txt | grep -v 'l-plot-tiny-llm-export' + grep ERROR doc.txt | grep -v 'l-plot-tiny-llm-export' | grep 'Unexpected section title or transition.' exit 1 fi - if [[ $(grep WARNING doc.txt | grep -v 'l-plot-tiny-llm-export' | grep -v 'Inline emphasis start-string' | grep -v 'Definition list ends without a blank line' | grep -v 'Unexpected section title or transition' | grep -v 'Inline strong start-string' | grep -v 'MambaCache' | grep 'Unexpected section title or transition.') ]]; then + if [[ $(grep WARNING doc.txt | grep -v 'l-plot-tiny-llm-export' | grep -v 'Inline emphasis start-string' | grep -v 'Definition list ends without a blank line' | grep -v 'Unexpected section title or transition' | grep -v 'Inline strong start-string' | grep -v 'MambaCache' ) ]]; then echo "Documentation produces warnings." - grep WARNING doc.txt | grep -v 'l-plot-tiny-llm-export' | grep -v 'Inline emphasis start-string' | grep -v 'Definition list ends without a blank line' | grep -v 'Unexpected section title or transition' | grep -v 'Inline strong start-string' | grep -v 'MambaCache' | grep 'Unexpected section title or transition.' + grep WARNING doc.txt | grep -v 'l-plot-tiny-llm-export' | grep -v 'Inline emphasis start-string' | grep -v 'Definition list ends without a blank line' | grep -v 'Unexpected section title or transition' | grep -v 'Inline strong start-string' | grep -v 'MambaCache' exit 1 fi From 4cfdaed07ab7f713e0700497eaa846242baa5d04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= Date: Tue, 23 Dec 2025 13:25:13 +0100 Subject: [PATCH 7/7] documentation --- .github/workflows/documentation.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index fc226af0..20c268cc 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -104,9 +104,9 @@ jobs: - name: Check for errors and warnings run: | - if [[ $(grep ERROR doc.txt | grep -v 'l-plot-tiny-llm-export' | grep 'Unexpected section title or transition.') ]]; then + if [[ $(grep ERROR doc.txt | grep -v 'l-plot-tiny-llm-export' | grep -v 'Unexpected section title or transition.') ]]; then echo "Documentation produces errors." - grep ERROR doc.txt | grep -v 'l-plot-tiny-llm-export' | grep 'Unexpected section title or transition.' + grep ERROR doc.txt | grep -v 'l-plot-tiny-llm-export' | grep -v 'Unexpected section title or transition.' exit 1 fi if [[ $(grep WARNING doc.txt | grep -v 'l-plot-tiny-llm-export' | grep -v 'Inline emphasis start-string' | grep -v 'Definition list ends without a blank line' | grep -v 'Unexpected section title or transition' | grep -v 'Inline strong start-string' | grep -v 'MambaCache' ) ]]; then