diff --git a/.github/workflows/models448.yml b/.github/workflows/models448.yml index 889bea1f..486c5e32 100644 --- a/.github/workflows/models448.yml +++ b/.github/workflows/models448.yml @@ -12,6 +12,8 @@ jobs: run: name: to-${{ matrix.torch }}-tr-${{ matrix.transformers }}-ci ${{ matrix.os }}-${{ matrix.python }} runs-on: ${{ matrix.os }} + timeout-minutes: 15 + strategy: fail-fast: false matrix: @@ -64,5 +66,6 @@ jobs: run: python -m pip freeze - name: Phi-4-multimodal-instruct - vision + continue-on-error: true run: | PYTHONPATH=. python -m onnx_diagnostic.ci_models.export_phi4_mm -m microsoft/Phi-4-multimodal-instruct --device cpu --dtype float16 --exporter custom --no-pretrained --no-second-input --atol 100000164640 --mismatch01 1 --part vision diff --git a/_doc/status/patches_diff.rst b/_doc/status/patches_diff.rst index 75576ed4..c2b84241 100644 --- a/_doc/status/patches_diff.rst +++ b/_doc/status/patches_diff.rst @@ -33,8 +33,8 @@ helps fixing some issues for many models. Class :class:`PatchDetails ` gives an example on how to retrieve the list of involded patches for a specific model. -Those patches belongs to the following list which depends on transformers and -pytorch versions. +Those patches belong to the following list which depends on :epkg:`transformers` and +:epkg:`pytorch` versions. .. runpython:: :showcode: @@ -62,7 +62,7 @@ Those two versions leads to the following list of patches. ): pass for patch in details.patched: - print(f"* {patch.family} - {patch.function_to_patch}") + print(f"* {patch.family} - {getattr(patch.function_to_patch, '__name__', patch.function_to_patch)}") print() print() for patch in details.patched: diff --git a/codecov.yml b/codecov.yml index 5df0f7d2..7c548ce7 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,3 +1,3 @@ ignore: - - onnx_diagnostic/big_models/* - - onnx_diagnostic/ci_models/export* + - "onnx_diagnostic/big_models/*" + - "onnx_diagnostic/ci_models/export*" diff --git a/onnx_diagnostic/export/api.py b/onnx_diagnostic/export/api.py index d57ce3bf..57215dcc 100644 --- a/onnx_diagnostic/export/api.py +++ b/onnx_diagnostic/export/api.py @@ -1,3 +1,4 @@ +import os from typing import Any, Dict, List, Optional, Sequence, Tuple, Union import torch from .onnx_plug import EagerDirectReplacementWithOnnx @@ -138,7 +139,7 @@ def to_onnx( else None ) - return _to_onnx( + proto, opt_stats = _to_onnx( mod, args=args, kwargs=kwargs, @@ -155,11 +156,47 @@ def to_onnx( inline=inline, dispatcher=main_dispatcher, optimize=optimize, + return_optimize_report=True, **(exporter_kwargs or {}), ) + if opt_stats and filename and os.path.exists(filename): + import pandas + + stat_filename = f"{os.path.splitext(filename)[0]}.opt.xlsx" + pattern_stats = [] + for k, v in opt_stats.items(): + if "time" in k: + pattern_stats.append(dict(level="main", pattern=k, time_in=v)) + pattern_stats.extend( + [{**obs, "level": "detailed"} for obs in opt_stats["optimization"]] + ) + df = pandas.DataFrame(pattern_stats) + df.to_excel(stat_filename, index=False) + cols = [ + c + for c in [ + "level", + "pattern", + "time_in", + "iteration", + "inlined", + "removed", + "added", + "instances", + "changed", + "scale", + ] + if c in df.columns + ] + agg = {k: "sum" for k in cols if k not in ("level", "pattern")} + agg.update(dict(iteration="max", instances="mean")) + agg = {k: v for k, v in agg.items() if k in df.columns} + stat_filename = f"{os.path.splitext(filename)[0]}.opt.agg.xlsx" + df[cols].groupby(["level", "pattern"]).agg(agg).to_excel(stat_filename) + + return proto if exporter in ("dynamo", "onnx-dynamo"): - import os from ..helpers import flatten_object import onnxscript.rewriter.ort_fusions as ort_fusions @@ -226,7 +263,6 @@ def to_onnx( return epo if exporter == "modelbuilder": - import os from ..helpers import flatten_object, string_type from ..helpers.model_builder_helper import create_model_builder, save_model_builder diff --git a/onnx_diagnostic/torch_models/validate.py b/onnx_diagnostic/torch_models/validate.py index 6b2f48bd..748e312c 100644 --- a/onnx_diagnostic/torch_models/validate.py +++ b/onnx_diagnostic/torch_models/validate.py @@ -2449,6 +2449,41 @@ def call_torch_export_custom( ) ), ) + if "optimization" in opt_stats and dump_folder: + import pandas + + pattern_stats = [] + for k, v in opt_stats.items(): + if "time" in k: + pattern_stats.append(dict(level="main", pattern=k, time_in=v)) + pattern_stats.extend( + [{**obs, "level": "detailed"} for obs in opt_stats["optimization"]] + ) + stat_filename = os.path.join(dump_folder, "optimization_stats.xlsx") + df = pandas.DataFrame(pattern_stats) + df.to_excel(stat_filename, index=False) + cols = [ + c + for c in [ + "level", + "pattern", + "time_in", + "iteration", + "inlined", + "removed", + "added", + "instances", + "changed", + "scale", + ] + if c in df.columns + ] + agg = {k: "sum" for k in cols if k not in ("level", "pattern")} + agg.update(dict(iteration="max", instances="mean")) + agg = {k: v for k, v in agg.items() if k in df.columns} + stat_filename = os.path.join(dump_folder, "optimization_stats.agg.xlsx") + df[cols].groupby(["level", "pattern"]).agg(agg).to_excel(stat_filename) + if "ERR_export_onnx_c" in summary: return summary, data