fix documentation (#370)

* fix documentation

* dump

* fix

* fix validate

* fix aggregation

* documentation

* output statistics

* fix mypy

* add timeout

Author: xadupre

.github/workflows/models448.yml

@@ -12,6 +12,8 @@ jobs:
   run:
     name: to-${{ matrix.torch }}-tr-${{ matrix.transformers }}-ci ${{ matrix.os }}-${{ matrix.python }}
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 15
+    
     strategy:
       fail-fast: false
       matrix:
@@ -64,5 +66,6 @@ jobs:
         run: python -m pip freeze
 
       - name: Phi-4-multimodal-instruct - vision
+        continue-on-error: true
         run: |
           PYTHONPATH=. python -m onnx_diagnostic.ci_models.export_phi4_mm -m microsoft/Phi-4-multimodal-instruct --device cpu --dtype float16 --exporter custom --no-pretrained --no-second-input --atol 100000164640 --mismatch01 1 --part vision

_doc/status/patches_diff.rst

@@ -33,8 +33,8 @@ helps fixing some issues for many models.
 
 Class :class:`PatchDetails <onnx_diagnostic.torch_export_patches.patch_details.PatchDetails>`
 gives an example on how to retrieve the list of involded patches for a specific model.
-Those patches belongs to the following list which depends on transformers and
-pytorch versions.
+Those patches belong to the following list which depends on :epkg:`transformers` and
+:epkg:`pytorch` versions.
 
 .. runpython::
     :showcode:
@@ -62,7 +62,7 @@ Those two versions leads to the following list of patches.
     ):
         pass
     for patch in details.patched:
-        print(f"* {patch.family} - {patch.function_to_patch}")
+        print(f"* {patch.family} - {getattr(patch.function_to_patch, '__name__', patch.function_to_patch)}")
     print()
     print()
     for patch in details.patched:

codecov.yml

@@ -1,3 +1,3 @@
 ignore:
-  - onnx_diagnostic/big_models/*
-  - onnx_diagnostic/ci_models/export*
+  - "onnx_diagnostic/big_models/*"
+  - "onnx_diagnostic/ci_models/export*"

onnx_diagnostic/export/api.py

@@ -1,3 +1,4 @@
+import os
 from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
 import torch
 from .onnx_plug import EagerDirectReplacementWithOnnx
@@ -138,7 +139,7 @@ def to_onnx(
             else None
         )
 
-        return _to_onnx(
+        proto, opt_stats = _to_onnx(
             mod,
             args=args,
             kwargs=kwargs,
@@ -155,11 +156,47 @@ def to_onnx(
             inline=inline,
             dispatcher=main_dispatcher,
             optimize=optimize,
+            return_optimize_report=True,
             **(exporter_kwargs or {}),
         )
+        if opt_stats and filename and os.path.exists(filename):
+            import pandas
+
+            stat_filename = f"{os.path.splitext(filename)[0]}.opt.xlsx"
+            pattern_stats = []
+            for k, v in opt_stats.items():
+                if "time" in k:
+                    pattern_stats.append(dict(level="main", pattern=k, time_in=v))
+            pattern_stats.extend(
+                [{**obs, "level": "detailed"} for obs in opt_stats["optimization"]]
+            )
+            df = pandas.DataFrame(pattern_stats)
+            df.to_excel(stat_filename, index=False)
+            cols = [
+                c
+                for c in [
+                    "level",
+                    "pattern",
+                    "time_in",
+                    "iteration",
+                    "inlined",
+                    "removed",
+                    "added",
+                    "instances",
+                    "changed",
+                    "scale",
+                ]
+                if c in df.columns
+            ]
+            agg = {k: "sum" for k in cols if k not in ("level", "pattern")}
+            agg.update(dict(iteration="max", instances="mean"))
+            agg = {k: v for k, v in agg.items() if k in df.columns}
+            stat_filename = f"{os.path.splitext(filename)[0]}.opt.agg.xlsx"
+            df[cols].groupby(["level", "pattern"]).agg(agg).to_excel(stat_filename)
+
+        return proto
 
     if exporter in ("dynamo", "onnx-dynamo"):
-        import os
         from ..helpers import flatten_object
         import onnxscript.rewriter.ort_fusions as ort_fusions
 
@@ -226,7 +263,6 @@ def to_onnx(
         return epo
 
     if exporter == "modelbuilder":
-        import os
         from ..helpers import flatten_object, string_type
         from ..helpers.model_builder_helper import create_model_builder, save_model_builder
 

onnx_diagnostic/torch_models/validate.py

@@ -2449,6 +2449,41 @@ def call_torch_export_custom(
             )
         ),
     )
+    if "optimization" in opt_stats and dump_folder:
+        import pandas
+
+        pattern_stats = []
+        for k, v in opt_stats.items():
+            if "time" in k:
+                pattern_stats.append(dict(level="main", pattern=k, time_in=v))
+        pattern_stats.extend(
+            [{**obs, "level": "detailed"} for obs in opt_stats["optimization"]]
+        )
+        stat_filename = os.path.join(dump_folder, "optimization_stats.xlsx")
+        df = pandas.DataFrame(pattern_stats)
+        df.to_excel(stat_filename, index=False)
+        cols = [
+            c
+            for c in [
+                "level",
+                "pattern",
+                "time_in",
+                "iteration",
+                "inlined",
+                "removed",
+                "added",
+                "instances",
+                "changed",
+                "scale",
+            ]
+            if c in df.columns
+        ]
+        agg = {k: "sum" for k in cols if k not in ("level", "pattern")}
+        agg.update(dict(iteration="max", instances="mean"))
+        agg = {k: v for k, v in agg.items() if k in df.columns}
+        stat_filename = os.path.join(dump_folder, "optimization_stats.agg.xlsx")
+        df[cols].groupby(["level", "pattern"]).agg(agg).to_excel(stat_filename)
+
     if "ERR_export_onnx_c" in summary:
         return summary, data