refactor: update output handler to support summary and full

Author: aiwantaozi

benchmark_runner/output_summary_json.py

@@ -1,9 +1,11 @@
 """
-Output handler for serializing generative benchmark reports to JSON.
+Output handler for serializing generative benchmark reports to JSON (both summary and full).
 
-This module implements a file-based output for saving benchmark results in JSON format.
-It extends GenerativeBenchmarkerOutput and supports both directory and explicit file path
-output, automatically creating parent directories as needed.
+This module implements a dual-output JSON handler that saves both:
+1. Summary JSON - Excludes large fields like individual requests and detailed metrics
+2. Full JSON - Contains complete benchmark data including all requests
+
+Both files are saved to the same directory with clear naming conventions.
 """
 
 from __future__ import annotations
@@ -18,20 +20,79 @@
 from guidellm.benchmark.outputs.output import GenerativeBenchmarkerOutput
 from guidellm.benchmark.schemas import GenerativeBenchmarksReport
 
-__all__ = ["GenerativeBenchmarkerSummaryJson"]
+__all__ = ["GenerativeBenchmarkerDualJson", "AutoMarshalJSONEncoder"]
+
 
+class AutoMarshalJSONEncoder(json.JSONEncoder):
+    """
+    Custom JSON encoder with auto-marshal support (similar to Golang's MarshalJSON).
 
-@GenerativeBenchmarkerOutput.register("summary_json")
-class GenerativeBenchmarkerSummaryJson(GenerativeBenchmarkerOutput):
+    This encoder automatically checks if objects have __class_json__() or __json__()
+    methods and calls them for serialization, providing a Golang-like interface for
+    custom JSON marshaling in Python.
     """
-    Output handler for serializing benchmark reports to JSON files.
 
-    This class saves generative benchmark reports to a specified file or directory in JSON format.
-    If a directory is provided, a default filename is used. Certain fields can be excluded from the output.
+    def default(self, o):
+        """
+        Override default serialization for non-serializable objects.
+
+        Args:
+            o: Object to serialize.
+
+        Returns:
+            Serializable representation of the object.
+        """
+        # Check if the object has a __class_json__ method (for class objects)
+        if isinstance(o, type) and hasattr(o, "__class_json__"):
+            return o.__class_json__()
+
+        # Check if the object has a __json__ method (for instances)
+        if hasattr(o, "__json__") and callable(getattr(o, "__json__")):
+            return o.__json__()
+
+        # Handle class/type objects (like response handler classes)
+        if isinstance(o, type):
+            # Try to find the registered name for this handler class
+            from guidellm.backends.response_handlers import (
+                GenerationResponseHandlerFactory,
+            )
+
+            registry = GenerationResponseHandlerFactory.registry or {}
+            class_to_name = {v: k for k, v in registry.items()}
+
+            handler_name = class_to_name.get(o)
+            if handler_name:
+                return handler_name
+            else:
+                # Fallback: use the full class name
+                return f"{o.__module__}.{o.__name__}"
+
+        # Let the base class handle other types or raise TypeError
+        return super().default(o)
+
+
+@GenerativeBenchmarkerOutput.register("dual_json")
+class GenerativeBenchmarkerDualJson(GenerativeBenchmarkerOutput):
+    """
+    Output handler for serializing benchmark reports to both summary and full JSON files.
+
+    This class saves two JSON files:
+    1. Summary JSON - Excludes large fields (requests, detailed metrics) for quick overview
+    2. Full JSON - Contains complete benchmark data including all requests and metrics
+
+    If a directory is provided, default filenames are used. If a file path is provided,
+    the summary uses that path and the full version adds a suffix.
 
     Example:
-        output = GenerativeBenchmarkerSummaryJson(output_path="/path/to/output.json")
-        result_path = await output.finalize(report)
+        # Using directory
+        output = GenerativeBenchmarkerDualJson(output_path="/path/to/dir")
+        # Creates: /path/to/dir/benchmarks.json (summary)
+        #          /path/to/dir/benchmarks.full.json (full)
+
+        # Using file path
+        output = GenerativeBenchmarkerDualJson(output_path="/path/to/results.json")
+        # Creates: /path/to/results.json (summary)
+        #          /path/to/results.full.json (full)
     """
 
     DEFAULT_FILE: ClassVar[str] = "benchmarks.json"
@@ -79,7 +140,7 @@ def validated_kwargs(
             output_path = (
                 output_path if isinstance(output_path, Path) else Path(output_path)
             )
-            if output_path.suffix.lower() == ".summary_json":
+            if output_path.suffix.lower() == ".dual_json":
                 output_path = output_path.with_suffix(".json")
             validated["output_path"] = output_path
 
@@ -91,29 +152,45 @@ def validated_kwargs(
 
     async def finalize(self, report: GenerativeBenchmarksReport) -> Path:
         """
-        Serialize and save the benchmark report to the configured output path in JSON format.
+        Serialize and save the benchmark report to both summary and full JSON files.
 
         Args:
             report: The generative benchmarks report to serialize.
         Returns:
-            Path to the saved report file.
+            Path to the saved summary report file.
         """
-        output_path = self.output_path
-        if output_path.is_dir():
-            output_path = output_path / self.DEFAULT_FILE
+        # Determine output paths
+        summary_path = self.output_path
+        if summary_path.is_dir():
+            summary_path = summary_path / self.DEFAULT_FILE
+
+        # Create full path by inserting ".full" before the extension
+        full_path = (
+            summary_path.parent / f"{summary_path.stem}.full{summary_path.suffix}"
+        )
 
-        output_path.parent.mkdir(parents=True, exist_ok=True)
+        # Ensure parent directory exists
+        summary_path.parent.mkdir(parents=True, exist_ok=True)
 
-        # Exclude specified fields from the report, but keep a small error sample
+        # Prepare data
         full_dict = report.model_dump()
         summary_dict = report.model_dump(exclude=self.EXCLUDE_FIELDS)
         self._attach_error_samples(summary_dict, full_dict)
-        save_str = json.dumps(summary_dict, indent=4)
 
-        with output_path.open("w", encoding="utf-8") as file:
-            file.write(save_str)
+        # Use custom encoder to handle response handler classes
+        encoder_cls = AutoMarshalJSONEncoder
+
+        # Save summary JSON
+        summary_str = json.dumps(summary_dict, indent=4, cls=encoder_cls)
+        with summary_path.open("w", encoding="utf-8") as file:
+            file.write(summary_str)
+
+        # Save full JSON
+        full_str = json.dumps(full_dict, indent=4, cls=encoder_cls)
+        with full_path.open("w", encoding="utf-8") as file:
+            file.write(full_str)
 
-        return output_path
+        return summary_path
 
     def _attach_error_samples(
         self, summary_dict: dict[str, Any], full_dict: dict[str, Any]