[ALMIOPEN-1856] Simplify dnn-benchmarking engine comparison CLI

projects/hipdnn/tools/dnn-benchmarking/src/dnn_benchmarking/cli/internal_profiling.py

@@ -24,7 +24,7 @@
 
 from ..common.exceptions import GraphLoadError
 from ..config.benchmark_config import MetricsConfig, SuiteConfig
-from ..execution.suite_runner import _run_single_provider_engine
+from ..execution.suite_runner import _run_single_provider_engine, _set_plugin_path
 from ..graph.loader import GraphLoader
 
 
@@ -50,8 +50,14 @@ def run_internal_profiling(args: argparse.Namespace) -> int:
         return 1
 
     try:
+        plugin_path = None
         if args.plugin_path is not None:
-            hipdnn.set_engine_plugin_paths([str(args.plugin_path)])
+            plugin_path = (
+                args.plugin_path[0]
+                if isinstance(args.plugin_path, list)
+                else args.plugin_path
+            )
+        _set_plugin_path(hipdnn, plugin_path)
         handle = hipdnn.Handle()
     except RuntimeError as e:
         print(
@@ -88,7 +94,7 @@ def run_internal_profiling(args: argparse.Namespace) -> int:
         reference_provider="none",
         verbose=False,
         metrics=MetricsConfig(tier="off"),
-        plugin_path=args.plugin_path,
+        plugin_path=plugin_path,
     )
 
     try:
@@ -101,6 +107,7 @@ def run_internal_profiling(args: argparse.Namespace) -> int:
             handle=handle,
             provider="profiling-inner",
             engine_id=engine_id,
+            plugin_path=plugin_path,
             ref_provider=None,
             validation_requested=False,
             graph_json=graph_json,

projects/hipdnn/tools/dnn-benchmarking/src/dnn_benchmarking/cli/main.py

@@ -26,7 +26,6 @@
 
 from ..common.exceptions import GraphLoadError
 from ..reporting.reporter import Reporter
-from .ab_runner_cli import run_ab_cli
 from .internal_profiling import run_internal_profiling
 from .parser import create_parser
 from .pytorch_runner_cli import run_pytorch_cli
@@ -82,15 +81,7 @@ def main() -> int:
         return 1
 
     try:
-        if args.AId is not None or args.BId is not None:
-            if len(resolved_files) > 1:
-                reporter.print_error(
-                    "A/B testing requires a single graph file, not a glob pattern"
-                )
-                return 1
-            return run_ab_cli(args, Path(resolved_files[0]), reporter)
-
-        elif args.backend == "pytorch":
+        if args.backend == "pytorch":
             if len(resolved_files) > 1:
                 reporter.print_error(
                     "Suite mode is not supported with --backend pytorch"

projects/hipdnn/tools/dnn-benchmarking/src/dnn_benchmarking/cli/parser.py

@@ -13,14 +13,16 @@ def _parse_engine_list(s: str) -> List[int]:
 
     Engine IDs are deterministic FNV-1a hashes of the engine name and may
     be negative when interpreted as signed int64, so we accept any int.
-    Duplicates are removed while preserving first-seen order.
+    Duplicate IDs are preserved because each comma-delimited entry is an
+    ordered execution selection; this allows comparing the same engine ID
+    from different plugin paths.
 
     Examples:
       "1"                      -> [1]
       "1,2,3"                  -> [1, 2, 3]
       "1, 2"                   -> [1, 2]
-      "1,1,2"                  -> [1, 2]
-      "3,1,3,2"                -> [3, 1, 2]
+      "1,1,2"                  -> [1, 1, 2]
+      "3,1,3,2"                -> [3, 1, 3, 2]
       "-4567890123456789012"   -> [-4567890123456789012]
     """
     parts = [p.strip() for p in s.split(",")]
@@ -31,15 +33,15 @@ def _parse_engine_list(s: str) -> List[int]:
         ids = [int(p) for p in parts]
     except ValueError:
         raise argparse.ArgumentTypeError(f"--engine expects integer ID(s), got {s!r}")
-    # Deduplicate while preserving first-seen order
-    seen: set = set()
-    deduped: List[int] = []
-    for i in ids:
-        if i not in seen:
-            seen.add(i)
-            deduped.append(i)
-    return deduped
+    return ids
 
+def _parse_plugin_path_list(s: str) -> List[Path]:
+    """Parse --plugin-path as a comma-separated list of plugin directories."""
+    parts = [p.strip() for p in s.split(",")]
+    parts = [p for p in parts if p]
+    if not parts:
+        raise argparse.ArgumentTypeError("--plugin-path requires at least one path")
+    return [Path(p) for p in parts]
 
 def create_parser() -> argparse.ArgumentParser:
     """Create the argument parser for dnn-benchmark CLI.
@@ -61,7 +63,7 @@ def create_parser() -> argparse.ArgumentParser:
   dnn-benchmark --graph ./graphs/conv1_fwd.json --warmup 20 --iters 200
   dnn-benchmark -g ./graphs/conv1_fwd.json -e 1
   dnn-benchmark -g ./graphs/conv1_fwd.json -v        # verbose per-engine output
-  dnn-benchmark -g ./graphs/conv1_fwd.json -e 1,2    # compare engines 1 and 2
+  dnn-benchmark -g ./graphs/conv1_fwd.json -e 1,2 --compare-engines
 
 PyTorch Backend (GPU via PyTorch):
   dnn-benchmark -g ./graph.json --backend pytorch
@@ -71,9 +73,9 @@ def create_parser() -> argparse.ArgumentParser:
   dnn-benchmark -g ./graph.json --validate pytorch
   dnn-benchmark -g ./graph.json --validate pytorch --rtol 1e-3
 
-A/B Testing:
-  dnn-benchmark -g ./graph.json --AId 1 --BId 2
-  dnn-benchmark -g ./graph.json --APath /path/pluginA --AId 1 --BPath /path/pluginB --BId 2
+Engine Comparison:
+  dnn-benchmark -g ./graph.json --engine 1,2,3 --compare-engines
+  dnn-benchmark -g ./graph.json --engine 1,2 --plugin-path /path/pluginA,/path/pluginB --compare-engines
 
 Suite Mode (multiple graphs):
   dnn-benchmark -g graphs/                           # all .json/.tar.gz files in directory
@@ -166,38 +168,18 @@ def create_parser() -> argparse.ArgumentParser:
         "(default: summary table)",
     )
 
-    # A/B Testing arguments
-    ab_group = parser.add_argument_group("A/B Testing")
-    ab_group.add_argument(
-        "--APath",
-        type=Path,
-        default=None,
-        metavar="PATH",
-        help="Plugin path for configuration A (default: use system default)",
-    )
-    ab_group.add_argument(
-        "--AId",
-        type=int,
-        default=None,
-        metavar="ID",
-        help="Engine ID for configuration A",
-    )
-    ab_group.add_argument(
-        "--BPath",
-        type=Path,
-        default=None,
-        metavar="PATH",
-        help="Plugin path for configuration B (default: use system default)",
-    )
-    ab_group.add_argument(
-        "--BId",
-        type=int,
-        default=None,
-        metavar="ID",
-        help="Engine ID for configuration B",
-    )
-    # Comparison tolerances (used by A/B testing, validation, and suite mode)
+    # Comparison controls
     comparison_group = parser.add_argument_group("Comparison")
+    comparison_group.add_argument(
+        "--compare-engines",
+        action="store_true",
+        default=False,
+        help=(
+            "Add per-engine comparison columns. The first engine selected by "
+            "--engine is the baseline; when --engine is omitted, the backend's "
+            "first ranked engine is the baseline."
+        ),
+    )
     comparison_group.add_argument(
         "--rtol",
         type=float,
@@ -229,10 +211,14 @@ def create_parser() -> argparse.ArgumentParser:
     suite_group = parser.add_argument_group("Suite Options")
     suite_group.add_argument(
         "--plugin-path",
-        type=Path,
+        type=_parse_plugin_path_list,
         default=None,
-        metavar="DIR",
-        help="Path to directory containing hipDNN engine plugin .so files",
+        metavar="PATHS",
+        help=(
+            "Directory containing hipDNN engine plugin .so files, or a "
+            "comma-separated list matching --engine order. A single path is "
+            "shared by all selected engines."
+        ),
     )
 
     # Metrics options