observatory: promote observatory & fx_viewer to shared devtools

Move the Observatory debugging framework and fx_viewer graph visualization
from backends/qualcomm/ to devtools/ so all ExecuTorch backends can use
them. Backend-specific patches (QNN ptq_calibrate, XNNPACK quantize, QNN
dataset capture) are extracted into their respective backend directories
with a register_backend_patches() hook mechanism.

Each backend maintains its own CLI runner (--accuracy opt-in flag):
- python -m executorch.devtools.observatory (generic)
- python -m executorch.backends.qualcomm.debugger.observatory (QNN)
- python -m executorch.backends.xnnpack.debugger.observatory (XNNPACK)

Includes RFC document at devtools/observatory/RFC_OBSERVATORY_SHARED_FRAMEWORK.md.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: quic-boyuc

backends/qualcomm/debugger/observatory/Questions.md

@@ -3,7 +3,3 @@
 #
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
-
-from .observatory import Observatory
-
-__all__ = ["Observatory"]

backends/qualcomm/debugger/observatory/__init__.py

@@ -4,277 +4,75 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
-"""Observatory CLI — run mode and visualize mode.
+"""Qualcomm Observatory CLI — QNN-specific lens configuration.
 
-Run mode (default):
-    python -m backends.qualcomm.debugger.observatory.cli \\
-        [--no-accuracy] [--json-only] [--report-title TITLE] \\
-        [--report-dir DIR] [--report-html PATH] [--report-json PATH] \\
-        SCRIPT [SCRIPT_ARGS...]
+Graph collection only (default):
+    python -m executorch.backends.qualcomm.debugger.observatory SCRIPT [ARGS...]
 
-Visualize mode (JSON -> HTML, no script execution):
-    python -m backends.qualcomm.debugger.observatory.cli visualize \\
-        --input report.json --output report.html [--title "My Report"]
+With accuracy debugging:
+    python -m executorch.backends.qualcomm.debugger.observatory --accuracy SCRIPT [ARGS...]
 
-See backends/qualcomm/debugger/observatory/USAGE.md for full workflow docs.
+Visualize mode (JSON -> HTML):
+    python -m executorch.backends.qualcomm.debugger.observatory visualize \\
+        --input report.json --output report.html [--title "My Report"]
 """
 
 from __future__ import annotations
 
-import logging
-import os
-import runpy
 import sys
 
-logging.basicConfig(level=logging.INFO, format="%(message)s")
-
-
-def _usage_run() -> str:
-    return (
-        "Usage: python -m backends.qualcomm.debugger.observatory.cli "
-        "[--no-accuracy] [--json-only] [--no-report] "
-        "[--report-title TITLE] [--report-dir DIR] "
-        "[--report-html PATH] [--report-json PATH] "
-        "SCRIPT [SCRIPT_ARGS...]"
-    )
-
-
-def _usage_visualize() -> str:
-    return (
-        "Usage: python -m backends.qualcomm.debugger.observatory.cli visualize "
-        "--input report.json --output report.html [--title TITLE]"
-    )
-
-
-def _read_value(argv: list[str], index: int, flag_name: str) -> tuple[str, int]:
-    """Read next token as value for a flag and return (value, next_index)."""
-    if index + 1 >= len(argv):
-        print(f"Missing value for {flag_name}")
-        print(_usage_run())
-        sys.exit(1)
-    return argv[index + 1], index + 1
 
-
-def _parse_visualize_args() -> tuple[str, str, str]:
-    """Parse argv for: cli visualize --input X --output Y [--title T]"""
-    argv = sys.argv[2:]  # skip 'visualize' token
-    input_path = None
-    output_path = None
-    title = "Observatory Report"
-
-    i = 0
-    while i < len(argv):
-        arg = argv[i]
-        if arg in ("--input", "-i"):
-            input_path, i = _read_value(argv, i, arg)
-        elif arg.startswith("--input="):
-            input_path = arg.split("=", 1)[1]
-        elif arg in ("--output", "-o"):
-            output_path, i = _read_value(argv, i, arg)
-        elif arg.startswith("--output="):
-            output_path = arg.split("=", 1)[1]
-        elif arg == "--title":
-            title, i = _read_value(argv, i, arg)
-        elif arg.startswith("--title="):
-            title = arg.split("=", 1)[1]
-        elif arg in ("-h", "--help"):
-            print(_usage_visualize())
-            sys.exit(0)
-        else:
-            print(f"Unknown flag for visualize: {arg}")
-            print(_usage_visualize())
-            sys.exit(1)
-        i += 1
-
-    if not input_path or not output_path:
-        print("visualize requires --input and --output")
-        print(_usage_visualize())
-        sys.exit(1)
-
-    return input_path, output_path, title
-
-
-def _run_visualize() -> None:
-    input_path, output_path, title = _parse_visualize_args()
-
-    if not os.path.isfile(input_path):
-        logging.error("[Observatory CLI] Input JSON not found: %s", input_path)
-        sys.exit(1)
-
-    from .observatory import Observatory
-
-    Observatory.clear()
-    Observatory.generate_html_from_json(input_path, output_path, title=title)
-    logging.info(
-        "[Observatory CLI] visualize: html=%s from json=%s", output_path, input_path
+def main():
+    from executorch.devtools.observatory.cli import (
+        parse_observatory_args,
+        run_observatory,
+        run_visualize,
     )
 
-
-def _parse_observatory_args():
-    """Extract observatory flags from argv before SCRIPT, then pass all remaining args through."""
-
-    obs_flags = {
-        "--no-accuracy": False,
-        "--no-report": False,
-        "--json-only": False,
-        "--report-title": None,
-        "--report-dir": None,
-        "--report-html": None,
-        "--report-json": None,
-    }
-    script_path = None
-    script_argv = []
-    argv = list(sys.argv[1:])
-
-    i = 0
-    while i < len(argv):
-        arg = argv[i]
-        if script_path is not None:
-            script_argv.append(arg)
-            i += 1
-            continue
-
-        if arg == "--":
-            if i + 1 >= len(argv):
-                print(_usage_run())
-                sys.exit(1)
-            script_path = argv[i + 1]
-            script_argv.extend(argv[i + 2 :])
-            break
-        if arg == "--no-accuracy":
-            obs_flags["--no-accuracy"] = True
-        elif arg == "--no-report":
-            obs_flags["--no-report"] = True
-        elif arg == "--json-only":
-            obs_flags["--json-only"] = True
-        elif arg in ("--report-title", "--obs-report-title"):
-            value, i = _read_value(argv, i, arg)
-            obs_flags["--report-title"] = value
-        elif arg.startswith("--report-title=") or arg.startswith("--obs-report-title="):
-            obs_flags["--report-title"] = arg.split("=", 1)[1]
-        elif arg in ("--report-dir", "--obs-report-dir"):
-            value, i = _read_value(argv, i, arg)
-            obs_flags["--report-dir"] = value
-        elif arg.startswith("--report-dir=") or arg.startswith("--obs-report-dir="):
-            obs_flags["--report-dir"] = arg.split("=", 1)[1]
-        elif arg in ("--report-html", "--obs-report-html"):
-            value, i = _read_value(argv, i, arg)
-            obs_flags["--report-html"] = value
-        elif arg.startswith("--report-html=") or arg.startswith("--obs-report-html="):
-            obs_flags["--report-html"] = arg.split("=", 1)[1]
-        elif arg in ("--report-json", "--obs-report-json"):
-            value, i = _read_value(argv, i, arg)
-            obs_flags["--report-json"] = value
-        elif arg.startswith("--report-json=") or arg.startswith("--obs-report-json="):
-            obs_flags["--report-json"] = arg.split("=", 1)[1]
-        elif arg.startswith("-"):
-            print(f"Unknown observatory flag before SCRIPT: {arg}")
-            print(_usage_run())
-            sys.exit(1)
-        else:
-            script_path = arg
-        i += 1
-
-    if script_path is None:
-        print(_usage_run())
-        sys.exit(1)
-
-    return obs_flags, script_path, script_argv
-
-
-def _infer_report_dir(script_argv: list[str]) -> str:
-    """Infer output directory from target script args (artifact/output_dir), else cwd."""
-    keys = ("-a", "--artifact", "-o", "--output_dir")
-    for idx, arg in enumerate(script_argv):
-        if arg in keys and idx + 1 < len(script_argv):
-            return script_argv[idx + 1]
-        if arg.startswith("--artifact="):
-            return arg.split("=", 1)[1]
-        if arg.startswith("--output_dir="):
-            return arg.split("=", 1)[1]
-    return "."
-
-
-def _resolve_report_paths(obs_flags: dict[str, object], script_argv: list[str]) -> tuple[str, str]:
-    report_dir = obs_flags["--report-dir"] or _infer_report_dir(script_argv)
-    report_html = obs_flags["--report-html"]
-    report_json = obs_flags["--report-json"]
-
-    if report_html is None:
-        report_html = os.path.join(str(report_dir), "observatory_report.html")
-    if report_json is None:
-        if report_html.endswith(".html"):
-            report_json = report_html[:-5] + ".json"
-        else:
-            report_json = os.path.join(str(report_dir), "observatory_report.json")
-
-    return str(report_html), str(report_json)
-
-
-def main():
     if len(sys.argv) > 1 and sys.argv[1] == "visualize":
-        _run_visualize()
+        run_visualize()
         return
 
     if len(sys.argv) > 1 and sys.argv[1] in ("-h", "--help"):
-        print(_usage_run())
-        print()
-        print(_usage_visualize())
+        print(
+            "Usage: python -m executorch.backends.qualcomm.debugger.observatory "
+            "[--accuracy] [--json-only] [--no-report] "
+            "[--report-title TITLE] [--report-dir DIR] "
+            "[--report-html PATH] [--report-json PATH] "
+            "SCRIPT [SCRIPT_ARGS...]"
+        )
         sys.exit(0)
 
-    obs_flags, script_path, script_argv = _parse_observatory_args()
+    usage = (
+        "Usage: python -m executorch.backends.qualcomm.debugger.observatory "
+        "[--accuracy] [--json-only] [--no-report] "
+        "[--report-title TITLE] [--report-dir DIR] "
+        "[--report-html PATH] [--report-json PATH] "
+        "SCRIPT [SCRIPT_ARGS...]"
+    )
+    obs_flags, script_path, script_argv = parse_observatory_args(usage_str=usage)
+
+    from executorch.devtools.observatory.observatory import Observatory
+    from executorch.devtools.observatory.lenses.pipeline_graph_collector import (
+        PipelineGraphCollectorLens,
+    )
 
-    from .observatory import Observatory
-    from .lenses.pipeline_graph_collector import PipelineGraphCollectorLens
+    from .lenses.qnn_patches import install_qnn_patches
 
     Observatory.clear()
-    Observatory.register_lens(PipelineGraphCollectorLens)
-
-    if not obs_flags["--no-accuracy"]:
-        from .lenses.accuracy import AccuracyLens
 
-        Observatory.register_lens(AccuracyLens)
+    PipelineGraphCollectorLens.register_backend_patches(install_qnn_patches)
+    Observatory.register_lens(PipelineGraphCollectorLens)
 
-    sys.argv = [script_path] + script_argv
+    if obs_flags["--accuracy"]:
+        from executorch.devtools.observatory.lenses.accuracy import AccuracyLens
 
-    config: dict = {}
-    report_html, report_json = _resolve_report_paths(obs_flags, script_argv)
+        from .lenses.qnn_dataset_patches import install_qnn_dataset_patches
 
-    title = obs_flags["--report-title"] or f"Observatory: {os.path.basename(script_path)}"
+        AccuracyLens.register_dataset_patches(install_qnn_dataset_patches)
+        Observatory.register_lens(AccuracyLens)
 
-    try:
-        with Observatory.enable_context(config=config):
-            runpy.run_path(script_path, run_name="__main__")
-    except SystemExit:
-        pass
-    except Exception as exc:
-        logging.error("[Observatory CLI] Script raised: %s", exc)
-    finally:
-        if not obs_flags["--no-report"]:
-            os.makedirs(os.path.dirname(report_html) or ".", exist_ok=True)
-            os.makedirs(os.path.dirname(report_json) or ".", exist_ok=True)
-            Observatory.export_json(report_json)
-            collected = Observatory.list_collected()
-            if not obs_flags["--json-only"]:
-                Observatory.export_html_report(report_html, title=title, config=config)
-            if collected:
-                if obs_flags["--json-only"]:
-                    logging.info(
-                        "[Observatory CLI] Reports: json=%s (%d records: %s)",
-                        report_json,
-                        len(collected),
-                        ", ".join(collected),
-                    )
-                else:
-                    logging.info(
-                        "[Observatory CLI] Reports: html=%s json=%s (%d records: %s)",
-                        report_html,
-                        report_json,
-                        len(collected),
-                        ", ".join(collected),
-                    )
-            else:
-                logging.warning("[Observatory CLI] No records collected")
+    run_observatory(obs_flags, script_path, script_argv, Observatory)
 
 
 if __name__ == "__main__":

backends/qualcomm/debugger/observatory/cli.py

@@ -28,8 +28,8 @@
 
 import torch
 
-from executorch.backends.qualcomm.debugger.observatory import Observatory
-from executorch.backends.qualcomm.debugger.observatory.lenses.pipeline_graph_collector import (
+from executorch.devtools.observatory import Observatory
+from executorch.devtools.observatory.lenses.pipeline_graph_collector import (
     PipelineGraphCollectorLens,
 )
 from executorch.devtools.etrecord import ETRecord

backends/qualcomm/debugger/observatory/examples/demo_etrecord_auto_collect.py

@@ -32,8 +32,8 @@
 
 import torch
 
-from executorch.backends.qualcomm.debugger.observatory import Observatory
-from executorch.backends.qualcomm.debugger.observatory.interfaces import (
+from executorch.devtools.observatory import Observatory
+from executorch.devtools.observatory.interfaces import (
     AnalysisResult,
     Frontend,
     Lens,
@@ -44,7 +44,7 @@
     TableRecordSpec,
     ViewList,
 )
-from executorch.backends.qualcomm.utils.fx_viewer import (
+from executorch.devtools.fx_viewer import (
     GraphExtensionNodePayload,
     GraphExtensionPayload,
 )

backends/qualcomm/debugger/observatory/examples/demo_graphview_accuracy_compare.py

@@ -14,8 +14,8 @@
 import json
 import os
 
-from executorch.backends.qualcomm.debugger.observatory.html_template import get_html_template
-from executorch.backends.qualcomm.utils.fx_viewer.exporter import FXGraphExporter
+from executorch.devtools.observatory.html_template import get_html_template
+from executorch.devtools.fx_viewer.exporter import FXGraphExporter
 
 
 def _sample_graph_assets() -> dict: