Skip to content

Commit 144154a

Browse files
quic-boyucquic-boyuc
committed
Fix json float nan issue, simplify cli, prepare rfc draft
1 parent 901a477 commit 144154a

11 files changed

Lines changed: 344 additions & 423 deletions

File tree

‎backends/qualcomm/debugger/observatory/cli.py‎

Lines changed: 29 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,19 @@
44
# This source code is licensed under the BSD-style license found in the
55
# LICENSE file in the root directory of this source tree.
66

7-
"""Qualcomm Observatory CLI — QNN-specific lens configuration.
7+
"""Qualcomm Observatory CLI -- QNN-specific lens configuration.
88
9-
Graph collection only (default):
10-
python -m executorch.backends.qualcomm.debugger.observatory SCRIPT [ARGS...]
9+
Collection mode (default):
10+
python -m executorch.backends.qualcomm.debugger.observatory \\
11+
[--output-html PATH] [--output-json PATH] SCRIPT [SCRIPT_ARGS...]
1112
1213
With accuracy debugging:
13-
python -m executorch.backends.qualcomm.debugger.observatory --accuracy SCRIPT [ARGS...]
14+
python -m executorch.backends.qualcomm.debugger.observatory \\
15+
--lense_recipe=accuracy SCRIPT [SCRIPT_ARGS...]
1416
1517
Visualize mode (JSON -> HTML):
1618
python -m executorch.backends.qualcomm.debugger.observatory visualize \\
17-
--input report.json --output report.html [--title "My Report"]
19+
--input-json report.json --output-html report.html
1820
"""
1921

2022
from __future__ import annotations
@@ -24,60 +26,55 @@
2426

2527
def main():
2628
from executorch.devtools.observatory.cli import (
27-
parse_observatory_args,
29+
make_collect_parser,
30+
make_visualize_parser,
2831
run_observatory,
2932
run_visualize,
3033
)
3134

3235
if len(sys.argv) > 1 and sys.argv[1] == "visualize":
33-
run_visualize()
36+
parser = make_visualize_parser()
37+
args = parser.parse_args(sys.argv[2:])
38+
run_visualize(args.input_json, args.output_html)
3439
return
3540

36-
if len(sys.argv) > 1 and sys.argv[1] in ("-h", "--help"):
37-
print(
38-
"Usage: python -m executorch.backends.qualcomm.debugger.observatory "
39-
"[--accuracy] [--json-only] [--no-report] "
40-
"[--report-title TITLE] [--report-dir DIR] "
41-
"[--report-html PATH] [--report-json PATH] "
42-
"SCRIPT [SCRIPT_ARGS...]"
43-
)
44-
sys.exit(0)
45-
46-
usage = (
47-
"Usage: python -m executorch.backends.qualcomm.debugger.observatory "
48-
"[--accuracy] [--json-only] [--no-report] "
49-
"[--report-title TITLE] [--report-dir DIR] "
50-
"[--report-html PATH] [--report-json PATH] "
51-
"SCRIPT [SCRIPT_ARGS...]"
41+
parser = make_collect_parser(
42+
prog="python -m executorch.backends.qualcomm.debugger.observatory"
43+
)
44+
parser.add_argument(
45+
"--lense_recipe",
46+
choices=["accuracy"],
47+
default=None,
48+
help="Lens recipe to enable (e.g. accuracy)",
5249
)
53-
obs_flags, script_path, script_argv = parse_observatory_args(usage_str=usage)
50+
args = parser.parse_args(sys.argv[1:])
5451

5552
from executorch.devtools.observatory.observatory import Observatory
5653
from executorch.devtools.observatory.lenses.pipeline_graph_collector import (
5754
PipelineGraphCollectorLens,
5855
)
59-
6056
from .lenses.qnn_patches import install_qnn_patches
6157

6258
Observatory.clear()
63-
6459
PipelineGraphCollectorLens.register_backend_patches(install_qnn_patches)
6560
Observatory.register_lens(PipelineGraphCollectorLens)
6661

67-
if obs_flags["--accuracy"]:
62+
if args.lense_recipe == "accuracy":
6863
from executorch.devtools.observatory.lenses.accuracy import AccuracyLens
69-
7064
from .lenses.qnn_dataset_patches import install_qnn_dataset_patches
7165

7266
AccuracyLens.register_dataset_patches(install_qnn_dataset_patches)
7367
Observatory.register_lens(AccuracyLens)
7468

75-
from executorch.devtools.observatory.lenses.per_layer_accuracy import PerLayerAccuracyLens
76-
Observatory.register_lens(PerLayerAccuracyLens)
77-
69+
from executorch.devtools.observatory.lenses.per_layer_accuracy import (
70+
PerLayerAccuracyLens,
71+
)
7872

73+
Observatory.register_lens(PerLayerAccuracyLens)
7974

80-
run_observatory(obs_flags, script_path, script_argv, Observatory)
75+
run_observatory(
76+
args.script, args.script_args, Observatory, args.output_html, args.output_json
77+
)
8178

8279

8380
if __name__ == "__main__":

‎backends/xnnpack/debugger/observatory/cli.py‎

Lines changed: 29 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,19 @@
44
# This source code is licensed under the BSD-style license found in the
55
# LICENSE file in the root directory of this source tree.
66

7-
"""XNNPACK Observatory CLI — XNNPACK-specific lens configuration.
7+
"""XNNPACK Observatory CLI -- XNNPACK-specific lens configuration.
88
9-
Graph collection only (default):
10-
python -m executorch.backends.xnnpack.debugger.observatory SCRIPT [ARGS...]
9+
Collection mode (default):
10+
python -m executorch.backends.xnnpack.debugger.observatory \\
11+
[--output-html PATH] [--output-json PATH] SCRIPT [SCRIPT_ARGS...]
1112
1213
With accuracy debugging:
13-
python -m executorch.backends.xnnpack.debugger.observatory --accuracy SCRIPT [ARGS...]
14+
python -m executorch.backends.xnnpack.debugger.observatory \\
15+
--lense_recipe=accuracy SCRIPT [SCRIPT_ARGS...]
1416
1517
Visualize mode (JSON -> HTML):
1618
python -m executorch.backends.xnnpack.debugger.observatory visualize \\
17-
--input report.json --output report.html [--title "My Report"]
19+
--input-json report.json --output-html report.html
1820
"""
1921

2022
from __future__ import annotations
@@ -24,57 +26,53 @@
2426

2527
def main():
2628
from devtools.observatory.cli import (
27-
parse_observatory_args,
29+
make_collect_parser,
30+
make_visualize_parser,
2831
run_observatory,
2932
run_visualize,
3033
)
3134

3235
if len(sys.argv) > 1 and sys.argv[1] == "visualize":
33-
run_visualize()
36+
parser = make_visualize_parser()
37+
args = parser.parse_args(sys.argv[2:])
38+
run_visualize(args.input_json, args.output_html)
3439
return
3540

36-
if len(sys.argv) > 1 and sys.argv[1] in ("-h", "--help"):
37-
print(
38-
"Usage: python -m executorch.backends.xnnpack.debugger.observatory "
39-
"[--accuracy] [--json-only] [--no-report] "
40-
"[--report-title TITLE] [--report-dir DIR] "
41-
"[--report-html PATH] [--report-json PATH] "
42-
"SCRIPT [SCRIPT_ARGS...]"
43-
)
44-
sys.exit(0)
45-
46-
usage = (
47-
"Usage: python -m executorch.backends.xnnpack.debugger.observatory "
48-
"[--accuracy] [--json-only] [--no-report] "
49-
"[--report-title TITLE] [--report-dir DIR] "
50-
"[--report-html PATH] [--report-json PATH] "
51-
"SCRIPT [SCRIPT_ARGS...]"
41+
parser = make_collect_parser(
42+
prog="python -m executorch.backends.xnnpack.debugger.observatory"
43+
)
44+
parser.add_argument(
45+
"--lense_recipe",
46+
choices=["accuracy"],
47+
default=None,
48+
help="Lens recipe to enable (e.g. accuracy)",
5249
)
53-
obs_flags, script_path, script_argv = parse_observatory_args(usage_str=usage)
50+
args = parser.parse_args(sys.argv[1:])
5451

5552
from devtools.observatory.observatory import Observatory
5653
from devtools.observatory.lenses.pipeline_graph_collector import (
5754
PipelineGraphCollectorLens,
5855
)
59-
6056
from .lenses.xnnpack_patches import install_xnnpack_patches
6157

6258
Observatory.clear()
63-
6459
PipelineGraphCollectorLens.register_backend_patches(install_xnnpack_patches)
6560
Observatory.register_lens(PipelineGraphCollectorLens)
6661

67-
if obs_flags["--accuracy"]:
62+
if args.lense_recipe == "accuracy":
6863
from devtools.observatory.lenses.accuracy import AccuracyLens
6964

7065
Observatory.register_lens(AccuracyLens)
7166

72-
from devtools.observatory.lenses.per_layer_accuracy import PerLayerAccuracyLens
73-
Observatory.register_lens(PerLayerAccuracyLens)
74-
67+
from devtools.observatory.lenses.per_layer_accuracy import (
68+
PerLayerAccuracyLens,
69+
)
7570

71+
Observatory.register_lens(PerLayerAccuracyLens)
7672

77-
run_observatory(obs_flags, script_path, script_argv, Observatory)
73+
run_observatory(
74+
args.script, args.script_args, Observatory, args.output_html, args.output_json
75+
)
7876

7977

8078
if __name__ == "__main__":

‎devtools/observatory/README.md‎

Lines changed: 18 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -36,37 +36,32 @@ A standalone HTML report containing:
3636
### CLI (zero-config)
3737

3838
Point the CLI at any ExecuTorch export script:
39-
The simplest invocation: point the CLI at your script and pass its arguments through.
40-
Use `--report-html` to set output paths explicitly:
4139
```bash
42-
python -m devtools.observatory.cli \
43-
--report-html /path/to/output_report.html \
44-
{your original script and arguments}
40+
python -m executorch.devtools.observatory SCRIPT [SCRIPT_ARGS...]
4541
```
46-
For example:
42+
Use `--output-html` / `--output-json` to set output paths explicitly:
4743

4844
```bash
49-
python -m devtools.observatory.cli \
50-
--report-html /tmp/obs/report.html \
51-
--report-json /tmp/obs/report.json \
52-
--report-title "Swin V2-T Qualcomm" \
45+
python -m executorch.devtools.observatory \
46+
--output-html /tmp/obs/report.html \
47+
--output-json /tmp/obs/report.json \
5348
examples/qualcomm/oss_scripts/swin_v2_t.py \
54-
--model SM8650 -b ./build-android -d imagenet-mini/val -a ./swin_v2_t
49+
--model SM8650 -b ./build-android -d imagenet-mini/val -a ./swin_v2_t
5550
```
5651

5752
Use backend-specific observatory cli for additional customized lenses and hooks (for example, xnnpack backend with per-layer accuracy analysis)
5853

5954
```bash
60-
python -m backends.xnnpack.debugger.observatory.cli \
61-
--report-html /tmp/obs/report.html \
62-
--accuracy \
55+
python -m executorch.backends.xnnpack.debugger.observatory \
56+
--output-html /tmp/obs/report.html \
57+
--lense_recipe=accuracy \
6358
examples/xnnpack/aot_compiler.py \
6459
--model_name=mv2 --delegate --quantize --output_dir /tmp/mv2
6560
```
6661

6762
This produces:
68-
- `/tmp/mv2/observatory_report.html` (interactive report)
69-
- `/tmp/mv2/observatory_report.json` (raw data)
63+
- `/tmp/obs/report.html` (interactive report)
64+
- `/tmp/obs/report.json` (raw data, path auto-derived from HTML path)
7065

7166
### Python API
7267

@@ -125,18 +120,18 @@ See [lenses/LENSES.md](lenses/LENSES.md) for detailed lens documentation.
125120

126121
Observatory separates **runtime collection** from **report generation**:
127122

128-
1. **Step 1**: Run your script, collect data, export JSON (`--json-only`)
129-
2. **Step 2**: Convert JSON to HTML any time later (`cli visualize`)
123+
1. **Step 1**: Run your script — both JSON and HTML are exported automatically
124+
2. **Step 2**: Re-generate HTML any time later from the JSON (`cli visualize`)
130125

131-
This means you can collect data in CI and generate reports locally, or regenerate HTML after updating lens code without re-running expensive export scripts.
126+
This means you can collect data in CI and re-generate reports locally, or regenerate HTML after updating lens code without re-running expensive export scripts.
132127

133128
```bash
134129
# Step 1: collect (e.g., in CI)
135-
python -m devtools.observatory.cli --json-only script.py ...
130+
python -m executorch.devtools.observatory script.py ...
136131

137-
# Step 2: visualize (e.g., locally)
138-
python -m devtools.observatory.cli visualize \
139-
--input report.json --output report.html
132+
# Step 2: re-visualize (e.g., locally)
133+
python -m executorch.devtools.observatory visualize \
134+
--input-json observatory_report.json --output-html report.html
140135
```
141136

142137
### Fx-Viewer
@@ -148,9 +143,7 @@ Fx-Viewer (`devtools/utils/fx_viewer`) is the graph visualization component used
148143
See [USAGE.md](USAGE.md) for the full CLI usage guide, including:
149144

150145
- Zero-config e2e workflow
151-
- JSON-only export for CI
152146
- Visualize mode (JSON to HTML)
153-
- Disabling lenses
154147
- Manual collection points in arbitrary code
155148
- Demo script batch modes
156149

‎devtools/observatory/RFC_OBSERVATORY_SHARED_FRAMEWORK.md‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -175,20 +175,17 @@ Each backend maintains its own CLI that registers the appropriate lenses. Featur
175175
# Generic (framework lenses only, graph collection)
176176
python -m executorch.devtools.observatory SCRIPT [ARGS...]
177177

178-
# Generic with accuracy debugging
179-
python -m executorch.devtools.observatory --accuracy SCRIPT [ARGS...]
180-
181178
# Qualcomm — graph collection only (default)
182179
python -m executorch.backends.qualcomm.debugger.observatory SCRIPT [ARGS...]
183180

184181
# Qualcomm — with accuracy debugging
185-
python -m executorch.backends.qualcomm.debugger.observatory --accuracy SCRIPT [ARGS...]
182+
python -m executorch.backends.qualcomm.debugger.observatory --lense_recipe=accuracy SCRIPT [ARGS...]
186183

187184
# XNNPACK — graph collection only (default)
188185
python -m executorch.backends.xnnpack.debugger.observatory SCRIPT [ARGS...]
189186

190187
# XNNPACK — with accuracy debugging
191-
python -m executorch.backends.xnnpack.debugger.observatory --accuracy SCRIPT [ARGS...]
188+
python -m executorch.backends.xnnpack.debugger.observatory --lense_recipe=accuracy SCRIPT [ARGS...]
192189
```
193190

194191
### 6.7 fx_viewer Extension API
@@ -433,7 +430,7 @@ Rejected: Adds maintenance burden and delays migration. A clean break is simpler
433430

434431
1. Should the generic CLI (`python -m executorch.devtools.observatory`) auto-discover backend patches if the backend package is installed, or should it always require explicit backend CLI usage?
435432
2. Should Observatory integrate with the existing Inspector API, or remain a parallel workflow?
436-
3. What is the right granularity for the `--accuracy` flag? Should there be finer-grained lens selection (e.g., `--lenses graph,metadata`)?
433+
3. ~~What is the right granularity for the `--accuracy` flag?~~ Resolved: backend CLIs use `--lense_recipe=accuracy` to opt-in to accuracy lenses. The generic CLI has no accuracy flag.
437434

438435
## 15. Expected Outcome
439436

0 commit comments

Comments
 (0)