Qualcomm AI Engine Direct - Update CLI tool to dump inference performace

Author: chenweng-quic

backends/qualcomm/tests/test_qnn_delegate.py

@@ -8311,6 +8311,7 @@ def test_cli(self):
                 cmds.extend(["--host", self.host])
             subprocess.run(cmds, stdout=subprocess.DEVNULL)
             self.assertTrue(os.path.isfile(f"{tmp_dir}/e_out/Result_0/output_0.pt"))
+            self.assertTrue(os.path.isfile(f"{tmp_dir}/e_out/performance.json"))
 
     def test_cli_with_input_list_assignment(self):
         with tempfile.TemporaryDirectory() as tmp_dir:
@@ -8379,11 +8380,10 @@ def test_cli_with_input_list_assignment(self):
                 "--input_list",
                 f"{tmp_dir}/input_list",
             ]
-            if self.host:
-                cmds.extend(["--host", self.host])
             subprocess.run(cmds, stdout=subprocess.DEVNULL)
             output_file = f"{tmp_dir}/e_out/Result_0/output_0.pt"
             self.assertTrue(os.path.isfile(output_file))
+            self.assertTrue(os.path.isfile(f"{tmp_dir}/e_out/performance.json"))
             device_output = torch.load(output_file, weights_only=True)
             golden_output = ep.module()(sample_input, sample_input2)
             self._assert_outputs_equal(golden_output, device_output)

examples/qualcomm/util_scripts/cli.py

@@ -8,7 +8,9 @@
 # and executing models under various configuration flags.
 
 import argparse
+import csv
 import importlib
+import json
 import logging
 import os
 import re
@@ -37,6 +39,7 @@
     QNN_TENSOR_TYPE_MAP,
     to_edge_transform_and_lower_to_qnn,
 )
+from executorch.devtools import Inspector
 from executorch.examples.qualcomm.qaihub_scripts.utils.utils import preprocess_binary
 from executorch.examples.qualcomm.utils import (
     get_backend_type,
@@ -271,7 +274,13 @@ def execute(args):
         args.artifact,
         verification=Verification.Minimal,
     )
-    input_order_func = program.load_method(INPUT_ORDER)
+    try:
+        input_order_func = program.load_method(INPUT_ORDER)
+    except:
+        logger.error(
+            "Missing INPUT_ORDER in the .pte. The CLI execute command only supports .pte files generated by the CLI compile command, which preserves the input order."
+        )
+        exit(1)
     input_order = input_order_func.execute([])
 
     # load input files
@@ -357,8 +366,27 @@ def post_process():
             )
             torch.save(output, f"{output_result_folder}/output_{output_index}.pt")
 
+    def post_process_etdump():
+        etdump_path = f"{args.output_folder}/etdump.etdp"
+        csv_path = f"{args.output_folder}/etdump.csv"
+        json_path = f"{args.output_folder}/performance.json"
+        inspector = Inspector(etdump_path=etdump_path)
+        inspector.save_data_to_tsv(csv_path)
+        # Create a list to hold the data
+        data = []
+        # Open the CSV file and read its contents
+        with open(csv_path, encoding="utf-8") as csv_file:
+            csv_reader = csv.DictReader(csv_file, delimiter="\t")
+            # Convert each row into a dictionary and add it to the list
+            for row in csv_reader:
+                data.append(row)
+        # Write the data to a JSON file
+        with open(json_path, "w", encoding="utf-8") as json_file:
+            json.dump(data, json_file, indent=4)
+
     logger.info("collecting output data")
-    adb.pull(tmp_dir, post_process)
+    adb.pull(tmp_dir, callback=post_process)
+    adb.pull_etdump(args.output_folder, callback=post_process_etdump)
     shutil.rmtree(tmp_dir)
     logger.info(f"execution finished, please check {args.output_folder} for results")
 

examples/qualcomm/utils.py

@@ -201,14 +201,15 @@ def _adb(self, cmd, output_callback: Optional[Callable[[str], None]] = None):
             )
             output_callback(result)
         else:
-            subprocess.run(
+            result = subprocess.run(
                 cmds, stdout=subprocess.DEVNULL if self.error_only else sys.stdout
             )
+        if result.returncode != 0:
+            raise RuntimeError(f"adb command failed: {cmds}")
 
     def push(
         self,
         inputs=None,
-        input_list=None,
         files=None,
         backends: Optional[Set[QnnExecuTorchBackendType]] = None,
         init_env=True,
@@ -905,7 +906,7 @@ def setup_common_args_and_variables():
         "-H",
         "--host",
         help="hostname where android device is connected.",
-        default=None,
+        default="localhost",
         type=str,
     )