NXP backend: Add e2e test for profiling feature (pytorch#20785)

irtrukhina · web-flow · commit 27bd1bb014b9 · 2026-07-16T09:59:04.000+02:00
### Summary Add an end-to-end test for the profiling feature that generates ETDump and ETRecord artifacts and validates the resulting profiling data using the ExecuTorch Inspector. Changes: 1. Enable DevTools and Event Tracer support when building nxp_executor_runner. 2. Save profiling artifacts for all models converted with the profiling flag: - ETDump (trace.etdump) is generated and saved by nxp_executor_runner. - ETRecord (etrecord.bin) is generated and saved in nsys_testing.py. Add Inspector-based validation that: - Loads the generated ETDump and ETRecord. - Parses Neutron delegate metadata. - Verifies kernel profiling events and profiling dump events. - Confirms the profiling data can be successfully processed by the ExecuTorch Inspector. Note: This test is currently marked as xfail and should be enabled after the Neutron SW 3.2 release. ### Test plan Test included. cc @robert-kalmar @JakeStevens @digantdesai @rascani
diff --git a/backends/nxp/runtime/NeutronBackend.cpp b/backends/nxp/runtime/NeutronBackend.cpp
@@ -586,7 +586,7 @@ class NeutronBackend final : public PyTorchBackendInterface {
       char* profile_info =
           static_cast<char*>(cfg->dcfg.outputs[profiling_index]);
       NeutronFullProfilingEvent* neutron_events =
-          reinterpreter_cast<NeutronFullProfilingEvent*>(profile_info);
+          reinterpret_cast<NeutronFullProfilingEvent*>(profile_info);
       executorch::runtime::EventTracer* tracer = context.event_tracer();
       uint32_t start_time = 0;
       int index = 0;
diff --git a/backends/nxp/tests/generic_tests/test_profiling.py b/backends/nxp/tests/generic_tests/test_profiling.py
@@ -4,7 +4,9 @@
 # LICENSE file in the root directory of this source tree.
 import ast
 import logging
+import os
 import re
+from typing import Any, Union
 
 import numpy as np
 import pytest
@@ -13,17 +15,20 @@
 from executorch.backends.nxp.tests.model_output_comparator import (
     NumericalStatsOutputComparator,
 )
-
 from executorch.backends.nxp.tests.models import AvgPool2dModule, SoftmaxModule
-from executorch.backends.nxp.tests.nsys_testing import lower_run_compare
+from executorch.backends.nxp.tests.nsys_testing import (
+    get_test_name,
+    lower_run_compare,
+    OUTPUTS_DIR,
+)
 
+from executorch.devtools.inspector._inspector import Inspector
 from executorch.examples.models.mlperf_tiny import (
     DeepAutoEncoder,
     DSCNNKWS,
     MobileNetV1025,
     ResNet8,
 )
-
 from executorch.examples.nxp.experimental.cifar_net.cifar_net import CifarNetModel
 
 
@@ -46,6 +51,103 @@ def extract_map_from_logs(caplog):
     return None
 
 
+def inspector_check(test_name: str) -> None:
+    """
+    Validate ExecuTorch Inspector profiling output.
+
+    Checks:
+      1. Required profiling artifacts (etrecord.bin, trace.etdump) exist.
+      2. Inspector can be created and profiling data can be parsed.
+      3. All numeric delegate events except the last contain
+         "Neutron kernel" metadata.
+      4. The last numeric delegate event contains
+         "Profiling dump" metadata.
+      5. The profiling dump event does not have associated op types.
+    """
+
+    def parse_delegate_metadata(
+        delegate_metadatas: list[bytes],
+    ) -> Union[list[str], dict[str, Any]]:
+        """Metadata parser for Neutron Backend metadata.
+
+        The parser is a callable that deserializes the data and returns neutron kernel number.
+        The deserialized data is then added back to the corresponding event in the event block for user consumption.
+        """
+
+        metadata_list = []
+        for metadata_bytes in delegate_metadatas:
+            if len(metadata_bytes) == 1:
+                function_code = metadata_bytes[0]
+                if function_code == 0:
+                    metadata_list.append("Profiling dump")
+                else:
+                    metadata_list.append("Neutron kernel " + str(function_code))
+            else:
+                metadata_list.append("Invalid metadata size")
+        return metadata_list
+
+    npu_results_path = os.path.join(OUTPUTS_DIR, test_name, "results_npu")
+    etrecord_path = os.path.join(npu_results_path, "etrecord.bin")
+    etdump_path = os.path.join(npu_results_path, "trace.etdump")
+
+    # Verify profiling artifacts were generated.
+    for file_path in (etrecord_path, etdump_path):
+        assert os.path.isfile(
+            file_path
+        ), f"Required profiling file does not exist: {file_path}"
+
+    # Create Inspector and parse profiling data.
+    try:
+        inspector = Inspector(
+            etdump_path=etdump_path,
+            etrecord=etrecord_path,
+            delegate_metadata_parser=parse_delegate_metadata,
+        )
+        inspector.print_data_tabular(include_delegate_debug_data=True)
+
+    except Exception as e:
+        raise RuntimeError(
+            "Failed to create or run Inspector for "
+            f"etdump='{etdump_path}', "
+            f"etrecord='{etrecord_path}'"
+        ) from e
+
+    # Collect delegated profiling events whose names are numeric
+    # (0, 1, 2, ..., N). These events are emitted by the Neutron backend.
+    numeric_events = [
+        event
+        for event_block in inspector.event_blocks
+        for event in event_block.events
+        if str(event.name).isdigit()
+    ]
+
+    assert numeric_events, "No numeric delegate profiling events found"
+
+    # All delegate events except the last one should describe
+    # individual Neutron kernels.
+    for event in numeric_events[:-1]:
+        metadata = str(event.delegate_debug_metadatas)
+
+        assert "Neutron kernel" in metadata, (
+            f"Event {event.name}: expected 'Neutron kernel', " f"got {metadata}"
+        )
+
+    # The final numeric event should represent the profiling dump.
+    profiling_dump_event = numeric_events[-1]
+    profiling_metadata = str(profiling_dump_event.delegate_debug_metadatas)
+
+    assert "Profiling dump" in profiling_metadata, (
+        f"Event {profiling_dump_event.name}: "
+        f"expected 'Profiling dump', got {profiling_metadata}"
+    )
+
+    # Profiling dump event is expected to have no associated operators.
+    assert profiling_dump_event.op_types == [], (
+        f"Event {profiling_dump_event.name}: expected empty op_types, "
+        f"got {profiling_dump_event.op_types}"
+    )
+
+
 class SimpleParallelPoolModel(torch.nn.Module):
     def __init__(self, channels: int):
         super().__init__()
@@ -84,7 +186,10 @@ def forward(self, x):
 
 
 class TestProfiling:
-    @pytest.mark.xfail(reason="SoftMax support PR is not merged so far.", strict=True)
+    @pytest.mark.xfail(
+        reason="Profiling support for cmodel and SoftMax fix will be available in Neutron SW 3.2.",
+        strict=True,
+    )
     def test__softmax(self, caplog, request):
         caplog.set_level(logging.INFO)
         model = SoftmaxModule(-1)
@@ -164,6 +269,7 @@ def test__cifar(self, caplog, request):
             10: (21,),  # Slice
             11: (),  # Neutron Dump
         }
+        inspector_check(get_test_name(request))
 
     def test__avg_pool(self, caplog, request):
         caplog.set_level(logging.INFO)
diff --git a/backends/nxp/tests/nsys_testing.py b/backends/nxp/tests/nsys_testing.py
@@ -153,6 +153,15 @@ def wrapper(*args, **kwargs):
 
     save_pte_program(delegated_program, test_name + "_delegated", test_dir)
 
+    # Generate ETRecord if profiling flag is set.
+    if use_profiling:
+        etrecord_path = os.path.join(npu_results_dir, "etrecord.bin")
+        # Create directory if it doesn't exist
+        os.makedirs(os.path.dirname(etrecord_path), exist_ok=True)
+        # Save ETRecord
+        delegated_program.get_etrecord().save(etrecord_path)
+        logging.info(f"The ETRecord for the model was saved to {etrecord_path}.")
+
     # Preparation of quantized dataset, requires quantization parameters from converted delegated model
     if remove_quant_io_ops:
         dataset_dir_quant = os.path.join(test_dir, "dataset_quant")
diff --git a/examples/nxp/executor_runner/CMakeLists.txt b/examples/nxp/executor_runner/CMakeLists.txt
@@ -6,6 +6,11 @@
 cmake_minimum_required(VERSION 3.16)
 project(nxp_executor_runner C CXX)
 
+option(EVENT_TRACER "Build with event tracing support" ON)
+
+set(NXP_RUNNER_DEFINES NEUTRON_CMODEL)
+set(NXP_RUNNER_LIBS)
+
 set(EIQ_NEUTRON_TARGET
     "imxrt700"
     CACHE STRING "Neutron Target to build the executor_runner"
@@ -74,14 +79,18 @@ else()
   message(STATUS "Neutron Driver -- found")
 endif()
 
-# Check if the ExecuTorch Target is aleady defined, what indicate this
+# Check if the ExecuTorch Target is already defined, what indicate this
 # CMakeLists.txt is involved as subproject:
 message(STATUS "Looking for executorch target")
 if(NOT TARGET executorch)
   message(STATUS "Looking for executorch target -- not found, adding")
   set(EXECUTORCH_BUILD_NXP_NEUTRON ON)
   set(EXECUTORCH_BUILD_KERNELS_QUANTIZED ON)
   set(EXECUTORCH_BUILD_KERNELS_QUANTIZED_AOT ON)
+  if(EVENT_TRACER)
+    set(EXECUTORCH_BUILD_DEVTOOLS ON)
+    set(EXECUTORCH_ENABLE_EVENT_TRACER ON)
+  endif()
   add_subdirectory(
     "${EXECUTORCH_PATH}" "${CMAKE_CURRENT_BINARY_DIR}/executorch"
     EXCLUDE_FROM_ALL
@@ -90,6 +99,22 @@ else()
   message(STATUS "Looking for executorch target -- found")
 endif()
 
+if(EVENT_TRACER)
+  list(APPEND NXP_RUNNER_DEFINES ET_EVENT_TRACER_ENABLED)
+  list(APPEND NXP_RUNNER_LIBS etdump flatccrt)
+
+  message(STATUS "Looking for flatccrt")
+  if(NOT TARGET flatccrt)
+    message(STATUS "Looking for flatccrt -- not found, adding")
+    add_subdirectory(
+      "${EXECUTORCH_PATH}/third-party/flatcc"
+      "${CMAKE_CURRENT_BINARY_DIR}/flatcc" EXCLUDE_FROM_ALL
+    )
+  else()
+    message(STATUS "Looking for flatcc -- found")
+  endif()
+endif()
+
 message(STATUS "Looking for gflags")
 if(NOT TARGET gflags::gflags)
   message(STATUS "Looking for gflags -- not found, adding")
@@ -108,7 +133,7 @@ add_executable(
   ${EXECUTORCH_PATH}/extension/data_loader/file_data_loader.cpp
 )
 
-target_compile_options(nxp_executor_runner PRIVATE -DNEUTRON_CMODEL)
+target_compile_definitions(nxp_executor_runner PRIVATE ${NXP_RUNNER_DEFINES})
 
 target_link_libraries(
   nxp_executor_runner
@@ -119,4 +144,5 @@ target_link_libraries(
           "-Wl,--whole-archive"
           executorch_delegate_neutron
           "-Wl,--no-whole-archive"
+          ${NXP_RUNNER_LIBS}
 )
diff --git a/examples/nxp/executor_runner/nxp_executor_runner.cpp b/examples/nxp/executor_runner/nxp_executor_runner.cpp
@@ -13,6 +13,7 @@
  * Neutron Backend.
  */
 
+#include <executorch/devtools/etdump/etdump_flatcc.h>
 #include <executorch/extension/data_loader/file_data_loader.h>
 #include <executorch/runtime/executor/method.h>
 #include <executorch/runtime/executor/program.h>
@@ -204,6 +205,28 @@ Error saveOutputs(
   return Error::Ok;
 }
 
+#ifdef ET_EVENT_TRACER_ENABLED
+void saveETDump(
+    executorch::etdump::ETDumpGen* etdump_gen,
+    const std::string& outputPath) {
+  struct stat st;
+  const auto trace = etdump_gen->get_etdump_data();
+  if (trace.buf && trace.size > 0) {
+    if (stat(outputPath.c_str(), &st) == -1) {
+      mkdir(outputPath.c_str(), 0700);
+    }
+    std::string fileName = outputPath + "/trace.etdump";
+    printf("Saving file %s\n", fileName.c_str());
+    FILE* f = fopen(fileName.c_str(), "w+");
+    if (f != NULL) {
+      fwrite(static_cast<uint8_t*>(trace.buf), 1, trace.size, f);
+      fclose(f);
+    }
+    free(trace.buf);
+  }
+}
+#endif
+
 template <typename T>
 Error printClassificationOutput(
     const torch::executor::EValue& value,
@@ -438,8 +461,20 @@ int main(int argc, char* argv[]) {
       &method_allocator, &planned_memory, &tmp_allocator);
 
   {
-    Result<torch::executor::Method> method =
-        program->load_method(method_name, &memory_manager);
+#ifdef ET_EVENT_TRACER_ENABLED
+    // Create ETDumpGen before inference
+    auto etdump_gen_ptr = std::make_unique<executorch::etdump::ETDumpGen>();
+    executorch::etdump::ETDumpGen* etdump_gen = etdump_gen_ptr.get();
+#endif
+    Result<torch::executor::Method> method = program->load_method(
+        method_name,
+        &memory_manager,
+#ifdef ET_EVENT_TRACER_ENABLED
+        etdump_gen
+#else
+        nullptr
+#endif
+    );
     if (!method.ok()) {
       fprintf(
           stderr,
@@ -572,6 +607,12 @@ int main(int argc, char* argv[]) {
         }
       }
     }
+
+#ifdef ET_EVENT_TRACER_ENABLED
+    // Save ETDump.
+    saveETDump(etdump_gen, FLAGS_output);
+#endif
+
   } // Destruct the method object before destroying the Neutron Device.
 
   printf("Finished...\n");
