Skip to content

Commit 3924b6e

Browse files
authored
[QNN EP] Update tensor naming logic in DebugNodeInputsOutputs (microsoft#26660)
### Description Introduced a new environment variable ORT_DEBUG_NODE_IO_PREPEND_EP_TO_FILE_NAME to control whether the Execution Provider (EP) type is prepended to tensor file names. Updated MakeTensorFileName() and related logic to include the EP type in the format: `<EPType>_<TensorName>` when the variable is set to a non-zero value. This change helps avoid file name conflicts and improves clarity when analyzing outputs from multi-EP runs. ### Motivation and Context Accuracy unit tests may run the same model multiple times using different EPs, such as ORT CPU and QNN HTP. Previously, dumped tensor files used identical names when tensor names matched across sessions, causing file overwrites. ### Example When ORT_DEBUG_NODE_IO_PREPEND_EP_TO_FILE_NAME=1, dumped tensors include EP-specific prefixes: ``` CPU_input.tensorproto CPU_input_token_0.tensorproto CPU_output.tensorproto QNN_constant.tensorproto QNN_input.tensorproto QNN_input_token_0.tensorproto QNN_output.tensorproto ```
1 parent 5d59608 commit 3924b6e

2 files changed

Lines changed: 30 additions & 3 deletions

File tree

onnxruntime/core/framework/debug_node_inputs_outputs_utils.cc

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -99,9 +99,23 @@ struct TensorMetadata {
9999
std::string producer;
100100
std::string consumer;
101101
std::string device_type;
102+
std::string ep_type;
102103
size_t step;
103104
};
104105

106+
std::string GetCleanedEpType(const Node& node) {
107+
std::string ep_type = node.GetExecutionProviderType();
108+
109+
// Remove "ExecutionProvider" suffix from ep type to reduce length.
110+
const std::string suffix_to_remove = "ExecutionProvider";
111+
size_t pos = ep_type.find(suffix_to_remove);
112+
if (pos != std::string::npos) {
113+
ep_type.erase(pos, suffix_to_remove.length());
114+
}
115+
116+
return ep_type;
117+
}
118+
105119
bool FilterNode(const NodeDumpOptions& dump_options, const Node& node) {
106120
auto match_pattern =
107121
[](const std::string& value, const std::string& delimited_patterns) {
@@ -139,14 +153,18 @@ void DumpTensorToStdOut(const Tensor& tensor, const NodeDumpOptions& dump_option
139153
}
140154
}
141155

142-
PathString MakeTensorFileName(const std::string& tensor_name, const NodeDumpOptions& dump_options) {
156+
PathString MakeTensorFileName(const TensorMetadata& tensor_metadata, const NodeDumpOptions& dump_options) {
143157
auto make_valid_name = [](std::string name) {
144158
std::replace_if(
145159
name.begin(), name.end(), [](char c) { return !std::isalnum(c); }, '_');
146160
return name;
147161
};
148162

149-
return path_utils::MakePathString(make_valid_name(tensor_name), dump_options.file_suffix, ".tensorproto");
163+
if (dump_options.prepend_ep_to_file_name) {
164+
return path_utils::MakePathString(make_valid_name(tensor_metadata.ep_type + "_" + tensor_metadata.name), dump_options.file_suffix, ".tensorproto");
165+
} else {
166+
return path_utils::MakePathString(make_valid_name(tensor_metadata.name), dump_options.file_suffix, ".tensorproto");
167+
}
150168
}
151169

152170
void DumpTensorToFile(const Tensor& tensor, const std::string& tensor_name, const std::filesystem::path& file_path) {
@@ -375,7 +393,7 @@ void DumpCpuTensor(
375393
break;
376394
}
377395
case NodeDumpOptions::DataDestination::TensorProtoFiles: {
378-
const std::filesystem::path tensor_file = dump_options.output_dir / MakeTensorFileName(tensor_metadata.name, dump_options);
396+
const std::filesystem::path tensor_file = dump_options.output_dir / MakeTensorFileName(tensor_metadata, dump_options);
379397
DumpTensorToFile(tensor, tensor_metadata.name, tensor_file);
380398
break;
381399
}
@@ -485,6 +503,8 @@ const NodeDumpOptions& NodeDumpOptionsFromEnvironmentVariables() {
485503
debug_node_inputs_outputs_env_vars::kHalfOverflowThreshold, " shall be a positive integer <= ", kMaxHalfThreshold);
486504
opts.half_overflow_threshold = static_cast<float>(threshold);
487505

506+
opts.prepend_ep_to_file_name = ParseEnvironmentVariableWithDefault<bool>(env_vars::kPrependEpToFileName, false);
507+
488508
if (ParseEnvironmentVariableWithDefault<bool>(env_vars::kAppendRankToFileName, false)) {
489509
std::string rank = Env::Default().GetEnvironmentVar("OMPI_COMM_WORLD_RANK");
490510
if (rank.empty()) {
@@ -582,6 +602,7 @@ void DumpNodeInputs(
582602
tensor_metadata.name = input_defs[i]->Name();
583603
tensor_metadata.step = dump_context.iteration;
584604
tensor_metadata.consumer = node.Name() + ":" + std::to_string(i);
605+
tensor_metadata.ep_type = GetCleanedEpType(node);
585606

586607
TensorStatisticsData tensor_statistics;
587608
DumpTensor(dump_options, *tensor, tensor_metadata, tensor_statistics, session_state);
@@ -678,6 +699,7 @@ void DumpNodeOutputs(
678699
tensor_metadata.name = output_defs[i]->Name();
679700
tensor_metadata.step = dump_context.iteration;
680701
tensor_metadata.producer = node.Name() + ":" + std::to_string(i);
702+
tensor_metadata.ep_type = GetCleanedEpType(node);
681703

682704
TensorStatisticsData tensor_statistics;
683705
DumpTensor(dump_options, *tensor, tensor_metadata, tensor_statistics, session_state);

onnxruntime/core/framework/debug_node_inputs_outputs_utils.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,8 @@ constexpr const char* kOpTypeFilter = "ORT_DEBUG_NODE_IO_OP_TYPE_FILTER";
5454
constexpr const char* kDumpDataDestination = "ORT_DEBUG_NODE_IO_DUMP_DATA_DESTINATION";
5555
// set to non-zero to append OpenMPI world rank to filename
5656
constexpr const char* kAppendRankToFileName = "ORT_DEBUG_NODE_IO_APPEND_RANK_TO_FILE_NAME";
57+
// set to non-zero to prepend ep type to filename
58+
constexpr const char* kPrependEpToFileName = "ORT_DEBUG_NODE_IO_PREPEND_EP_TO_FILE_NAME";
5759
// specify the output directory for any data files produced
5860
constexpr const char* kOutputDir = "ORT_DEBUG_NODE_IO_OUTPUT_DIR";
5961
// specify the file prefix for sqlite3 db (process id will be appended)
@@ -117,6 +119,9 @@ struct NodeDumpOptions {
117119
SqliteDb
118120
} data_destination{DataDestination::StdOut};
119121

122+
// Whether to prepend ep type to output file name.
123+
bool prepend_ep_to_file_name{false};
124+
120125
std::string file_suffix;
121126
// the output directory for dumped data files
122127
std::filesystem::path output_dir;

0 commit comments

Comments
 (0)