Skip to content

Commit 1ceee33

Browse files
authored
Merge pull request #1119 from intel/sync_msft_4_6_26
Backmerge pr Msft
2 parents 8f777da + 7b01e75 commit 1ceee33

31 files changed

Lines changed: 2157 additions & 165 deletions

include/onnxruntime/core/session/onnxruntime_lite_custom_op.h

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -330,6 +330,12 @@ struct TensorArray : public ArgBase {
330330
case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT:
331331
tensor = std::make_unique<Custom::Tensor<float>>(ctx, ith_input, true);
332332
break;
333+
case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16:
334+
tensor = std::make_unique<Custom::Tensor<Ort::Float16_t>>(ctx, ith_input, true);
335+
break;
336+
case ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16:
337+
tensor = std::make_unique<Custom::Tensor<Ort::BFloat16_t>>(ctx, ith_input, true);
338+
break;
333339
case ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE:
334340
tensor = std::make_unique<Custom::Tensor<double>>(ctx, ith_input, true);
335341
break;
@@ -360,6 +366,18 @@ struct TensorArray : public ArgBase {
360366
case ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING:
361367
tensor = std::make_unique<Custom::Tensor<std::string>>(ctx, ith_input, true);
362368
break;
369+
case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FN:
370+
tensor = std::make_unique<Custom::Tensor<Ort::Float8E4M3FN_t>>(ctx, ith_input, true);
371+
break;
372+
case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FNUZ:
373+
tensor = std::make_unique<Custom::Tensor<Ort::Float8E4M3FNUZ_t>>(ctx, ith_input, true);
374+
break;
375+
case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2:
376+
tensor = std::make_unique<Custom::Tensor<Ort::Float8E5M2_t>>(ctx, ith_input, true);
377+
break;
378+
case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2FNUZ:
379+
tensor = std::make_unique<Custom::Tensor<Ort::Float8E5M2FNUZ_t>>(ctx, ith_input, true);
380+
break;
363381
default:
364382
ORT_CXX_API_THROW("unknown input type", ORT_RUNTIME_EXCEPTION);
365383
break;

onnxruntime/core/optimizer/constant_folding.cc

Lines changed: 59 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -249,9 +249,57 @@ static int64_t EstimateIdentityOutputSizeInBytes(const Node& node) {
249249
return EstimateTensorSizeInBytes(*input_defs[0]);
250250
}
251251

252+
// ConstantOfShape's output shape is determined by the values of its first input (a shape tensor),
253+
// which is required to be a constant initializer for the node to be eligible for constant folding.
254+
// Compute the output byte size directly from that initializer so we do not have to rely on ONNX
255+
// shape inference having propagated the shape onto the output NodeArg (which is not guaranteed
256+
// for all opsets, all shapes-as-initializers, or all build configurations).
257+
static int64_t EstimateConstantOfShapeOutputSizeInBytes(const Node& node, const Graph& graph) {
258+
const auto& input_defs = node.InputDefs();
259+
if (input_defs.empty() || input_defs[0] == nullptr || !input_defs[0]->Exists()) {
260+
return -1;
261+
}
262+
263+
constexpr bool check_outer_scope = true;
264+
const ONNX_NAMESPACE::TensorProto* shape_init =
265+
graph.GetConstantInitializer(input_defs[0]->Name(), check_outer_scope);
266+
if (shape_init == nullptr) {
267+
return -1;
268+
}
269+
270+
Initializer shape_data{graph, *shape_init, graph.ModelPath()};
271+
if (shape_data.data_type() != ONNX_NAMESPACE::TensorProto_DataType_INT64) {
272+
return -1;
273+
}
274+
275+
SafeInt<int64_t> num_elements = 1;
276+
for (int64_t dim : shape_data.DataAsSpan<int64_t>()) {
277+
if (dim < 0) {
278+
return -1; // Invalid shape value; let the kernel reject it.
279+
}
280+
num_elements *= dim;
281+
}
282+
283+
// Determine the element size of the output. The ONNX spec for ConstantOfShape defaults the
284+
// element type to float when the optional 'value' attribute is absent.
285+
size_t element_size = sizeof(float);
286+
const auto& attrs = node.GetAttributes();
287+
auto it = attrs.find("value");
288+
if (it != attrs.end() && it->second.type() == ONNX_NAMESPACE::AttributeProto::TENSOR) {
289+
const auto elem_type = static_cast<ONNX_NAMESPACE::TensorProto_DataType>(
290+
it->second.t().data_type());
291+
const size_t es = GetElementSizeForConstantFolding(elem_type);
292+
if (es != 0) {
293+
element_size = es;
294+
}
295+
}
296+
297+
return num_elements * static_cast<int64_t>(element_size);
298+
}
299+
252300
// Estimate the total output size in bytes for a node using shape inference results.
253301
// Returns -1 if the output size cannot be estimated (e.g., unknown shapes or types).
254-
static int64_t EstimateNodeOutputSizeInBytes(const Node& node) {
302+
static int64_t EstimateNodeOutputSizeInBytes(const Node& node, const Graph& graph) {
255303
if (node.OpType() == "Identity" && node.Domain().empty()) {
256304
return EstimateIdentityOutputSizeInBytes(node);
257305
}
@@ -260,6 +308,15 @@ static int64_t EstimateNodeOutputSizeInBytes(const Node& node) {
260308
return EstimateUniqueOutputSizeInBytes(node);
261309
}
262310

311+
if (node.OpType() == "ConstantOfShape" && node.Domain().empty()) {
312+
const int64_t size = EstimateConstantOfShapeOutputSizeInBytes(node, graph);
313+
if (size >= 0) {
314+
return size;
315+
}
316+
// Fall through to the generic estimator if we could not derive a size from the input
317+
// initializer (e.g., the shape input is not a recognizable constant initializer).
318+
}
319+
263320
SafeInt<int64_t> total_size = 0;
264321
for (const auto* output_def : node.OutputDefs()) {
265322
if (!output_def->Exists()) {
@@ -391,7 +448,7 @@ Status ConstantFolding::ApplyImpl(Graph& graph, bool& modified, int graph_level,
391448
if (max_output_size > 0) {
392449
int64_t estimated_size = -1;
393450
try {
394-
estimated_size = EstimateNodeOutputSizeInBytes(*node);
451+
estimated_size = EstimateNodeOutputSizeInBytes(*node, graph);
395452
} catch (const std::exception&) {
396453
// SafeInt overflow means the size is astronomically large - definitely skip
397454
LOGS(logger, WARNING) << "Integer overflow while estimating output size of "

onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.cc

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -183,12 +183,23 @@ bool DropQDQNodeGroupSelector::Check(const GraphViewer& graph_viewer, const Node
183183
return false;
184184
}
185185

186-
// Resize with mode != "nearest" (linear/cubic) is not order-preserving on integers,
187-
// so dropping the surrounding Q/DQ pair would change numerical results. Only the
188-
// default nearest-neighbor mode is safe to fold. See issue #21319.
186+
// Resize output depends on both the interpolation mode and the coordinate transformation mode:
187+
// - "nearest" mode copies existing input values, so it can operate directly on quantized integers.
188+
// - Non-nearest modes (e.g., "linear", "cubic") interpolate using float arithmetic, which is only
189+
// correct in the dequantized (float) domain, so QDQ must be preserved.
190+
// - "tf_crop_and_resize" writes extrapolation_value (authored in the float domain) into out-of-crop
191+
// positions; dropping QDQ would store that float value raw in the quantized domain, even for nearest.
192+
// Only allow dropping QDQ for nearest mode without tf_crop_and_resize coordinate transformation.
189193
if (node.OpType() == "Resize") {
190-
const auto* mode_attr = graph_utils::GetNodeAttribute(node, "mode");
191-
if (mode_attr != nullptr && mode_attr->s() != "nearest") {
194+
const auto& attrs = node.GetAttributes();
195+
// "mode" defaults to "nearest" when absent. It is always present after Graph::Resolve() injects
196+
// schema defaults; the absence check is a defensive fallback for hand-built/unresolved graphs.
197+
const auto mode_iter = attrs.find("mode");
198+
if (mode_iter != attrs.end() && mode_iter->second.s() != "nearest") {
199+
return false;
200+
}
201+
const auto coord_mode_iter = attrs.find("coordinate_transformation_mode");
202+
if (coord_mode_iter != attrs.end() && coord_mode_iter->second.s() == "tf_crop_and_resize") {
192203
return false;
193204
}
194205
}
@@ -989,6 +1000,13 @@ bool GemmNodeGroupSelector::Check(const GraphViewer& graph_viewer, const Node& n
9891000
return true;
9901001
}
9911002

1003+
// When bias is present, QGemm folds bias into the int32 accumulator before
1004+
// applying the alpha*sa*sb output scale, which would incorrectly scale the
1005+
// bias by alpha. Require alpha==1 and beta==1 so the fused path is exact.
1006+
if (node.GetAttributes().at("alpha").f() != 1.0) {
1007+
return false;
1008+
}
1009+
9921010
if (node.GetAttributes().at("beta").f() != 1.0) { // beta needs to be 1.0
9931011
return false;
9941012
}

onnxruntime/core/providers/cpu/math/element_wise_ops.cc

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1890,7 +1890,7 @@ Status PRelu<float>::Compute(OpKernelContext* context) const {
18901890
ProcessBroadcastSpanFuncs funcs{
18911891
[](BroadcastHelper& per_iter_bh) {
18921892
float input0 = per_iter_bh.ScalarInput0<float>();
1893-
if (input0 > 0)
1893+
if (input0 >= 0)
18941894
per_iter_bh.OutputEigen<float>().array() = input0;
18951895
else
18961896
per_iter_bh.OutputEigen<float>() = input0 * per_iter_bh.EigenInput1<float>().array();
@@ -1901,7 +1901,7 @@ Status PRelu<float>::Compute(OpKernelContext* context) const {
19011901
float* output = per_iter_bh.OutputEigen<float>().data();
19021902
size_t size = per_iter_bh.OutputEigen<float>().size();
19031903
for (size_t i = 0; i < size; i++) {
1904-
output[i] = input0[i] > 0 ? input0[i] : input0[i] * input1;
1904+
output[i] = (input0[i] >= 0) ? input0[i] : input0[i] * input1;
19051905
}
19061906
},
19071907
[](BroadcastHelper& per_iter_bh) {
@@ -1910,7 +1910,7 @@ Status PRelu<float>::Compute(OpKernelContext* context) const {
19101910
float* output = per_iter_bh.OutputEigen<float>().data();
19111911
size_t size = per_iter_bh.OutputEigen<float>().size();
19121912
for (size_t i = 0; i < size; i++) {
1913-
output[i] = input0[i] > 0 ? input0[i] : input0[i] * input1[i];
1913+
output[i] = (input0[i] >= 0) ? input0[i] : input0[i] * input1[i];
19141914
}
19151915
}};
19161916

onnxruntime/core/providers/cpu/math/element_wise_ops.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33

44
#pragma once
55

6+
#include <type_traits>
7+
68
#include "core/common/common.h"
79
#include "core/common/inlined_containers.h"
810
#include "core/common/narrow.h"
@@ -482,6 +484,11 @@ class BitwiseXor final : public OpKernel {
482484
// PRelu is activation function, but it's closer to binary elementwise ops in implementation
483485
template <typename T>
484486
class PRelu final : public OpKernel {
487+
// Currently only registered for float. Integer types would require addressing
488+
// overflow in multiplication and avoiding 0.0*inf=NaN in branchless formulations.
489+
static_assert(std::is_floating_point_v<T>,
490+
"PRelu is only supported for floating-point types.");
491+
485492
public:
486493
PRelu(const OpKernelInfo& info) : OpKernel(info) {
487494
}

onnxruntime/core/providers/cpu/ml/tree_ensemble_aggregator.h

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -480,6 +480,7 @@ class TreeAggregatorClassifier : public TreeAggregatorSum<InputType, ThresholdTy
480480
private:
481481
const std::vector<int64_t>& class_labels_;
482482
bool binary_case_;
483+
bool weights_are_all_positive_;
483484
int64_t positive_label_;
484485
int64_t negative_label_;
485486

@@ -490,11 +491,13 @@ class TreeAggregatorClassifier : public TreeAggregatorSum<InputType, ThresholdTy
490491
const std::vector<ThresholdType>& base_values,
491492
const std::vector<int64_t>& class_labels,
492493
bool binary_case,
494+
bool weights_are_all_positive,
493495
int64_t positive_label = 1,
494496
int64_t negative_label = 0) : TreeAggregatorSum<InputType, ThresholdType, OutputType>(n_trees, n_targets_or_classes,
495497
post_transform, base_values),
496498
class_labels_(class_labels),
497499
binary_case_(binary_case),
500+
weights_are_all_positive_(weights_are_all_positive),
498501
positive_label_(positive_label),
499502
negative_label_(negative_label) {}
500503

@@ -523,12 +526,22 @@ class TreeAggregatorClassifier : public TreeAggregatorSum<InputType, ThresholdTy
523526
ThresholdType score1, unsigned char has_score1) const {
524527
ThresholdType pos_weight = has_score1 ? score1 : (has_score0 ? score0 : 0); // only 1 class
525528
if (binary_case_) {
526-
if (pos_weight > 0) {
527-
write_additional_scores = 2;
528-
return class_labels_[1]; // positive label
529+
if (weights_are_all_positive_) {
530+
if (pos_weight > 0.5) {
531+
write_additional_scores = 0;
532+
return class_labels_[1]; // positive label
533+
} else {
534+
write_additional_scores = 1;
535+
return class_labels_[0]; // negative label
536+
}
529537
} else {
530-
write_additional_scores = 3;
531-
return class_labels_[0]; // negative label
538+
if (pos_weight > 0) {
539+
write_additional_scores = 2;
540+
return class_labels_[1]; // positive label
541+
} else {
542+
write_additional_scores = 3;
543+
return class_labels_[0]; // negative label
544+
}
532545
}
533546
}
534547
return (pos_weight > 0)

onnxruntime/core/providers/cpu/ml/tree_ensemble_common.h

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -983,6 +983,7 @@ TreeEnsembleCommon<InputType, ThresholdType, OutputType>::ProcessTreeNodeLeave(
983983
template <typename InputType, typename ThresholdType, typename OutputType>
984984
class TreeEnsembleCommonClassifier : public TreeEnsembleCommon<InputType, ThresholdType, OutputType> {
985985
private:
986+
bool weights_are_all_positive_;
986987
bool binary_case_;
987988
std::vector<std::string> classlabels_strings_;
988989
std::vector<int64_t> classlabels_int64s_;
@@ -1018,7 +1019,15 @@ Status TreeEnsembleCommonClassifier<InputType, ThresholdType, OutputType>::Init(
10181019

10191020
InlinedHashSet<int64_t> weights_classes;
10201021
weights_classes.reserve(attributes.target_class_ids.size());
1021-
weights_classes.insert(attributes.target_class_ids.begin(), attributes.target_class_ids.end());
1022+
weights_are_all_positive_ = true;
1023+
for (size_t i = 0, end = attributes.target_class_ids.size(); i < end; ++i) {
1024+
weights_classes.insert(attributes.target_class_ids[i]);
1025+
if (weights_are_all_positive_ && (attributes.target_class_weights_as_tensor.empty()
1026+
? static_cast<ThresholdType>(attributes.target_class_weights[i])
1027+
: attributes.target_class_weights_as_tensor[i]) < 0) {
1028+
weights_are_all_positive_ = false;
1029+
}
1030+
}
10221031
binary_case_ = this->n_targets_or_classes_ == 2 && weights_classes.size() == 1;
10231032
if (!classlabels_strings_.empty()) {
10241033
class_labels_.reserve(classlabels_strings_.size());
@@ -1039,7 +1048,8 @@ Status TreeEnsembleCommonClassifier<InputType, ThresholdType, OutputType>::compu
10391048
TreeAggregatorClassifier<InputType, ThresholdType, OutputType>(
10401049
this->roots_.size(), this->n_targets_or_classes_,
10411050
this->post_transform_, this->base_values_,
1042-
classlabels_int64s_, binary_case_));
1051+
classlabels_int64s_, binary_case_,
1052+
weights_are_all_positive_));
10431053
} else {
10441054
int64_t N = X->Shape().NumDimensions() == 1 ? 1 : X->Shape()[0];
10451055
AllocatorPtr alloc;
@@ -1050,7 +1060,8 @@ Status TreeEnsembleCommonClassifier<InputType, ThresholdType, OutputType>::compu
10501060
TreeAggregatorClassifier<InputType, ThresholdType, OutputType>(
10511061
this->roots_.size(), this->n_targets_or_classes_,
10521062
this->post_transform_, this->base_values_,
1053-
class_labels_, binary_case_));
1063+
class_labels_, binary_case_,
1064+
weights_are_all_positive_));
10541065
const int64_t* plabel = label_int64.Data<int64_t>();
10551066
std::string* labels = label->MutableData<std::string>();
10561067
for (size_t i = 0; i < (size_t)N; ++i)

0 commit comments

Comments
 (0)