Skip to content

Commit 6cd7ecd

Browse files
authored
portable: add fast path for contiguous innermost-dim reduction in amax.out and amin.out (pytorch#21142)
### Motivation `sum.IntList_out`, `mean.out`, and `var_mean.correction_out` special-case the contiguous, single innermost-dim reduction shape with a direct pointer loop instead of the generic `ReduceOverDimListPlan`, which recomputes a stride-based starting index per output element via `get_init_index` (pytorch#18789, pytorch#18790, pytorch#18791). `amax.out` and `amin.out` do not have this fast path. ### Tests No new tests were added. The existing tests already cover both fast and generic paths. `sum`'s (pytorch#18789), `mean`'s (pytorch#18790), and `var_mean`'s (pytorch#18791) fast-path PRs had no test changes either. | Test | Shape | `dim` | Fast path? | Dtypes covered | | --- | --- | --- | --- | --- | | `AllRealInputOutputPasses` | `{2,3,4}` | `{2}` (innermost) | Yes | `ET_FORALL_REALHBBF16_TYPES` (incl. `Bool`, `Half`, `BFloat16`) | | `AllRealInputOutputPasses` | `{2,3,4}` | `{-2}` (not innermost) | No, fallback | same | | `AllRealInputOutputPasses` | `{2,3,4}` | `{0,1}` (multi-dim) | No, fallback | same | | `AllRealInputOutputPasses` | `{2,2,4}` | null / empty | No, fallback | same | | `InfinityAndNANTest` | `{2,3,4}` | `{-1}` (innermost) | Yes | `ET_FORALL_FLOATHBF16_TYPES` | Same coverage in both `op_amax_test.cpp` and `op_amin_test.cpp`. ``` $ ninja -C cmake-out-bench portable_kernels_test -j16 [6/6] Linking CXX executable kernels/test/portable_kernels_test $ ./cmake-out-bench/kernels/test/portable_kernels_test [==========] Running 1625 tests from 190 test suites. [ PASSED ] 1521 tests. # 0 FAILED, 209 SKIPPED (pre-existing DISABLED/Aten-only tests, unrelated to this change) $ ./cmake-out-bench/kernels/test/portable_kernels_test --gtest_filter="OpAmaxOutTest.*:OpAminOutTest.*" [==========] 10 tests from 2 test suites ran. [ PASSED ] 10 tests. $ lintrunner -a kernels/portable/cpu/op_amax.cpp kernels/portable/cpu/op_amin.cpp ok No lint issues. ``` cc @larryliu0820 @manuelcandales Signed-off-by: Youngsik Yang <vacu9708@gmail.com>
1 parent ff0f92e commit 6cd7ecd

2 files changed

Lines changed: 60 additions & 4 deletions

File tree

kernels/portable/cpu/op_amax.cpp

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,11 +44,39 @@ Tensor& amax_out(
4444
ET_KERNEL_CHECK(
4545
ctx, tensors_have_same_dim_order(in, out), InvalidArgument, out);
4646

47-
ReduceOverDimListPlan plan(in, dim_list);
48-
4947
// @lint-ignore CLANGTIDY facebook-hte-CArray
5048
static constexpr const char op_name[] = "amax.out";
5149

50+
// Fast path: contiguous tensor, single innermost dim reduction.
51+
// Bypasses the generic ReduceOverDimListPlan (per-element stride/index
52+
// recomputation via get_init_index)
53+
if (in.numel() > 0 && dim_list.size() == 1 &&
54+
!executorch::runtime::isComplexType(in.scalar_type()) &&
55+
tensor_is_default_dim_order(in)) {
56+
const int64_t d = dim_list[0] < 0 ? dim_list[0] + in.dim() : dim_list[0];
57+
if (d >= 0 && d < in.dim() && d == in.dim() - 1 &&
58+
tensor_is_contiguous(in)) {
59+
const int64_t reduce_size = in.size(d);
60+
const int64_t outer_size = in.numel() / reduce_size;
61+
62+
ET_SWITCH_REALHBBF16_TYPES(in.scalar_type(), ctx, op_name, CTYPE, [&]() {
63+
const CTYPE* in_data = in.const_data_ptr<CTYPE>();
64+
CTYPE* out_data = out.mutable_data_ptr<CTYPE>();
65+
for (int64_t i = 0; i < outer_size; i++) {
66+
const CTYPE* row = in_data + i * reduce_size;
67+
CTYPE max_v = row[0];
68+
for (int64_t j = 1; j < reduce_size; j++) {
69+
const CTYPE v = row[j];
70+
max_v = utils::isnan_override(v) || v > max_v ? v : max_v;
71+
}
72+
out_data[i] = max_v;
73+
}
74+
});
75+
return out;
76+
}
77+
}
78+
79+
ReduceOverDimListPlan plan(in, dim_list);
5280
ET_SWITCH_REALHBBF16_TYPES(in.scalar_type(), ctx, op_name, CTYPE, [&]() {
5381
CTYPE* out_data = out.mutable_data_ptr<CTYPE>();
5482
const bool success = parallel_for_each_reduce_over_dim_list_output_index(

kernels/portable/cpu/op_amin.cpp

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,11 +43,39 @@ Tensor& amin_out(
4343
ET_KERNEL_CHECK(
4444
ctx, tensors_have_same_dim_order(in, out), InvalidArgument, out);
4545

46-
ReduceOverDimListPlan plan(in, dim_list);
47-
4846
// @lint-ignore CLANGTIDY facebook-hte-CArray
4947
static constexpr const char op_name[] = "amin.out";
5048

49+
// Fast path: contiguous tensor, single innermost dim reduction.
50+
// Bypasses the generic ReduceOverDimListPlan (per-element stride/index
51+
// recomputation via get_init_index) for a tight, directly-indexed loop.
52+
if (in.numel() > 0 && dim_list.size() == 1 &&
53+
!executorch::runtime::isComplexType(in.scalar_type()) &&
54+
tensor_is_default_dim_order(in)) {
55+
const int64_t d = dim_list[0] < 0 ? dim_list[0] + in.dim() : dim_list[0];
56+
if (d >= 0 && d < in.dim() && d == in.dim() - 1 &&
57+
tensor_is_contiguous(in)) {
58+
const int64_t reduce_size = in.size(d);
59+
const int64_t outer_size = in.numel() / reduce_size;
60+
61+
ET_SWITCH_REALHBBF16_TYPES(in.scalar_type(), ctx, op_name, CTYPE, [&]() {
62+
const CTYPE* in_data = in.const_data_ptr<CTYPE>();
63+
CTYPE* out_data = out.mutable_data_ptr<CTYPE>();
64+
for (int64_t i = 0; i < outer_size; i++) {
65+
const CTYPE* row = in_data + i * reduce_size;
66+
CTYPE min_v = row[0];
67+
for (int64_t j = 1; j < reduce_size; j++) {
68+
const CTYPE v = row[j];
69+
min_v = utils::isnan_override(v) || v < min_v ? v : min_v;
70+
}
71+
out_data[i] = min_v;
72+
}
73+
});
74+
return out;
75+
}
76+
}
77+
78+
ReduceOverDimListPlan plan(in, dim_list);
5179
ET_SWITCH_REALHBBF16_TYPES(in.scalar_type(), ctx, op_name, CTYPE, [&]() {
5280
CTYPE* out_data = out.mutable_data_ptr<CTYPE>();
5381
const bool success = parallel_for_each_reduce_over_dim_list_output_index(

0 commit comments

Comments
 (0)