Skip to content

Commit 64ec62a

Browse files
committed
[Perf] Adstack max-reducer: gate spec recognizer on bound_expr capture so dense-adstack kernels skip the dispatch entirely
1 parent f19244c commit 64ec62a

2 files changed

Lines changed: 19 additions & 6 deletions

File tree

quadrants/codegen/llvm/codegen_llvm.cpp

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1994,10 +1994,17 @@ void TaskCodeGenLLVM::finalize_offloaded_task_function() {
19941994
current_task->ad_stack.allocas = ad_stack_allocas_info_;
19951995
current_task->ad_stack.size_exprs = ad_stack_size_exprs_;
19961996
current_task->ad_stack.bound_expr = ad_stack_static_bound_expr_;
1997-
// recognize `MaxOverRange` nodes that the runtime can reduce in parallel via the dedicated max-reducer dispatch
1997+
// Recognize `MaxOverRange` nodes that the runtime can reduce in parallel via the dedicated max-reducer dispatch
19981998
// instead of letting the per-thread sizer enumerate. Indexing matches `ad_stack_size_exprs_` (same iteration order
1999-
// as the pre-scan above).
2000-
current_task->ad_stack.max_reducer_specs = recognize_adstack_max_reducer_specs(ad_stack_size_exprs_);
1999+
// as the pre-scan above). Gated on the same `ad_stack_sparse_threshold_bytes` predicate that decides whether
2000+
// `analyze_adstack_static_bounds` captures `bound_expr`: when the dense worst-case adstack footprint fits under the
2001+
// threshold, the per-thread sizer's host-eval path resolves the size_exprs cheaply and the parallel dispatch's
2002+
// per-launch h2d copy + reducer kernel + d2h readback is strictly more expensive than the work it replaces. Above
2003+
// threshold, the kernel is in sparse mode and a captured `MaxOverRange` whose iteration count would otherwise blow
2004+
// the `1<<24` cap is the reason the max-reducer exists. Mirrors the gate in `spirv_codegen.cpp`.
2005+
if (ad_stack_static_bound_expr_.has_value()) {
2006+
current_task->ad_stack.max_reducer_specs = recognize_adstack_max_reducer_specs(ad_stack_size_exprs_);
2007+
}
20012008
// Snodes the task body mutates. Persisted on `OffloadedTask::snode_writes` so the LLVM
20022009
// launcher can invalidate the per-task adstack metadata cache when a kernel that runs in
20032010
// between mutated a SNode an enclosing `size_expr::FieldLoad` reads. Mirrors the SPIR-V

quadrants/codegen/spirv/spirv_codegen.cpp

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -203,10 +203,16 @@ TaskCodegen::Result TaskCodegen::run() {
203203

204204
task_attribs_.ad_stack.per_thread_stride_float_compile_time = ad_stack_heap_per_thread_stride_float_;
205205
task_attribs_.ad_stack.per_thread_stride_int_compile_time = ad_stack_heap_per_thread_stride_int_;
206-
// recognize `MaxOverRange` nodes the runtime can reduce in parallel via the dedicated max-reducer dispatch instead of
206+
// Recognize `MaxOverRange` nodes the runtime can reduce in parallel via the dedicated max-reducer dispatch instead of
207207
// letting the per-thread sizer enumerate. Indexing matches `task_attribs_.ad_stack.allocas` (each entry's `size_expr`
208-
// is the per-stack tree captured above).
209-
{
208+
// is the per-stack tree captured above). Gated on the same `ad_stack_sparse_threshold_bytes` predicate that decides
209+
// whether `analyze_adstack_static_bounds` captures `bound_expr`: when the dense worst-case adstack footprint fits
210+
// under the threshold, the per-thread sizer's host-eval path resolves the size_exprs cheaply and the parallel
211+
// dispatch's per-launch buffer fill + cmdlist + `submit_synced` is strictly more expensive than the work it replaces.
212+
// Above threshold, the kernel is in sparse mode and a captured `MaxOverRange` whose iteration count would otherwise
213+
// blow the `1<<24` cap is the reason the max-reducer exists. The recognizer is the same in both cases; the gate just
214+
// suppresses the per-launch dispatch site by leaving `max_reducer_specs` empty.
215+
if (adstack_analysis.bound_expr.has_value()) {
210216
std::vector<SerializedSizeExpr> per_stack_size_exprs;
211217
per_stack_size_exprs.reserve(task_attribs_.ad_stack.allocas.size());
212218
for (const auto &a : task_attribs_.ad_stack.allocas) {

0 commit comments

Comments
 (0)