Skip to content

Commit 5d1a023

Browse files
authored
ZJIT: Ensure we specialize the results of the last inlining operation (ruby#17479)
When we introduced the inliner we also added repeated passes of the optimization pipeline. The idea being that we want to optimize the results of inlining and, because we only inline one level deep, allow us to perform inlining on the result of the last inlining operation. The optimization loop would exit if we couldn't inline any more. If we could inline more, there's an upper bound that kicks us out of the loop so we don't try to inline the world. However, if we exited the loop by hitting that upper bound, we didn't end up specializing the results of the last inlining pass. This PR rectifies that. This is immediately visible in the 30k_methods benchmark, where performance roughly doubles. Before: ``` ❯ WARMUP_ITRS=0 MIN_BENCH_ITRS=10 MIN_BENCH_TIME=0 ./run_benchmarks.rb --chruby 'ruby-master --zjit-inline-threshold=30' 30k_methods Running benchmark "30k_methods" (1/1) + /Users/nirvdrum/.rubies/ruby-master/bin/ruby --zjit-inline-threshold\=30 -I harness /Users/nirvdrum/dev/worktrees/ruby-bench/main/benchmarks/30k_methods.rb ruby 4.1.0dev (2026-06-23T13:29:36Z master 13fe77d) +ZJIT dev +PRISM [arm64-darwin25] itr: time #1: 2689ms #2: 33ms #3: 32ms #4: 32ms #5: 32ms #6: 32ms #7: 32ms ruby#8: 35ms ruby#9: 33ms ruby#10: 33ms ``` After: ``` ❯ WARMUP_ITRS=0 MIN_BENCH_ITRS=10 MIN_BENCH_TIME=0 ./run_benchmarks.rb --chruby 'ruby-zjit-opt-last-inline --zjit-inline-threshold=30' 30k_methods Running benchmark "30k_methods" (1/1) + /Users/nirvdrum/.rubies/ruby-zjit-opt-last-inline/bin/ruby --zjit-inline-threshold\=30 -I harness /Users/nirvdrum/dev/worktrees/ruby-bench/main/benchmarks/30k_methods.rb ruby 4.1.0dev (2026-06-25T13:56:41Z zjit-opt-last-inline 18ce64d) +ZJIT dev +PRISM [arm64-darwin25] itr: time #1: 2700ms #2: 17ms #3: 16ms #4: 16ms #5: 17ms #6: 16ms #7: 16ms ruby#8: 17ms ruby#9: 16ms ruby#10: 16ms ``` Fixes Shopify#998.
1 parent 18ce64d commit 5d1a023

3 files changed

Lines changed: 81 additions & 10 deletions

File tree

zjit/src/hir.rs

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6221,19 +6221,27 @@ impl Function {
62216221
}
62226222

62236223
// The optimization pipeline runs in a fixed-point loop so that inlining and
6224-
// type specialization can feed each other: the first iteration inlines direct
6225-
// calls and specializes the inlined code, and subsequent iterations can inline
6226-
// calls that only became monomorphic after the previous round of specialization.
6227-
// Termination is guaranteed because each iteration either inlines at least one
6228-
// call (growing the function toward the inlining budget) or reaches a fixed point.
6229-
for _ in 0..get_option!(inline_max_iterations) {
6224+
// type specialization can feed each other: an iteration inlines direct calls and
6225+
// the next one specializes the freshly inlined code, which in turn can expose
6226+
// calls that only became monomorphic after that specialization. Inlining naturally
6227+
// stops when it reaches a fixed point, while inline_max_iterations sets an upper bound
6228+
// on inlining passes. If we reach the max, we run the loop one more time with inlining
6229+
// disabled in order to optimize the results of the last inlining operation.
6230+
let inline_max_iterations = get_option!(inline_max_iterations);
6231+
for iteration in 0..=inline_max_iterations {
62306232
// Function is assumed to have types inferred already
62316233
run_pass!(type_specialize);
62326234
// The trivial inliner runs first to handle simple cases (constant returns,
62336235
// parameter returns, etc.) without frame push/pop overhead. The general
62346236
// inliner then handles more complex methods that require full inlining.
62356237
run_pass!(inline_trivial);
6236-
let did_inline = run_pass!(inline_methods);
6238+
// Cap inlining at inline_max_iterations passes; the trailing iteration (see above)
6239+
// runs the rest of the pipeline with inlining off.
6240+
let did_inline = if iteration < inline_max_iterations {
6241+
run_pass!(inline_methods)
6242+
} else {
6243+
false
6244+
};
62376245
run_pass!(optimize_c_calls);
62386246
run_pass!(convert_no_profile_sends);
62396247
run_pass!(optimize_load_store);

zjit/src/hir/opt_tests.rs

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17985,6 +17985,66 @@ mod hir_opt_tests {
1798517985
");
1798617986
}
1798717987

17988+
#[test]
17989+
fn test_final_inline_iteration_specializes_inlined_iseq_send() {
17990+
eval("
17991+
def inner(x)
17992+
x + 1
17993+
end
17994+
def outer(x)
17995+
inner(x)
17996+
end
17997+
def test(n)
17998+
outer(n)
17999+
end
18000+
test(1)
18001+
test(1)
18002+
");
18003+
18004+
let old_threshold = get_option!(inline_threshold);
18005+
let old_max_iterations = get_option!(inline_max_iterations);
18006+
unsafe {
18007+
OPTIONS.as_mut().unwrap().inline_threshold = 30;
18008+
OPTIONS.as_mut().unwrap().inline_max_iterations = 1;
18009+
}
18010+
let result = hir_string("test");
18011+
unsafe {
18012+
OPTIONS.as_mut().unwrap().inline_threshold = old_threshold;
18013+
OPTIONS.as_mut().unwrap().inline_max_iterations = old_max_iterations;
18014+
}
18015+
18016+
assert!(result.contains("PushInlineFrame"),
18017+
"Expected outer to be inlined with inline_max_iterations=1:\n{result}");
18018+
assert!(result.contains(" = SendDirect "),
18019+
"Expected the Send inside the final inlined body to be specialized to SendDirect:\n{result}");
18020+
assert!(!result.contains(" = Send "),
18021+
"Expected no unspecialized Send after the final specialization round:\n{result}");
18022+
18023+
assert_snapshot!(result, @"
18024+
fn test@<compiled>:9:
18025+
bb1():
18026+
EntryPoint interpreter
18027+
v1:BasicObject = LoadSelf
18028+
v2:CPtr = LoadSP
18029+
v3:BasicObject = LoadField v2, :n@0x1000
18030+
Jump bb3(v1, v3)
18031+
bb2():
18032+
EntryPoint JIT(0)
18033+
v6:BasicObject = LoadArg :self@0
18034+
v7:BasicObject = LoadArg :n@1
18035+
Jump bb3(v6, v7)
18036+
bb3(v9:BasicObject, v10:BasicObject):
18037+
PatchPoint MethodRedefined(Object@0x1008, outer@0x1010, cme:0x1018)
18038+
v23:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile
18039+
PushInlineFrame v23 (0x1040), v10
18040+
PatchPoint MethodRedefined(Object@0x1008, inner@0x1048, cme:0x1050)
18041+
v43:BasicObject = SendDirect v23, 0x1078, :inner (0x1088), v10
18042+
CheckInterrupts
18043+
PopInlineFrame
18044+
Return v43
18045+
");
18046+
}
18047+
1798818048
#[test]
1798918049
fn test_inline_budget_rejects_when_exceeded() {
1799018050
// The same workload as test_inline_arithmetic_method, which we know inlines

zjit/src/options.rs

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -178,9 +178,12 @@ pub struct Options {
178178
/// Upper bound on how many times the `optimize` fixed-point loop will iterate
179179
/// before giving up. Each iteration runs `type_specialize` → `inline` →
180180
/// `inline_methods` → the rest of the HIR pipeline; in steady state the loop
181-
/// terminates as soon as an iteration fails to inline anything new. The cap
182-
/// exists to bound compile time when something pathological prevents the loop
183-
/// from reaching a fixed point.
181+
/// terminates as soon as an iteration fails to inline anything new. If the
182+
/// cap is hit while inlining is still ongoing, the optimizer runs one final
183+
/// specialization/cleanup round without `inline_methods`, so the callee HIR
184+
/// inserted by the last iteration does not keep unspecialized `Send`s. The
185+
/// cap exists to bound compile time when something pathological prevents the
186+
/// loop from reaching a fixed point.
184187
pub inline_max_iterations: InlineDepth,
185188
}
186189

0 commit comments

Comments
 (0)