Skip to content

Commit 4bb8821

Browse files
Merge pull request #447 from AI-Hypercomputer:tile_size_search
PiperOrigin-RevId: 950883684
2 parents a369207 + 4588c99 commit 4bb8821

11 files changed

Lines changed: 1033 additions & 3 deletions

README.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
[![Unit Tests](https://github.com/AI-Hypercomputer/maxdiffusion/actions/workflows/UnitTests.yml/badge.svg)](https://github.com/AI-Hypercomputer/maxdiffusion/actions/workflows/UnitTests.yml)
1818

1919
# What's new?
20+
- **`2026/07/14`**: Automatic attention tile-size (`block_q`/`block_kv`) search for Wan is now supported.
21+
- **`2026/06/26`**: 2D ring (USP) attention with a custom splash kernel is now supported for Wan (`ulysses_ring_custom`), splitting context parallelism into an intra-chip Ulysses axis and a cross-chip ring axis.
2022
- **`2026/04/16`**: Support for Tokamax Ring Attention kernel is now added.
2123
- **`2026/03/31`**: Wan2.2 SenCache inference is now supported for T2V and I2V (up to 1.4x speedup)
2224
- **`2026/03/25`**: Wan2.1 and Wan2.2 Magcache inference is now supported
@@ -715,6 +717,10 @@ We added ring attention support for Wan models. Below are the stats for one `720
715717

716718
(* There are some known stability issues for ring attention on 16 TPUs, please use `tokamax_flash` attention instead.)
717719

720+
### Automatic Tile-Size Search
721+
722+
The optimal attention tile sizes (`block_q` / `block_kv`) depend on the sequence length, VMEM, sharding, and accelerator, and the feasibility edge is a VMEM OOM with no clean closed form — so we tune them empirically. Passing `enable_tile_search=true` to `generate_wan.py` runs a fast one-DiT-block grid search before inference and injects the winning block sizes into `flash_block_sizes` (`tile_search_mode=smart` by default; the search is opt-in and off by default). The core is model-agnostic (`utils/tile_size_grid_search.py`) with a per-model plug (`utils/wan_block_benchmark.py`), which also runs standalone via `python -m maxdiffusion.utils.wan_block_benchmark ... --smart-search`.
723+
718724
## Flux
719725

720726
First make sure you have permissions to access the Flux repos in Huggingface.

src/maxdiffusion/configs/base_wan_14b.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,14 @@ flash_block_sizes: {
142142
# "block_kv_dq" : 3072
143143
# }
144144
# GroupNorm groups
145+
# Tile-size auto-tuning. When enable_tile_search: True, generate_wan runs a fast one-DiT-block
146+
# grid search (maxdiffusion/utils/tile_size_grid_search.py) before inference and overwrites
147+
# flash_block_sizes' block_q/block_kv/block_kv_compute with the winner. Default off (no-op).
148+
enable_tile_search: False
149+
tile_search_mode: 'smart' # 'smart' (VMEM-capped candidate ladders) | 'full' (2D sweep)
150+
tile_search_iters: 10
151+
tile_search_out: '' # dir for the results CSV; '' -> print only
152+
145153
norm_num_groups: 32
146154

147155
# train text_encoder - Currently not supported for SDXL

src/maxdiffusion/configs/base_wan_1_3b.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,14 @@ flash_block_sizes: {
116116
"use_fused_bwd_kernel": False,
117117
}
118118
# GroupNorm groups
119+
# Tile-size auto-tuning. When enable_tile_search: True, generate_wan runs a fast one-DiT-block
120+
# grid search (maxdiffusion/utils/tile_size_grid_search.py) before inference and overwrites
121+
# flash_block_sizes' block_q/block_kv/block_kv_compute with the winner. Default off (no-op).
122+
enable_tile_search: False
123+
tile_search_mode: 'smart' # 'smart' (VMEM-capped candidate ladders) | 'full' (2D sweep)
124+
tile_search_iters: 10
125+
tile_search_out: '' # dir for the results CSV; '' -> print only
126+
119127
norm_num_groups: 32
120128

121129
# train text_encoder - Currently not supported for SDXL

src/maxdiffusion/configs/base_wan_27b.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,14 @@ flash_block_sizes: {
137137
# "use_fused_bwd_kernel": False,
138138
# }
139139
# GroupNorm groups
140+
# Tile-size auto-tuning. When enable_tile_search: True, generate_wan runs a fast one-DiT-block
141+
# grid search (maxdiffusion/utils/tile_size_grid_search.py) before inference and overwrites
142+
# flash_block_sizes' block_q/block_kv/block_kv_compute with the winner. Default off (no-op).
143+
enable_tile_search: False
144+
tile_search_mode: 'smart' # 'smart' (VMEM-capped candidate ladders) | 'full' (2D sweep)
145+
tile_search_iters: 10
146+
tile_search_out: '' # dir for the results CSV; '' -> print only
147+
140148
norm_num_groups: 32
141149

142150
# train text_encoder - Currently not supported for SDXL

src/maxdiffusion/configs/base_wan_animate.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,14 @@ flash_block_sizes: {
131131
# "use_fused_bwd_kernel": False,
132132
# }
133133
# GroupNorm groups
134+
# Tile-size auto-tuning. When enable_tile_search: True, generate_wan runs a fast one-DiT-block
135+
# grid search (maxdiffusion/utils/tile_size_grid_search.py) before inference and overwrites
136+
# flash_block_sizes' block_q/block_kv/block_kv_compute with the winner. Default off (no-op).
137+
enable_tile_search: False
138+
tile_search_mode: 'smart' # 'smart' (VMEM-capped candidate ladders) | 'full' (2D sweep)
139+
tile_search_iters: 10
140+
tile_search_out: '' # dir for the results CSV; '' -> print only
141+
134142
norm_num_groups: 32
135143

136144
# Text encoder training keys are unused by generate_wan_animate.py.

src/maxdiffusion/configs/base_wan_i2v_14b.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,14 @@ flash_block_sizes: {
129129
# "use_fused_bwd_kernel": False,
130130
# }
131131
# GroupNorm groups
132+
# Tile-size auto-tuning. When enable_tile_search: True, generate_wan runs a fast one-DiT-block
133+
# grid search (maxdiffusion/utils/tile_size_grid_search.py) before inference and overwrites
134+
# flash_block_sizes' block_q/block_kv/block_kv_compute with the winner. Default off (no-op).
135+
enable_tile_search: False
136+
tile_search_mode: 'smart' # 'smart' (VMEM-capped candidate ladders) | 'full' (2D sweep)
137+
tile_search_iters: 10
138+
tile_search_out: '' # dir for the results CSV; '' -> print only
139+
132140
norm_num_groups: 32
133141

134142
# train text_encoder - Currently not supported for SDXL

src/maxdiffusion/configs/base_wan_i2v_27b.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,14 @@ flash_block_sizes: {
130130
# "use_fused_bwd_kernel": False,
131131
# }
132132
# GroupNorm groups
133+
# Tile-size auto-tuning. When enable_tile_search: True, generate_wan runs a fast one-DiT-block
134+
# grid search (maxdiffusion/utils/tile_size_grid_search.py) before inference and overwrites
135+
# flash_block_sizes' block_q/block_kv/block_kv_compute with the winner. Default off (no-op).
136+
enable_tile_search: False
137+
tile_search_mode: 'smart' # 'smart' (VMEM-capped candidate ladders) | 'full' (2D sweep)
138+
tile_search_iters: 10
139+
tile_search_out: '' # dir for the results CSV; '' -> print only
140+
133141
norm_num_groups: 32
134142

135143
# train text_encoder - Currently not supported for SDXL

src/maxdiffusion/generate_wan.py

Lines changed: 56 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -195,8 +195,50 @@ def inference_generate_video(config, pipeline, filename_prefix=""):
195195
return
196196

197197

198+
def maybe_tune_block_sizes(config):
199+
"""If enable_tile_search, run a fast one-DiT-block tile-size grid search and overwrite
200+
flash_block_sizes' block_q/block_kv/block_kv_compute with the winner IN PLACE, before the
201+
transformer (which bakes block sizes in at construction) is built.
202+
203+
Flags are read defensively so this is a safe no-op (grid search OFF) for any config that
204+
doesn't declare them -- not every WAN yaml carries the tile_search_* keys."""
205+
keys = config.get_keys()
206+
if not keys.get("enable_tile_search", False):
207+
return
208+
from maxdiffusion.utils.tile_size_grid_search import grid_search
209+
from maxdiffusion.utils.wan_block_benchmark import WanBlockBenchmark
210+
211+
mesh = jax.sharding.Mesh(max_utils.create_device_mesh(config), config.mesh_axes)
212+
bench = WanBlockBenchmark.from_config(config, mesh)
213+
max_logging.log(f"[tile-search] tuning block sizes for {bench.label} before inference...")
214+
result = grid_search(
215+
bench,
216+
mode=keys.get("tile_search_mode", "smart"),
217+
iters=keys.get("tile_search_iters", 10),
218+
out_dir=(keys.get("tile_search_out", "") or None),
219+
log=max_logging.log,
220+
)
221+
if result.best is None:
222+
max_logging.log("[tile-search] no config succeeded; keeping configured flash_block_sizes")
223+
return
224+
fbs = dict(config.flash_block_sizes)
225+
fbs.update({
226+
"block_q": result.best.bq,
227+
"block_kv": result.best.bkv,
228+
"block_kv_compute": result.best.bkv_compute,
229+
"block_kv_compute_in": result.best.bkv_compute,
230+
})
231+
config.get_keys()["flash_block_sizes"] = fbs # config is immutable via setattr; mutate raw dict
232+
max_logging.log(
233+
f"[tile-search] using block_q={result.best.bq} block_kv={result.best.bkv} "
234+
f"(block-bench {result.best.mean_ms:.2f} ms)"
235+
)
236+
237+
198238
def run(config, pipeline=None, filename_prefix="", commit_hash=None):
199239
model_key = config.model_name
240+
if pipeline is None:
241+
maybe_tune_block_sizes(config)
200242
writer = max_utils.initialize_summary_writer(config)
201243
if jax.process_index() == 0 and writer:
202244
max_logging.log(f"TensorBoard logs will be written to: {config.tensorboard_dir}")
@@ -237,7 +279,10 @@ def run(config, pipeline=None, filename_prefix="", commit_hash=None):
237279
pipeline, _, _ = checkpoint_loader.load_checkpoint(checkpoint_step)
238280
else:
239281
pipeline = checkpoint_loader.load_pretrained_pipeline_or_diffusers(
240-
config, pipeline_cls, pretrained_state_sources, pretrained_config_transformer_attr
282+
config,
283+
pipeline_cls,
284+
pretrained_state_sources,
285+
pretrained_config_transformer_attr,
241286
)
242287
load_time = time.perf_counter() - load_start
243288
max_logging.log(f"load_time: {load_time:.1f}s")
@@ -377,7 +422,11 @@ def run(config, pipeline=None, filename_prefix="", commit_hash=None):
377422
num_videos = num_devices * config.per_device_batch_size
378423
if num_videos > 0:
379424
generation_time_per_video = generation_time / num_videos
380-
writer.add_scalar("inference/generation_time_per_video", generation_time_per_video, global_step=0)
425+
writer.add_scalar(
426+
"inference/generation_time_per_video",
427+
generation_time_per_video,
428+
global_step=0,
429+
)
381430
max_logging.log(f"generation time per video: {generation_time_per_video}")
382431
else:
383432
max_logging.log("Warning: Number of videos is zero, cannot calculate generation_time_per_video.")
@@ -421,7 +470,11 @@ def run(config, pipeline=None, filename_prefix="", commit_hash=None):
421470
generation_time_with_profiler = time.perf_counter() - s0
422471
max_logging.log(f"generation_time_with_profiler: {generation_time_with_profiler}")
423472
if writer and jax.process_index() == 0:
424-
writer.add_scalar("inference/generation_time_with_profiler", generation_time_with_profiler, global_step=0)
473+
writer.add_scalar(
474+
"inference/generation_time_with_profiler",
475+
generation_time_with_profiler,
476+
global_step=0,
477+
)
425478

426479
return saved_video_path
427480

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
"""
2+
Copyright 2026 Google LLC
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
https://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
"""
16+
17+
import time
18+
import unittest
19+
20+
from maxdiffusion.utils.tile_size_grid_search import (
21+
MXU_TILE,
22+
VPU_LANE,
23+
BenchResult,
24+
BlockBenchmark,
25+
bkv_candidates,
26+
bq_candidates,
27+
grid_search,
28+
padding_of,
29+
smart_grid,
30+
time_callable,
31+
vmem_bkv_ceiling,
32+
)
33+
34+
# per-shard seq the ring U=1 kernel tiles (75600 / 8); the empirical winner is bq=9472, bkv=1024.
35+
RING_SEQ = 9450
36+
VMEM_64MB = 64 * 1024 * 1024
37+
38+
39+
class _MockRingBench(BlockBenchmark):
40+
"""Synthetic benchmark encoding the measured ring behaviour: fewer Q-tiles is faster, the
41+
bkv_compute sweet spot is ~1024, odd-128 blocks pay a half-MXU-pass penalty, and a score
42+
tile that overflows VMEM OOMs. Lets us test the orchestrator with no TPU."""
43+
44+
label = "mock-ring"
45+
46+
def __init__(self, seq=RING_SEQ, vmem=VMEM_64MB):
47+
self.seq, self._vmem = seq, vmem
48+
49+
def tiled_seq_lens(self):
50+
return (self.seq, self.seq)
51+
52+
def vmem_bytes(self):
53+
return self._vmem
54+
55+
def run(self, bq, bkv, *, bkv_compute=None, iters=10, warmup=2):
56+
cmp = bkv_compute or bkv
57+
if bq * cmp * 4 + 15e6 > self._vmem: # score tile f32 + ~15MB ring overhead
58+
return BenchResult(bq, bkv, cmp, "oom")
59+
n_q = padding_of(self.seq, bq).n_blocks
60+
n_kv = padding_of(self.seq, bkv).n_blocks
61+
ms = 60 + 4.0 * n_q + 0.9 * n_kv - 3.0 * min(cmp, 1024) / 1024
62+
ms += 1.4 * (bq % MXU_TILE != 0) + 1.4 * (bkv % MXU_TILE != 0)
63+
return BenchResult(bq, bkv, cmp, "ok", mean_ms=round(ms, 2), std_ms=0.2, compile_ms=25000.0)
64+
65+
66+
class PaddingMathTest(unittest.TestCase):
67+
68+
def test_padding_of(self):
69+
p = padding_of(RING_SEQ, 1024)
70+
self.assertEqual(p.n_blocks, 10)
71+
self.assertEqual(p.padded_len, 10240)
72+
self.assertEqual(p.pad, 790)
73+
74+
def test_single_tile_is_low_pad(self):
75+
self.assertEqual(padding_of(RING_SEQ, 9472).pad, 22) # 37 * 256
76+
77+
78+
class CandidateTest(unittest.TestCase):
79+
80+
def test_bq_fewest_tile_ladder(self):
81+
# single-block ceiling fits, so the ladder starts at n=1 (bq=9472) and includes the winner.
82+
bqs = bq_candidates(RING_SEQ, k=3, spread=0)
83+
self.assertEqual(bqs[0], 9472)
84+
self.assertTrue(all(b % VPU_LANE == 0 for b in bqs))
85+
86+
def test_bkv_largest_fits_includes_winner(self):
87+
ceil = vmem_bkv_ceiling(9472, vmem_bytes=VMEM_64MB)
88+
bkvs = bkv_candidates(RING_SEQ, k=3, max_block=ceil)
89+
self.assertIn(1024, bkvs) # the measured optimum, largest 256-mult that fits at bq=9472
90+
91+
def test_smart_grid_pairs_winner(self):
92+
pairs = smart_grid(RING_SEQ, RING_SEQ, vmem_bytes=VMEM_64MB, dtype_bytes=4)
93+
self.assertIn((9472, 1024), pairs)
94+
95+
def test_candidates_not_strictly_256(self):
96+
# 128-multiples (e.g. 896) must be admissible, not filtered out.
97+
bkvs = bkv_candidates(RING_SEQ, k=3, max_block=vmem_bkv_ceiling(9472, vmem_bytes=VMEM_64MB))
98+
self.assertTrue(any(b % MXU_TILE != 0 for b in bkvs))
99+
100+
101+
class TimingTest(unittest.TestCase):
102+
103+
def test_compile_excluded_from_mean(self):
104+
calls = {"n": 0}
105+
106+
def fake_fn():
107+
calls["n"] += 1
108+
time.sleep(0.20 if calls["n"] == 1 else 0.01) # call #1 = "compile"
109+
return calls["n"]
110+
111+
mean, _, times, compile_ms = time_callable(fake_fn, iters=5, warmup=2)
112+
self.assertGreater(compile_ms, 150.0) # the 200ms first call is captured here...
113+
self.assertLess(mean, 30.0) # ...and NOT in the steady-state mean
114+
self.assertEqual(len(times), 5)
115+
116+
117+
class OrchestratorTest(unittest.TestCase):
118+
119+
def test_smart_search_picks_measured_winner(self):
120+
res = grid_search(_MockRingBench(), mode="smart", iters=10, log=lambda *a, **k: None)
121+
self.assertIsNotNone(res.best)
122+
self.assertEqual((res.best.bq, res.best.bkv), (9472, 1024))
123+
124+
def test_oom_configs_pruned_not_raised(self):
125+
# a tiny VMEM budget OOMs the big pairs; search must still return (or None), never raise.
126+
res = grid_search(
127+
_MockRingBench(vmem=8 * 1024 * 1024),
128+
mode="smart",
129+
iters=2,
130+
log=lambda *a, **k: None,
131+
)
132+
self.assertTrue(any(r.status == "oom" for r in res.results))
133+
134+
135+
if __name__ == "__main__":
136+
unittest.main()

0 commit comments

Comments
 (0)