Skip to content

Commit 028516f

Browse files
committed
Fix rank mismatch in MaxText synthetic data sharding
* Change SyntheticDataIterator to use get_input_data_sharding instead of manual 1D sharding. * This ensures the sharding spec is 2D, matching the rank of the output tensors. * Fixes AssertionError: (1, 2) in JAX sharding validation on some JAX builds. * Remove unused PartitionSpec import in synthetic_data_processing.py. * Add parameterized unit test `tests/unit/synthetic_data_test.py` to verify synthetic data sharding works for both 'auto' and 'explicit' shard modes.
1 parent be7748b commit 028516f

2 files changed

Lines changed: 81 additions & 3 deletions

File tree

src/maxtext/input_pipeline/synthetic_data_processing.py

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121

2222
import jax
2323
import jax.numpy as jnp
24-
from jax.sharding import PartitionSpec as P
24+
2525

2626
from maxtext.input_pipeline import multihost_dataloading
2727
from maxtext.configs import pyconfig
@@ -36,8 +36,7 @@ class SyntheticDataIterator:
3636
def __init__(self, config, mesh):
3737
self.mesh = mesh
3838
self.config = config
39-
data_pspec = sharding.remove_size_one_mesh_axis(P(*config.data_sharding), mesh)
40-
data_pspec_shardings = jax.tree_util.tree_map(lambda p: jax.sharding.NamedSharding(mesh, p), data_pspec)
39+
data_pspec_shardings = sharding.get_input_data_sharding(config, mesh)
4140
self.data_generator = jax.jit(
4241
SyntheticDataIterator.raw_generate_synthetic_data, out_shardings=data_pspec_shardings, static_argnums=0
4342
)

tests/unit/synthetic_data_test.py

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""Tests for synthetic data sharding."""
16+
17+
import sys
18+
from absl.testing import absltest
19+
from absl.testing import parameterized
20+
import jax
21+
from jax.sharding import Mesh
22+
from jax.experimental import mesh_utils
23+
24+
from maxtext.configs import pyconfig
25+
from maxtext.input_pipeline.synthetic_data_processing import SyntheticDataIterator
26+
from maxtext.utils import sharding as maxtext_sharding
27+
from tests.utils.test_helpers import get_test_config_path
28+
29+
30+
class SyntheticDataShardingTest(parameterized.TestCase):
31+
32+
@parameterized.product(
33+
mesh_shape=[(1, 1), (2, 1), (2, 2)],
34+
shard_mode=["auto", "explicit"],
35+
)
36+
def test_synthetic_data_sharding(self, mesh_shape, shard_mode):
37+
devices = jax.devices()
38+
num_devices = len(devices)
39+
40+
target_devices = mesh_shape[0] * mesh_shape[1]
41+
42+
if num_devices < target_devices:
43+
self.skipTest(f"Not enough devices. Required: {target_devices}, Available: {num_devices}")
44+
45+
mesh_devices = devices[:target_devices]
46+
devices_array = mesh_utils.create_device_mesh(mesh_shape, mesh_devices)
47+
mesh = Mesh(devices_array, ["data", "fsdp"])
48+
49+
# Initialize config
50+
init_kwargs = {
51+
"per_device_batch_size": 2.0,
52+
"num_target_devices": len(mesh_devices),
53+
"run_name": "test",
54+
"enable_checkpointing": False,
55+
"dataset_type": "synthetic",
56+
"model_name": "llama3.1-8b",
57+
"max_target_length": 16,
58+
"shard_mode": shard_mode,
59+
}
60+
config = pyconfig.initialize(
61+
[sys.argv[0], get_test_config_path()],
62+
**init_kwargs,
63+
)
64+
65+
iterator = SyntheticDataIterator(config, mesh)
66+
batch = next(iterator)
67+
68+
self.assertIn("inputs", batch)
69+
inputs = batch["inputs"]
70+
71+
self.assertEqual(inputs.shape, (config.global_batch_size_to_load, config.max_target_length))
72+
73+
# Expected sharding: mapping of logical axes to physical mesh
74+
expected_sharding = maxtext_sharding.get_input_data_sharding(config, mesh)
75+
self.assertEqual(inputs.sharding, expected_sharding)
76+
77+
78+
if __name__ == "__main__":
79+
absltest.main()

0 commit comments

Comments
 (0)