|
| 1 | +# Copyright 2026 Google LLC |
| 2 | +# |
| 3 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +# you may not use this file except in compliance with the License. |
| 5 | +# You may obtain a copy of the License at |
| 6 | +# |
| 7 | +# https://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +# |
| 9 | +# Unless required by applicable law or agreed to in writing, software |
| 10 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +# See the License for the specific language governing permissions and |
| 13 | +# limitations under the License. |
| 14 | + |
| 15 | +"""Tests for synthetic data sharding.""" |
| 16 | + |
| 17 | +import sys |
| 18 | +from absl.testing import absltest |
| 19 | +from absl.testing import parameterized |
| 20 | +import jax |
| 21 | +from jax.sharding import Mesh |
| 22 | +from jax.experimental import mesh_utils |
| 23 | + |
| 24 | +from maxtext.configs import pyconfig |
| 25 | +from maxtext.input_pipeline.synthetic_data_processing import SyntheticDataIterator |
| 26 | +from maxtext.utils import sharding as maxtext_sharding |
| 27 | +from tests.utils.test_helpers import get_test_config_path |
| 28 | + |
| 29 | + |
| 30 | +class SyntheticDataShardingTest(parameterized.TestCase): |
| 31 | + |
| 32 | + @parameterized.product( |
| 33 | + mesh_shape=[(1, 1), (2, 1), (2, 2)], |
| 34 | + shard_mode=["auto", "explicit"], |
| 35 | + ) |
| 36 | + def test_synthetic_data_sharding(self, mesh_shape, shard_mode): |
| 37 | + devices = jax.devices() |
| 38 | + num_devices = len(devices) |
| 39 | + |
| 40 | + target_devices = mesh_shape[0] * mesh_shape[1] |
| 41 | + |
| 42 | + if num_devices < target_devices: |
| 43 | + self.skipTest(f"Not enough devices. Required: {target_devices}, Available: {num_devices}") |
| 44 | + |
| 45 | + mesh_devices = devices[:target_devices] |
| 46 | + devices_array = mesh_utils.create_device_mesh(mesh_shape, mesh_devices) |
| 47 | + mesh = Mesh(devices_array, ["data", "fsdp"]) |
| 48 | + |
| 49 | + # Initialize config |
| 50 | + init_kwargs = { |
| 51 | + "per_device_batch_size": 2.0, |
| 52 | + "num_target_devices": len(mesh_devices), |
| 53 | + "run_name": "test", |
| 54 | + "enable_checkpointing": False, |
| 55 | + "dataset_type": "synthetic", |
| 56 | + "model_name": "llama3.1-8b", |
| 57 | + "max_target_length": 16, |
| 58 | + "shard_mode": shard_mode, |
| 59 | + } |
| 60 | + config = pyconfig.initialize( |
| 61 | + [sys.argv[0], get_test_config_path()], |
| 62 | + **init_kwargs, |
| 63 | + ) |
| 64 | + |
| 65 | + iterator = SyntheticDataIterator(config, mesh) |
| 66 | + batch = next(iterator) |
| 67 | + |
| 68 | + self.assertIn("inputs", batch) |
| 69 | + inputs = batch["inputs"] |
| 70 | + |
| 71 | + self.assertEqual(inputs.shape, (config.global_batch_size_to_load, config.max_target_length)) |
| 72 | + |
| 73 | + # Expected sharding: mapping of logical axes to physical mesh |
| 74 | + expected_sharding = maxtext_sharding.get_input_data_sharding(config, mesh) |
| 75 | + self.assertEqual(inputs.sharding, expected_sharding) |
| 76 | + |
| 77 | + |
| 78 | +if __name__ == "__main__": |
| 79 | + absltest.main() |
0 commit comments