Skip to content

Commit beff73e

Browse files
wangkuiyichanglan
authored andcommitted
Fix bug in input_tf_data.py
GitOrigin-RevId: 0c36bd0
1 parent 6534c36 commit beff73e

1 file changed

Lines changed: 8 additions & 2 deletions

File tree

axlearn/common/input_tf_data.py

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -261,7 +261,9 @@ def _maybe_shard_examples(
261261
"Proceed to split examples anyway. May result in error if "
262262
"number of examples < number of required_shards."
263263
)
264-
shard_index = read_config.shard_index or jax.process_index() # type: ignore
264+
shard_index = (
265+
read_config.shard_index if read_config.shard_index is not None else jax.process_index()
266+
)
265267
per_process_split = tfds.even_splits(
266268
per_process_split, n=required_shards, drop_remainder=False
267269
)[shard_index]
@@ -357,7 +359,11 @@ def fn() -> tf.data.Dataset:
357359
per_process_split = split
358360
available_shards = _infer_num_shards(builder, split)
359361
if available_shards is not None:
360-
required_shards = read_config.num_shards or jax.process_count() # type: ignore
362+
required_shards = (
363+
read_config.num_shards
364+
if read_config.num_shards is not None
365+
else jax.process_count()
366+
)
361367
if available_shards < required_shards:
362368
per_process_split = _maybe_shard_examples(
363369
builder=builder,

0 commit comments

Comments
 (0)