File tree Expand file tree Collapse file tree
Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -261,7 +261,9 @@ def _maybe_shard_examples(
261261 "Proceed to split examples anyway. May result in error if "
262262 "number of examples < number of required_shards."
263263 )
264- shard_index = read_config .shard_index or jax .process_index () # type: ignore
264+ shard_index = (
265+ read_config .shard_index if read_config .shard_index is not None else jax .process_index ()
266+ )
265267 per_process_split = tfds .even_splits (
266268 per_process_split , n = required_shards , drop_remainder = False
267269 )[shard_index ]
@@ -357,7 +359,11 @@ def fn() -> tf.data.Dataset:
357359 per_process_split = split
358360 available_shards = _infer_num_shards (builder , split )
359361 if available_shards is not None :
360- required_shards = read_config .num_shards or jax .process_count () # type: ignore
362+ required_shards = (
363+ read_config .num_shards
364+ if read_config .num_shards is not None
365+ else jax .process_count ()
366+ )
361367 if available_shards < required_shards :
362368 per_process_split = _maybe_shard_examples (
363369 builder = builder ,
You can’t perform that action at this time.
0 commit comments