fix

chiajunglien · chiajunglien · commit 80bbb53aa6c2 · 2026-07-23T04:02:03.000Z
diff --git a/src/maxtext/input_pipeline/hf_data_processing.py b/src/maxtext/input_pipeline/hf_data_processing.py
@@ -181,7 +181,7 @@ def vision_sft_preprocessing_pipeline(
       data_source=dataset,
       operations=operations,
       sampler=dummy_index_sampler,
-      worker_count=config.grain_worker_count,  # only supports <=1 for now, more workers results in duplicated data
+      worker_count=config.grain_worker_count,
       worker_buffer_size=1,
       read_options=grain.ReadOptions(num_threads=1, prefetch_buffer_size=batch_size * 4),
   )

Original file line number	Diff line number	Diff line change
`@@ -181,7 +181,7 @@ def vision_sft_preprocessing_pipeline(`
`181`	`181`	`data_source=dataset,`
`182`	`182`	`operations=operations,`
`183`	`183`	`sampler=dummy_index_sampler,`
`184`		`- worker_count=config.grain_worker_count, # only supports <=1 for now, more workers results in duplicated data`
	`184`	`+ worker_count=config.grain_worker_count,`
`185`	`185`	`worker_buffer_size=1,`
`186`	`186`	`read_options=grain.ReadOptions(num_threads=1, prefetch_buffer_size=batch_size * 4),`
`187`	`187`	`)`