Conditionally pass run_post_process to hf_builder.as_dataset.

PiperOrigin-RevId: 911883251

Author: The TensorFlow Datasets Authors

tensorflow_datasets/core/dataset_builders/huggingface_dataset_builder.py

@@ -29,6 +29,7 @@
 from collections.abc import Mapping, Sequence
 import dataclasses
 import functools
+import inspect
 import itertools
 import multiprocessing
 import os
@@ -136,9 +137,18 @@ def _write_shard(
   def get_serialized_examples_iter():
     nonlocal num_bytes
     nonlocal num_exceptions
-    dataset = hf_builder.as_dataset(
-        split=shard_spec.shard_split, run_post_process=False
-    )
+    as_dataset_kwargs = dict(split=shard_spec.shard_split)
+    # We dynamically construct the arguments because the 'run_post_process'
+    # parameter was only added in Hugging Face 'datasets' 2.9.1. In some
+    # environments, such as GitHub CI, an older version of the library
+    # may be installed due to dependency resolution conflicts with
+    # apache-beam.
+    if (
+        'run_post_process'
+        in inspect.signature(hf_builder.as_dataset).parameters
+    ):
+      as_dataset_kwargs['run_post_process'] = False
+    dataset = hf_builder.as_dataset(**as_dataset_kwargs)
     for i in range(shard_spec.num_examples):
       try:
         hf_value = dataset[i]