Fix some tests that use update compat flag.

Author: claudevdm

sdks/python/apache_beam/internal/pickler.py

@@ -33,7 +33,7 @@
 try:
   from apache_beam.internal import dill_pickler
 except ImportError:
-  dill_pickler = None
+  dill_pickler = None  # type: ignore[assignment]
 
 USE_CLOUDPICKLE = 'cloudpickle'
 USE_DILL = 'dill'

sdks/python/apache_beam/io/gcp/bigquery_file_loads_test.py

@@ -63,6 +63,11 @@
 except ImportError:
   raise unittest.SkipTest('GCP dependencies are not installed')
 
+try:
+  import dill
+except ImportError:
+  dill = None
+
 _LOGGER = logging.getLogger(__name__)
 
 _DESTINATION_ELEMENT_PAIRS = [
@@ -406,6 +411,13 @@ def test_partition_files_dofn_size_split(self):
           label='CheckSinglePartition')
 
 
+def maybe_skip(compat_version):
+  if compat_version and not dill:
+    raise unittest.SkipTest(
+        'Dill dependency not installed which is required for compat_version'
+        ' <= 2.67.0')
+
+
 class TestBigQueryFileLoads(_TestCaseWithTempDirCleanUp):
   def test_trigger_load_jobs_with_empty_files(self):
     destination = "project:dataset.table"
@@ -485,7 +497,9 @@ def test_records_traverse_transform_with_mocks(self):
       param(compat_version=None),
       param(compat_version="2.64.0"),
   ])
+  @pytest.mark.uses_dill
   def test_reshuffle_before_load(self, compat_version):
+    maybe_skip(compat_version)
     destination = 'project1:dataset1.table1'
 
     job_reference = bigquery_api.JobReference()
@@ -994,6 +1008,7 @@ def dynamic_destination_resolver(element, *side_inputs):
   ])
   def test_triggering_frequency(
       self, is_streaming, with_auto_sharding, compat_version):
+    maybe_skip(compat_version)
     destination = 'project1:dataset1.table1'
 
     job_reference = bigquery_api.JobReference()

sdks/python/apache_beam/pipeline_test.py

@@ -177,7 +177,9 @@ def expand(self, pcoll):
         _ = pipeline | ParentTransform() | beam.Map(lambda x: x + 1)
 
   @mock.patch('logging.info')
+  @pytest.mark.uses_dill
   def test_runner_overrides_default_pickler(self, mock_info):
+    pytest.importorskip("dill")
     with mock.patch.object(PipelineRunner,
                            'default_pickle_library_override') as mock_fn:
       mock_fn.return_value = 'dill'

sdks/python/apache_beam/transforms/util_test.py

@@ -83,6 +83,11 @@
 from apache_beam.utils.windowed_value import PaneInfoTiming
 from apache_beam.utils.windowed_value import WindowedValue
 
+try:
+  import dill
+except ImportError:
+  dill = None
+
 warnings.filterwarnings(
     'ignore', category=FutureWarning, module='apache_beam.transform.util_test')
 
@@ -112,6 +117,13 @@ def is_deterministic(self):
     return True
 
 
+def maybe_skip(compat_version):
+  if compat_version and not dill:
+    raise unittest.SkipTest(
+        'Dill dependency not installed which is required for compat_version'
+        ' <= 2.67.0')
+
+
 class CoGroupByKeyTest(unittest.TestCase):
   def test_co_group_by_key_on_tuple(self):
     with TestPipeline() as pipeline:
@@ -997,8 +1009,10 @@ def test_reshuffle_streaming_global_window_with_buckets(self):
       param(compat_version=None),
       param(compat_version="2.64.0"),
   ])
+  @pytest.mark.uses_dill
   def test_reshuffle_custom_window_preserves_metadata(self, compat_version):
     """Tests that Reshuffle preserves pane info."""
+    maybe_skip(compat_version)
     element_count = 12
     timestamp_value = timestamp.Timestamp(0)
     l = [
@@ -1098,10 +1112,11 @@ def test_reshuffle_custom_window_preserves_metadata(self, compat_version):
       param(compat_version=None),
       param(compat_version="2.64.0"),
   ])
+  @pytest.mark.uses_dill
   def test_reshuffle_default_window_preserves_metadata(self, compat_version):
     """Tests that Reshuffle preserves timestamp, window, and pane info
     metadata."""
-
+    maybe_skip(compat_version)
     no_firing = PaneInfo(
         is_first=True,
         is_last=True,

sdks/python/apache_beam/typehints/schemas_test.py

@@ -720,7 +720,8 @@ def test_named_fields_roundtrip(self, named_fields):
 @pytest.mark.uses_dill
 class PickleTest(unittest.TestCase):
   def setUp(self):
-    if PickleTest.pickler == 'dill':
+    # pylint: disable=access-member-before-definition
+    if self.pickler == 'dill':
       self.pickler = pytest.importorskip("dill")
 
   def test_generated_class_pickle_instance(self):

sdks/python/setup.py

@@ -407,10 +407,11 @@ def get_portability_package_data():
       extras_require={
           'dill': [
             # Dill doesn't have forwards-compatibility guarantees within minor
-            # version. Pickles created with a new version of dill may not unpickle
-            # using older version of dill. It is best to use the same version of
-            # dill on client and server, therefore list of allowed versions is
-            # very narrow. See: https://github.com/uqfoundation/dill/issues/341.
+            # version. Pickles created with a new version of dill may not
+            # unpickle using older version of dill. It is best to use the same
+            # version of dill on client and server, therefore list of allowed
+            # versions is very narrow. 
+            # See: https://github.com/uqfoundation/dill/issues/341.
             'dill>=0.3.1.1,<0.3.2',
           ],
           'docs': [