test: remove pandas version restrictions and fix precision issue (#17005)

This PR removes the pandas version restrictions from tests that were
skipped for Pandas >= 2.0 without a stated reason and enables testing by
addressing a precision issue in the tests.

It also adds a catch for ImportError when tqdm tries to import
matplotlib when matplotlib is not installed.

It also adds two small ignore pragmas to resolve mypy's struggles with
handling optional imports.

This PR partially fixes #16042 (for bigquery).

For completeness, we did an assessment of any remaining test skips and
they all appear to be legitimate skips that are applied when a
dependency is not installed.

1. PyArrow
2. tqdm (Progress Bar)
3. Pandas
4. OpenTelemetry
5. GeoPandas
6. Shapely

Author: chalmerlowe

packages/google-cloud-bigquery/google/cloud/bigquery/_pyarrow_helpers.py

@@ -26,7 +26,7 @@
 try:
     import pyarrow  # type: ignore
 except ImportError:
-    pyarrow = None
+    pyarrow = None  # type: ignore[assignment]
 
 try:
     import db_dtypes  # type: ignore

packages/google-cloud-bigquery/google/cloud/bigquery/_tqdm_helpers.py

@@ -70,7 +70,7 @@ def get_progress_bar(progress_bar_type, description, total, unit):
             )
         elif progress_bar_type == "tqdm_gui":
             return tqdm.tqdm_gui(desc=description, total=total, unit=unit)
-    except (KeyError, TypeError):  # pragma: NO COVER
+    except (KeyError, TypeError, ImportError):  # pragma: NO COVER
         # Protect ourselves from any tqdm errors. In case of
         # unexpected tqdm behavior, just fall back to showing
         # no progress bar.

packages/google-cloud-bigquery/google/cloud/bigquery/table.py

@@ -33,7 +33,7 @@
 try:
     import pyarrow  # type: ignore
 except ImportError:
-    pyarrow = None
+    pyarrow = None  # type: ignore[assignment]
 
 try:
     import db_dtypes  # type: ignore

packages/google-cloud-bigquery/tests/system/test_pandas.py

@@ -1053,7 +1053,6 @@ def test_list_rows_max_results_w_bqstorage(bigquery_client):
     assert len(dataframe.index) == 100
 
 
-@pytest.mark.skipif(PANDAS_INSTALLED_VERSION[0:2] not in ["0.", "1."], reason="")
 @pytest.mark.parametrize(
     ("max_results",),
     (

packages/google-cloud-bigquery/tests/unit/test__pandas_helpers.py

@@ -551,15 +551,15 @@ def test_bq_to_arrow_array_w_nullable_scalars(module_under_test, bq_type, rows):
     ],
 )
 @pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
-@pytest.mark.skipif(PANDAS_INSTALLED_VERSION[0:2] not in ["0.", "1."], reason="")
 @pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
 def test_bq_to_arrow_array_w_pandas_timestamp(module_under_test, bq_type, rows):
     rows = [pandas.Timestamp(row) for row in rows]
     series = pandas.Series(rows)
     bq_field = schema.SchemaField("field_name", bq_type)
     arrow_array = module_under_test.bq_to_arrow_array(series, bq_field)
     roundtrip = arrow_array.to_pandas()
-    assert series.equals(roundtrip)
+    series = series.astype(roundtrip.dtype)
+    pandas.testing.assert_series_equal(series, roundtrip)
 
 
 @pytest.mark.skipif(pandas is None, reason="Requires `pandas`")

packages/google-cloud-bigquery/tests/unit/test_magics.py

@@ -45,6 +45,18 @@
 pandas = pytest.importorskip("pandas")
 
 
+@pytest.fixture()
+def use_local_magics_context(monkeypatch):
+    if magics is not None:
+        local_context = magics.Context()
+        local_context._project = "unit-test-project"
+        mock_credentials = mock.create_autospec(
+            google.auth.credentials.Credentials, instance=True
+        )
+        local_context._credentials = mock_credentials
+        monkeypatch.setattr(magics, "context", local_context)
+
+
 @pytest.fixture(scope="session")
 def ipython():
     config = tools.default_config()
@@ -523,7 +535,7 @@ def test_bigquery_magic_default_connection_user_agent(monkeypatch):
 
 
 @pytest.mark.usefixtures("ipython_interactive")
-def test_bigquery_magic_with_legacy_sql(monkeypatch):
+def test_bigquery_magic_with_legacy_sql(monkeypatch, use_local_magics_context):
     ip = IPython.get_ipython()
     monkeypatch.setattr(bigquery, "bigquery_magics", None)
     bigquery.load_ipython_extension(ip)
@@ -543,7 +555,9 @@ def test_bigquery_magic_with_legacy_sql(monkeypatch):
 
 @pytest.mark.usefixtures("ipython_interactive")
 @pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
-def test_bigquery_magic_with_result_saved_to_variable(ipython_ns_cleanup, monkeypatch):
+def test_bigquery_magic_with_result_saved_to_variable(
+    ipython_ns_cleanup, monkeypatch, use_local_magics_context
+):
     ip = IPython.get_ipython()
     monkeypatch.setattr(bigquery, "bigquery_magics", None)
     bigquery.load_ipython_extension(ip)
@@ -577,7 +591,9 @@ def test_bigquery_magic_with_result_saved_to_variable(ipython_ns_cleanup, monkey
 
 
 @pytest.mark.usefixtures("ipython_interactive")
-def test_bigquery_magic_does_not_clear_display_in_verbose_mode(monkeypatch):
+def test_bigquery_magic_does_not_clear_display_in_verbose_mode(
+    monkeypatch, use_local_magics_context
+):
     ip = IPython.get_ipython()
     monkeypatch.setattr(bigquery, "bigquery_magics", None)
     bigquery.load_ipython_extension(ip)
@@ -599,7 +615,9 @@ def test_bigquery_magic_does_not_clear_display_in_verbose_mode(monkeypatch):
 
 
 @pytest.mark.usefixtures("ipython_interactive")
-def test_bigquery_magic_clears_display_in_non_verbose_mode(monkeypatch):
+def test_bigquery_magic_clears_display_in_non_verbose_mode(
+    monkeypatch, use_local_magics_context
+):
     ip = IPython.get_ipython()
     monkeypatch.setattr(bigquery, "bigquery_magics", None)
     bigquery.load_ipython_extension(ip)
@@ -624,7 +642,9 @@ def test_bigquery_magic_clears_display_in_non_verbose_mode(monkeypatch):
 @pytest.mark.skipif(
     bigquery_storage is None, reason="Requires `google-cloud-bigquery-storage`"
 )
-def test_bigquery_magic_with_bqstorage_from_argument(monkeypatch):
+def test_bigquery_magic_with_bqstorage_from_argument(
+    monkeypatch, use_local_magics_context
+):
     ip = IPython.get_ipython()
     monkeypatch.setattr(bigquery, "bigquery_magics", None)
     bigquery.load_ipython_extension(ip)
@@ -691,7 +711,9 @@ def warning_match(warning):
 @pytest.mark.skipif(
     bigquery_storage is None, reason="Requires `google-cloud-bigquery-storage`"
 )
-def test_bigquery_magic_with_rest_client_requested(monkeypatch):
+def test_bigquery_magic_with_rest_client_requested(
+    monkeypatch, use_local_magics_context
+):
     pandas = pytest.importorskip("pandas")
 
     ip = IPython.get_ipython()
@@ -1437,7 +1459,9 @@ def test_bigquery_magic_with_project(monkeypatch):
 
 
 @pytest.mark.usefixtures("ipython_interactive")
-def test_bigquery_magic_with_bigquery_api_endpoint(ipython_ns_cleanup, monkeypatch):
+def test_bigquery_magic_with_bigquery_api_endpoint(
+    ipython_ns_cleanup, monkeypatch, use_local_magics_context
+):
     ip = IPython.get_ipython()
     monkeypatch.setattr(bigquery, "bigquery_magics", None)
     bigquery.load_ipython_extension(ip)
@@ -1460,7 +1484,9 @@ def test_bigquery_magic_with_bigquery_api_endpoint(ipython_ns_cleanup, monkeypat
 
 
 @pytest.mark.usefixtures("ipython_interactive")
-def test_bigquery_magic_with_bigquery_api_endpoint_context_dict(monkeypatch):
+def test_bigquery_magic_with_bigquery_api_endpoint_context_dict(
+    monkeypatch, use_local_magics_context
+):
     ip = IPython.get_ipython()
     monkeypatch.setattr(bigquery, "bigquery_magics", None)
     bigquery.load_ipython_extension(ip)
@@ -1484,7 +1510,9 @@ def test_bigquery_magic_with_bigquery_api_endpoint_context_dict(monkeypatch):
 
 
 @pytest.mark.usefixtures("ipython_interactive")
-def test_bigquery_magic_with_bqstorage_api_endpoint(ipython_ns_cleanup, monkeypatch):
+def test_bigquery_magic_with_bqstorage_api_endpoint(
+    ipython_ns_cleanup, monkeypatch, use_local_magics_context
+):
     ip = IPython.get_ipython()
     monkeypatch.setattr(bigquery, "bigquery_magics", None)
     bigquery.load_ipython_extension(ip)
@@ -1507,7 +1535,9 @@ def test_bigquery_magic_with_bqstorage_api_endpoint(ipython_ns_cleanup, monkeypa
 
 
 @pytest.mark.usefixtures("ipython_interactive")
-def test_bigquery_magic_with_bqstorage_api_endpoint_context_dict(monkeypatch):
+def test_bigquery_magic_with_bqstorage_api_endpoint_context_dict(
+    monkeypatch, use_local_magics_context
+):
     ip = IPython.get_ipython()
     monkeypatch.setattr(bigquery, "bigquery_magics", None)
     bigquery.load_ipython_extension(ip)

packages/google-cloud-bigquery/tests/unit/test_table.py

@@ -4092,7 +4092,6 @@ def test_to_dataframe_no_tqdm(self):
 
     def test_to_dataframe_tqdm_error(self):
         pytest.importorskip("pandas")
-        pytest.importorskip("matplotlib")
         tqdm = pytest.importorskip("tqdm")
         mock.patch("tqdm.tqdm_gui", new=None)
         mock.patch("tqdm.notebook.tqdm", new=None)