Merge remote-tracking branch 'upstream/main' into tswast-pandas-gbq

tswast · tswast · commit d0ff584997e8 · 2026-04-09T14:23:02.000Z
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -12,4 +12,10 @@
 *                                             @googleapis/cloud-sdk-python-team @googleapis/cloud-sdk-librarian-team
 
 /packages/bigframes/                          @googleapis/cloud-sdk-python-team @googleapis/cloud-sdk-librarian-team @googleapis/bigquery-dataframe-team
+/packages/bigquery-magics/                    @googleapis/cloud-sdk-python-team @googleapis/cloud-sdk-librarian-team @googleapis/bigquery-dataframe-team
+/packages/django-google-spanner/              @googleapis/cloud-sdk-python-team @googleapis/cloud-sdk-librarian-team @googleapis/spanner-team
 /packages/google-auth/                        @googleapis/cloud-sdk-python-team @googleapis/cloud-sdk-librarian-team @googleapis/cloud-sdk-auth-team @googleapis/aion-team
+/packages/google-cloud-bigquery*/             @googleapis/cloud-sdk-python-team @googleapis/cloud-sdk-librarian-team @googleapis/bigquery-dataframe-team
+/packages/google-cloud-spanner/               @googleapis/cloud-sdk-python-team @googleapis/cloud-sdk-librarian-team @googleapis/spanner-team
+/packages/pandas-gbq/                         @googleapis/cloud-sdk-python-team @googleapis/cloud-sdk-librarian-team @googleapis/bigquery-dataframe-team
+/packages/sqlalchemy-bigquery/                @googleapis/cloud-sdk-python-team @googleapis/cloud-sdk-librarian-team @googleapis/bigquery-dataframe-team
diff --git a/packages/bigquery-magics/bigquery_magics/bigquery.py b/packages/bigquery-magics/bigquery_magics/bigquery.py
@@ -78,8 +78,10 @@
           the variable name (ex. ``$my_dict_var``). See ``In[6]`` and ``In[7]``
           in the Examples section below.
     * ``--engine <engine>`` (Optional[line argument]):
-          Set the execution engine, either 'pandas' (default) or 'bigframes'
-          (experimental).
+          [Deprecated] Set the execution engine, either 'pandas' (default) or
+          'bigframes'.
+          Please use ``%load_ext bigframes`` and the ``%%bqsql`` magic instead.
+          See: https://dataframes.bigquery.dev/notebooks/getting_started/magics.html
     * ``--pyformat`` (Optional[line argument]):
           Warning! Do not use with user-provided values.
           This doesn't escape values. Use --params instead for proper SQL escaping.
@@ -397,8 +399,10 @@ def _create_dataset_if_necessary(client, dataset_id):
     type=str,
     default=None,
     help=(
-        "Set the execution engine, either 'pandas' or 'bigframes'."
-        "Defaults to engine set in the query setting in console."
+        "[Deprecated] Set the execution engine, either 'pandas' or 'bigframes'. "
+        "Defaults to engine set in the query setting in console. "
+        "Please use %%load_ext bigframes and the %%%%bqsql magic instead. "
+        "See: https://dataframes.bigquery.dev/notebooks/getting_started/magics.html"
     ),
 )
 @magic_arguments.argument(
@@ -510,6 +514,13 @@ def _split_args_line(line: str) -> Tuple[str, str]:
 
 
 def _query_with_bigframes(query: str, params: List[Any], args: Any):
+    warnings.warn(
+        "The bigframes engine is deprecated. Please use %load_ext bigframes "
+        "and the %%bqsql magic instead. "
+        "See: https://dataframes.bigquery.dev/notebooks/getting_started/magics.html",
+        FutureWarning,
+        stacklevel=2,
+    )
     if args.dry_run:
         raise ValueError("Dry run is not supported by bigframes engine.")
 
diff --git a/packages/bigquery-magics/bigquery_magics/config.py b/packages/bigquery-magics/bigquery_magics/config.py
@@ -14,6 +14,7 @@
 
 from dataclasses import dataclass
 from typing import Optional
+import warnings
 
 import google.api_core.client_options as client_options
 import google.cloud.bigquery as bigquery
@@ -183,10 +184,15 @@ def project(self, value):
 
     @property
     def engine(self) -> str:
-        """Engine to run the query. Could either be "pandas" or "bigframes".
+        """[Deprecated] Engine to run the query. Could either be "pandas" or
+        "bigframes".
 
         If using "pandas", the query result will be stored in a Pandas dataframe.
-        If using "bigframes", the query result will be stored in a bigframes dataframe instead.
+        If using "bigframes", the query result will be stored in a bigframes
+        dataframe instead.
+
+        Please use ``%load_ext bigframes`` and the ``%%bqsql`` magic instead.
+        See: https://dataframes.bigquery.dev/notebooks/getting_started/magics.html
 
         Example:
             Manully setting the content engine:
@@ -200,6 +206,14 @@ def engine(self) -> str:
     def engine(self, value):
         if value != "pandas" and value != "bigframes":
             raise ValueError("engine must be either 'pandas' or 'bigframes'")
+        if value == "bigframes":
+            warnings.warn(
+                "The bigframes engine is deprecated. Please use %load_ext bigframes "
+                "and the %%bqsql magic instead. "
+                "See: https://dataframes.bigquery.dev/notebooks/getting_started/magics.html",
+                FutureWarning,
+                stacklevel=2,
+            )
         self._engine = value
 
 
diff --git a/packages/bigquery-magics/tests/unit/bigquery/test_deprecation.py b/packages/bigquery-magics/tests/unit/bigquery/test_deprecation.py
@@ -0,0 +1,79 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+from bigquery_magics import bigquery as magics
+import bigquery_magics.config
+
+
+@pytest.fixture(autouse=True)
+def mock_bq_client_and_credentials(mock_credentials):
+    from unittest import mock
+
+    with mock.patch("google.cloud.bigquery.Client", autospec=True):
+        with mock.patch("bigquery_magics.core.create_bq_client", autospec=True):
+            yield
+
+
+def test_config_engine_setter_warning():
+    context = bigquery_magics.config.Context()
+    with pytest.warns(FutureWarning, match="The bigframes engine is deprecated"):
+        context.engine = "bigframes"
+
+
+def test_query_with_bigframes_warning(mock_ipython):
+    # Mocking bigframes.pandas since it might not be installed
+    from unittest import mock
+
+    with mock.patch("bigquery_magics.bigquery.bpd") as mock_bpd:
+        mock_bpd.read_gbq_query.return_value = mock.MagicMock()
+
+        args = mock.MagicMock()
+        args.engine = "bigframes"
+        args.dry_run = False
+        args.max_results = None
+        args.destination_var = None
+        args.destination_table = None
+
+        with pytest.warns(FutureWarning, match="The bigframes engine is deprecated"):
+            magics._query_with_bigframes("SELECT 1", [], args)
+
+
+def test_cell_magic_engine_bigframes_warning(mock_ipython):
+    from unittest import mock
+
+    from IPython.testing.globalipapp import get_ipython
+
+    ip = get_ipython()
+    if ip is None:
+        from IPython.testing.globalipapp import start_ipython
+
+        ip = start_ipython()
+
+    ip.extension_manager.load_extension("bigquery_magics")
+
+    # Mock the actual execution to avoid needing real credentials/data
+    with mock.patch("bigquery_magics.bigquery.bpd") as mock_bpd:
+        mock_bpd.read_gbq_query.return_value = mock.MagicMock()
+        with pytest.warns(FutureWarning, match="The bigframes engine is deprecated"):
+            ip.run_cell_magic("bigquery", "--engine bigframes", "SELECT 1")
+
+
+@pytest.fixture
+def mock_ipython():
+    from unittest import mock
+
+    with mock.patch("bigquery_magics.bigquery.get_ipython") as mock_get_ipython:
+        yield mock_get_ipython
diff --git a/packages/google-api-core/google/api_core/_python_version_support.py b/packages/google-api-core/google/api_core/_python_version_support.py
@@ -16,6 +16,7 @@
 
 import datetime
 import enum
+import functools
 import logging
 import warnings
 import sys
@@ -152,15 +153,16 @@ def _get_pypi_package_name(module_name):  # pragma: NO COVER
 else:
     from importlib import metadata
 
+    @functools.cache
+    def _cached_packages_distributions():
+        return metadata.packages_distributions()
+
     def _get_pypi_package_name(module_name):
         """Determine the PyPI package name for a given module name."""
         try:
-            # Get the mapping of modules to distributions
-            module_to_distributions = metadata.packages_distributions()
+            module_to_distributions = _cached_packages_distributions()
 
-            # Check if the module is found in the mapping
             if module_name in module_to_distributions:  # pragma: NO COVER
-                # The value is a list of distribution names, take the first one
                 return module_to_distributions[module_name][0]
         except Exception as e:  # pragma: NO COVER
             _LOGGER.info(
@@ -195,7 +197,6 @@ def check_python_version(
         The support status of the current Python version.
     """
     today = today or datetime.date.today()
-    package_label, _ = _get_distribution_and_import_packages(package)
 
     python_version = sys.version_info
     version_tuple = (python_version.major, python_version.minor)
@@ -221,7 +222,14 @@ def min_python(date: datetime.date) -> str:
                 return f"{version[0]}.{version[1]}"
         return "at a currently supported version [https://devguide.python.org/versions]"
 
+    # Resolve the pretty package label lazily so we avoid any work on
+    # the happy path (supported Python version, no warning needed).
+    def get_package_label():
+        label, _ = _get_distribution_and_import_packages(package)
+        return label
+
     if gapic_end < today:
+        package_label = get_package_label()
         message = _flatten_message(
             f"""
             You are using a non-supported Python version ({py_version_str}).
@@ -236,6 +244,7 @@ def min_python(date: datetime.date) -> str:
 
     eol_date = version_info.python_eol + EOL_GRACE_PERIOD
     if eol_date <= today <= gapic_end:
+        package_label = get_package_label()
         message = _flatten_message(
             f"""
             You are using a Python version ({py_version_str})
@@ -250,6 +259,7 @@ def min_python(date: datetime.date) -> str:
         return PythonVersionStatus.PYTHON_VERSION_EOL
 
     if gapic_deprecation <= today <= gapic_end:
+        package_label = get_package_label()
         message = _flatten_message(
             f"""
             You are using a Python version ({py_version_str}) which Google will
diff --git a/packages/google-cloud-spanner/google/cloud/spanner_v1/_helpers.py b/packages/google-cloud-spanner/google/cloud/spanner_v1/_helpers.py
@@ -28,7 +28,7 @@
 from google.api_core.exceptions import Aborted
 from google.cloud._helpers import _date_from_iso8601_date
 from google.protobuf.internal.enum_type_wrapper import EnumTypeWrapper
-from google.protobuf.message import Message
+from google.protobuf.message import DecodeError, Message
 from google.protobuf.struct_pb2 import ListValue, Value
 from google.rpc.error_details_pb2 import RetryInfo
 
@@ -76,7 +76,7 @@
 
 GOOGLE_CLOUD_REGION_GLOBAL = "global"
 
-log = logging.getLogger(__name__)
+_LOGGER = logging.getLogger(__name__)
 
 _cloud_region: str = None
 
@@ -122,7 +122,7 @@ def _get_cloud_region() -> str:
         else:
             _cloud_region = GOOGLE_CLOUD_REGION_GLOBAL
     except Exception as e:
-        log.warning(
+        _LOGGER.warning(
             "Failed to detect GCP resource location for Spanner metrics, defaulting to 'global'. Error: %s",
             e,
         )
@@ -603,8 +603,14 @@ def _parse_proto(value_pb, column_info, field_name):
         default_proto_message = column_info.get(field_name)
         if isinstance(default_proto_message, Message):
             proto_message = type(default_proto_message)()
-            proto_message.ParseFromString(bytes_value)
-            return proto_message
+            try:
+                proto_message.ParseFromString(bytes_value)
+                return proto_message
+            except (DecodeError, RecursionError):
+                _LOGGER.warning(
+                    "Field could not be parsed as Proto due to excessive nesting/corruption. Returning raw bytes."
+                )
+                return bytes_value
     return bytes_value
 
 
diff --git a/packages/google-cloud-spanner/tests/unit/test__helpers.py b/packages/google-cloud-spanner/tests/unit/test__helpers.py
@@ -771,6 +771,59 @@ def test_w_proto_message(self):
             self._callFUT(value_pb, field_type, field_name, column_info), VALUE
         )
 
+    def test_w_proto_message_decode_error(self):
+        import base64
+        from unittest import mock
+
+        from google.protobuf.message import DecodeError
+        from google.protobuf.struct_pb2 import Value
+
+        from google.cloud.spanner_v1 import Type, TypeCode
+
+        from .testdata import singer_pb2
+
+        VALUE = singer_pb2.SingerInfo(singer_id=1, nationality="Canadian")
+        field_type = Type(code=TypeCode.PROTO)
+        field_name = "proto_message_column"
+        raw_bytes = VALUE.SerializeToString()
+        value_pb = Value(string_value=base64.b64encode(raw_bytes).decode("utf-8"))
+        column_info = {"proto_message_column": singer_pb2.SingerInfo()}
+
+        # Mock ParseFromString to raise DecodeError
+        with mock.patch.object(
+            singer_pb2.SingerInfo,
+            "ParseFromString",
+            side_effect=DecodeError("Mock Decode Error"),
+        ):
+            result = self._callFUT(value_pb, field_type, field_name, column_info)
+            # Should return raw bytes
+            self.assertEqual(result, raw_bytes)
+
+    def test_w_proto_message_recursion_error(self):
+        import base64
+        from unittest import mock
+
+        from google.protobuf.struct_pb2 import Value
+
+        from google.cloud.spanner_v1 import Type, TypeCode
+
+        from .testdata import singer_pb2
+
+        VALUE = singer_pb2.SingerInfo(singer_id=1, nationality="Canadian")
+        field_type = Type(code=TypeCode.PROTO)
+        field_name = "proto_message_column"
+        raw_bytes = VALUE.SerializeToString()
+        value_pb = Value(string_value=base64.b64encode(raw_bytes).decode("utf-8"))
+        column_info = {"proto_message_column": singer_pb2.SingerInfo()}
+
+        with mock.patch.object(
+            singer_pb2.SingerInfo,
+            "ParseFromString",
+            side_effect=RecursionError("Mock Recursion Error"),
+        ):
+            result = self._callFUT(value_pb, field_type, field_name, column_info)
+            self.assertEqual(result, raw_bytes)
+
     def test_w_proto_enum(self):
         from google.protobuf.struct_pb2 import Value
 
diff --git a/packages/google-cloud-storage/cloudbuild/run_zonal_tests.sh b/packages/google-cloud-storage/cloudbuild/run_zonal_tests.sh
@@ -4,10 +4,12 @@ echo '--- Installing git and cloning repository on VM ---'
 sudo apt-get update && sudo apt-get install -y git python3-pip python3-venv
 
 # Clone the repository and checkout the specific commit from the build trigger.
-git clone https://github.com/googleapis/python-storage.git
-cd python-storage
+git clone --no-checkout --depth 1 --sparse --filter=blob:none https://github.com/googleapis/google-cloud-python.git
+cd google-cloud-python
+git sparse-checkout set packages/google-cloud-storage
 git fetch origin "refs/pull/${_PR_NUMBER}/head"
 git checkout ${COMMIT_SHA}
+cd packages/google-cloud-storage
 
 
 echo '--- Installing Python and dependencies on VM ---'
@@ -27,4 +29,3 @@ export GCE_METADATA_MTLS_MODE=None
 CURRENT_ULIMIT=$(ulimit -n)
 echo '--- Running Zonal tests on VM with ulimit set to ---' $CURRENT_ULIMIT
 pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' tests/system/test_zonal.py
-pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' samples/snippets/zonal_buckets/zonal_snippets_test.py
diff --git a/packages/google-cloud-storage/cloudbuild/zb-system-tests-cloudbuild.yaml b/packages/google-cloud-storage/cloudbuild/zb-system-tests-cloudbuild.yaml
