Merge branch 'main' into validate_join_type

Author: TrevorBergeron

CHANGELOG.md

@@ -4,6 +4,20 @@
 
 [1]: https://pypi.org/project/bigframes/#history
 
+## [2.21.0](https://github.com/googleapis/python-bigquery-dataframes/compare/v2.20.0...v2.21.0) (2025-09-17)
+
+
+### Features
+
+* Add bigframes.bigquery.to_json ([#2078](https://github.com/googleapis/python-bigquery-dataframes/issues/2078)) ([0fc795a](https://github.com/googleapis/python-bigquery-dataframes/commit/0fc795a9fb56f469b62603462c3f0f56f52bfe04))
+* Support average='binary' in precision_score() ([#2080](https://github.com/googleapis/python-bigquery-dataframes/issues/2080)) ([920f381](https://github.com/googleapis/python-bigquery-dataframes/commit/920f381aec7e0a0b986886cdbc333e86335c6d7d))
+* Support pandas series in ai.generate_bool ([#2086](https://github.com/googleapis/python-bigquery-dataframes/issues/2086)) ([a3de53f](https://github.com/googleapis/python-bigquery-dataframes/commit/a3de53f68b2a24f4ed85a474dfaff9b59570a2f1))
+
+
+### Bug Fixes
+
+* Allow bigframes.options.bigquery.credentials to be `None` ([#2092](https://github.com/googleapis/python-bigquery-dataframes/issues/2092)) ([78f4001](https://github.com/googleapis/python-bigquery-dataframes/commit/78f4001e8fcfc77fc82f3893d58e0d04c0f6d3db))
+
 ## [2.20.0](https://github.com/googleapis/python-bigquery-dataframes/compare/v2.19.0...v2.20.0) (2025-09-16)
 
 

bigframes/bigquery/_operations/ai.py

@@ -19,16 +19,25 @@
 from __future__ import annotations
 
 import json
-from typing import Any, List, Literal, Mapping, Tuple
+from typing import Any, List, Literal, Mapping, Tuple, Union
 
-from bigframes import clients, dtypes, series
-from bigframes.core import log_adapter
+import pandas as pd
+
+from bigframes import clients, dtypes, series, session
+from bigframes.core import convert, log_adapter
 from bigframes.operations import ai_ops
 
+PROMPT_TYPE = Union[
+    series.Series,
+    pd.Series,
+    List[Union[str, series.Series, pd.Series]],
+    Tuple[Union[str, series.Series, pd.Series], ...],
+]
+
 
 @log_adapter.method_logger(custom_base_name="bigquery_ai")
 def generate_bool(
-    prompt: series.Series | List[str | series.Series] | Tuple[str | series.Series, ...],
+    prompt: PROMPT_TYPE,
     *,
     connection_id: str | None = None,
     endpoint: str | None = None,
@@ -51,7 +60,7 @@ def generate_bool(
         0    {'result': True, 'full_response': '{"candidate...
         1    {'result': True, 'full_response': '{"candidate...
         2    {'result': False, 'full_response': '{"candidat...
-        dtype: struct<result: bool, full_response: string, status: string>[pyarrow]
+        dtype: struct<result: bool, full_response: extension<dbjson<JSONArrowType>>, status: string>[pyarrow]
 
         >>> bbq.ai.generate_bool((df["col_1"], " is a ", df["col_2"])).struct.field("result")
         0     True
@@ -60,8 +69,9 @@ def generate_bool(
         Name: result, dtype: boolean
 
     Args:
-        prompt (series.Series | List[str|series.Series] | Tuple[str|series.Series, ...]):
-            A mixture of Series and string literals that specifies the prompt to send to the model.
+        prompt (Series | List[str|Series] | Tuple[str|Series, ...]):
+            A mixture of Series and string literals that specifies the prompt to send to the model. The Series can be BigFrames Series
+            or pandas Series.
         connection_id (str, optional):
             Specifies the connection to use to communicate with the model. For example, `myproject.us.myconnection`.
             If not provided, the connection from the current session will be used.
@@ -84,7 +94,7 @@ def generate_bool(
     Returns:
         bigframes.series.Series: A new struct Series with the result data. The struct contains these fields:
         * "result": a BOOL value containing the model's response to the prompt. The result is None if the request fails or is filtered by responsible AI.
-        * "full_response": a STRING value containing the JSON response from the projects.locations.endpoints.generateContent call to the model.
+        * "full_response": a JSON value containing the response from the projects.locations.endpoints.generateContent call to the model.
         The generated text is in the text element.
         * "status": a STRING value that contains the API response status for the corresponding row. This value is empty if the operation was successful.
     """
@@ -104,7 +114,7 @@ def generate_bool(
 
 
 def _separate_context_and_series(
-    prompt: series.Series | List[str | series.Series] | Tuple[str | series.Series, ...],
+    prompt: PROMPT_TYPE,
 ) -> Tuple[List[str | None], List[series.Series]]:
     """
     Returns the two values. The first value is the prompt with all series replaced by None. The second value is all the series
@@ -123,18 +133,19 @@ def _separate_context_and_series(
         return [None], [prompt]
 
     prompt_context: List[str | None] = []
-    series_list: List[series.Series] = []
+    series_list: List[series.Series | pd.Series] = []
 
+    session = None
     for item in prompt:
         if isinstance(item, str):
             prompt_context.append(item)
 
-        elif isinstance(item, series.Series):
+        elif isinstance(item, (series.Series, pd.Series)):
             prompt_context.append(None)
 
-            if item.dtype == dtypes.OBJ_REF_DTYPE:
-                # Multi-model support
-                item = item.blob.read_url()
+            if isinstance(item, series.Series) and session is None:
+                # Use the first available BF session if there's any.
+                session = item._session
             series_list.append(item)
 
         else:
@@ -143,7 +154,20 @@ def _separate_context_and_series(
     if not series_list:
         raise ValueError("Please provide at least one Series in the prompt")
 
-    return prompt_context, series_list
+    converted_list = [_convert_series(s, session) for s in series_list]
+
+    return prompt_context, converted_list
+
+
+def _convert_series(
+    s: series.Series | pd.Series, session: session.Session | None
+) -> series.Series:
+    result = convert.to_bf_series(s, default_index=None, session=session)
+
+    if result.dtype == dtypes.OBJ_REF_DTYPE:
+        # Support multimodel
+        return result.blob.read_url()
+    return result
 
 
 def _resolve_connection_id(series: series.Series, connection_id: str | None):

bigframes/ml/compose.py

@@ -29,6 +29,7 @@
 
 from bigframes.core import log_adapter
 import bigframes.core.compile.googlesql as sql_utils
+import bigframes.core.utils as core_utils
 from bigframes.ml import base, core, globals, impute, preprocessing, utils
 import bigframes.pandas as bpd
 
@@ -103,13 +104,12 @@ def __init__(self, sql: str, target_column: str = "transformed_{0}"):
         # TODO: More robust unescaping
         self._target_column = target_column.replace("`", "")
 
-    PLAIN_COLNAME_RX = re.compile("^[a-z][a-z0-9_]*$", re.IGNORECASE)
-
     def _compile_to_sql(
         self, X: bpd.DataFrame, columns: Optional[Iterable[str]] = None
     ) -> List[str]:
         if columns is None:
             columns = X.columns
+        columns, _ = core_utils.get_standardized_ids(columns)
         result = []
         for column in columns:
             current_sql = self._sql.format(sql_utils.identifier(column))

bigframes/ml/impute.py

@@ -23,6 +23,7 @@
 import bigframes_vendored.sklearn.impute._base
 
 from bigframes.core import log_adapter
+import bigframes.core.utils as core_utils
 from bigframes.ml import base, core, globals, utils
 import bigframes.pandas as bpd
 
@@ -62,6 +63,7 @@ def _compile_to_sql(
         Returns: a list of tuples sql_expr."""
         if columns is None:
             columns = X.columns
+        columns, _ = core_utils.get_standardized_ids(columns)
         return [
             self._base_sql_generator.ml_imputer(
                 column, self.strategy, f"imputer_{column}"

bigframes/ml/preprocessing.py

@@ -27,6 +27,7 @@
 import bigframes_vendored.sklearn.preprocessing._polynomial
 
 from bigframes.core import log_adapter
+import bigframes.core.utils as core_utils
 from bigframes.ml import base, core, globals, utils
 import bigframes.pandas as bpd
 
@@ -59,6 +60,7 @@ def _compile_to_sql(
         Returns: a list of tuples sql_expr."""
         if columns is None:
             columns = X.columns
+        columns, _ = core_utils.get_standardized_ids(columns)
         return [
             self._base_sql_generator.ml_standard_scaler(
                 column, f"standard_scaled_{column}"
@@ -136,6 +138,7 @@ def _compile_to_sql(
         Returns: a list of tuples sql_expr."""
         if columns is None:
             columns = X.columns
+        columns, _ = core_utils.get_standardized_ids(columns)
         return [
             self._base_sql_generator.ml_max_abs_scaler(
                 column, f"max_abs_scaled_{column}"
@@ -214,6 +217,7 @@ def _compile_to_sql(
         Returns: a list of tuples sql_expr."""
         if columns is None:
             columns = X.columns
+        columns, _ = core_utils.get_standardized_ids(columns)
         return [
             self._base_sql_generator.ml_min_max_scaler(
                 column, f"min_max_scaled_{column}"
@@ -304,6 +308,7 @@ def _compile_to_sql(
         Returns: a list of tuples sql_expr."""
         if columns is None:
             columns = X.columns
+        columns, _ = core_utils.get_standardized_ids(columns)
         array_split_points = {}
         if self.strategy == "uniform":
             for column in columns:
@@ -433,6 +438,7 @@ def _compile_to_sql(
         Returns: a list of tuples sql_expr."""
         if columns is None:
             columns = X.columns
+        columns, _ = core_utils.get_standardized_ids(columns)
         drop = self.drop if self.drop is not None else "none"
         # minus one here since BQML's implementation always includes index 0, and top_k is on top of that.
         top_k = (
@@ -547,6 +553,7 @@ def _compile_to_sql(
         Returns: a list of tuples sql_expr."""
         if columns is None:
             columns = X.columns
+        columns, _ = core_utils.get_standardized_ids(columns)
 
         # minus one here since BQML's inplimentation always includes index 0, and top_k is on top of that.
         top_k = (
@@ -644,6 +651,7 @@ def _compile_to_sql(
         Returns: a list of tuples sql_expr."""
         if columns is None:
             columns = X.columns
+        columns, _ = core_utils.get_standardized_ids(columns)
         output_name = "poly_feat"
         return [
             self._base_sql_generator.ml_polynomial_expand(

bigframes/operations/ai_ops.py

@@ -40,7 +40,7 @@ def output_type(self, *input_types: dtypes.ExpressionType) -> dtypes.ExpressionT
             pa.struct(
                 (
                     pa.field("result", pa.bool_()),
-                    pa.field("full_response", pa.string()),
+                    pa.field("full_response", dtypes.JSON_ARROW_TYPE),
                     pa.field("status", pa.string()),
                 )
             )

bigframes/version.py

@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "2.20.0"
+__version__ = "2.21.0"
 
 # {x-release-please-start-date}
-__release_date__ = "2025-09-16"
+__release_date__ = "2025-09-17"
 # {x-release-please-end}

tests/system/small/bigquery/test_ai.py

@@ -18,7 +18,7 @@
 import pyarrow as pa
 import pytest
 
-from bigframes import series
+from bigframes import dtypes, series
 import bigframes.bigquery as bbq
 import bigframes.pandas as bpd
 
@@ -35,7 +35,26 @@ def test_ai_generate_bool(session):
         pa.struct(
             (
                 pa.field("result", pa.bool_()),
-                pa.field("full_response", pa.string()),
+                pa.field("full_response", dtypes.JSON_ARROW_TYPE),
+                pa.field("status", pa.string()),
+            )
+        )
+    )
+
+
+def test_ai_generate_bool_with_pandas(session):
+    s1 = pd.Series(["apple", "bear"])
+    s2 = bpd.Series(["fruit", "tree"], session=session)
+    prompt = (s1, " is a ", s2)
+
+    result = bbq.ai.generate_bool(prompt, endpoint="gemini-2.5-flash")
+
+    assert _contains_no_nulls(result)
+    assert result.dtype == pd.ArrowDtype(
+        pa.struct(
+            (
+                pa.field("result", pa.bool_()),
+                pa.field("full_response", dtypes.JSON_ARROW_TYPE),
                 pa.field("status", pa.string()),
             )
         )
@@ -62,7 +81,7 @@ def test_ai_generate_bool_with_model_params(session):
         pa.struct(
             (
                 pa.field("result", pa.bool_()),
-                pa.field("full_response", pa.string()),
+                pa.field("full_response", dtypes.JSON_ARROW_TYPE),
                 pa.field("status", pa.string()),
             )
         )
@@ -81,7 +100,7 @@ def test_ai_generate_bool_multi_model(session):
         pa.struct(
             (
                 pa.field("result", pa.bool_()),
-                pa.field("full_response", pa.string()),
+                pa.field("full_response", dtypes.JSON_ARROW_TYPE),
                 pa.field("status", pa.string()),
             )
         )

tests/system/small/ml/test_preprocessing.py

@@ -19,6 +19,7 @@
 
 import bigframes.features
 from bigframes.ml import preprocessing
+import bigframes.pandas as bpd
 from bigframes.testing import utils
 
 ONE_HOT_ENCODED_DTYPE = (
@@ -62,7 +63,7 @@ def test_standard_scaler_normalizes(penguins_df_default_index, new_penguins_df):
     pd.testing.assert_frame_equal(result, expected, rtol=0.1)
 
 
-def test_standard_scaler_normalizeds_fit_transform(new_penguins_df):
+def test_standard_scaler_normalizes_fit_transform(new_penguins_df):
     # TODO(http://b/292431644): add a second test that compares output to sklearn.preprocessing.StandardScaler, when BQML's change is in prod.
     scaler = preprocessing.StandardScaler()
     result = scaler.fit_transform(
@@ -114,6 +115,37 @@ def test_standard_scaler_series_normalizes(penguins_df_default_index, new_pengui
     pd.testing.assert_frame_equal(result, expected, rtol=0.1)
 
 
+def test_standard_scaler_normalizes_non_standard_column_names(
+    new_penguins_df: bpd.DataFrame,
+):
+    new_penguins_df = new_penguins_df.rename(
+        columns={
+            "culmen_length_mm": "culmen?metric",
+            "culmen_depth_mm": "culmen/metric",
+        }
+    )
+    scaler = preprocessing.StandardScaler()
+    result = scaler.fit_transform(
+        new_penguins_df[["culmen?metric", "culmen/metric", "flipper_length_mm"]]
+    ).to_pandas()
+
+    # If standard-scaled correctly, mean should be 0.0
+    for column in result.columns:
+        assert math.isclose(result[column].mean(), 0.0, abs_tol=1e-3)
+
+    expected = pd.DataFrame(
+        {
+            "standard_scaled_culmen_metric": [1.313249, -0.20198, -1.111118],
+            "standard_scaled_culmen_metric_1": [1.17072, -1.272416, 0.101848],
+            "standard_scaled_flipper_length_mm": [1.251089, -1.196588, -0.054338],
+        },
+        dtype="Float64",
+        index=pd.Index([1633, 1672, 1690], name="tag_number", dtype="Int64"),
+    )
+
+    pd.testing.assert_frame_equal(result, expected, rtol=0.1)
+
+
 def test_standard_scaler_save_load(new_penguins_df, dataset_id):
     transformer = preprocessing.StandardScaler()
     transformer.fit(

third_party/bigframes_vendored/version.py

@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "2.20.0"
+__version__ = "2.21.0"
 
 # {x-release-please-start-date}
-__release_date__ = "2025-09-16"
+__release_date__ = "2025-09-17"
 # {x-release-please-end}