Merge remote-tracking branch 'github/main' into cte_extract2

Author: TrevorBergeron

.librarian/state.yaml

@@ -1,7 +1,7 @@
-image: us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/python-librarian-generator@sha256:1a2a85ab507aea26d787c06cc7979decb117164c81dd78a745982dfda80d4f68
+image: us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/python-librarian-generator@sha256:160860d189ff1c2f7515638478823712fa5b243e27ccc33a2728669fa1e2ed0c
 libraries:
   - id: bigframes
-    version: 2.36.0
+    version: 2.37.0
     last_generated_commit: ""
     apis: []
     source_roots:

CHANGELOG.md

@@ -4,6 +4,36 @@
 
 [1]: https://pypi.org/project/bigframes/#history
 
+## [2.37.0](https://github.com/googleapis/python-bigquery-dataframes/compare/v2.36.0...v2.37.0) (2026-03-03)
+
+
+### Documentation
+
+* Fix recall_score doc example (#2477) ([a6f499c1e225a962b53621158f9d4a19ca220ccd](https://github.com/googleapis/python-bigquery-dataframes/commit/a6f499c1e225a962b53621158f9d4a19ca220ccd))
+* add code sample and docstring for bpd.options.experiments.sql_compiler (#2474) ([867951bcabcff12e2fce88143b45d929d3237088](https://github.com/googleapis/python-bigquery-dataframes/commit/867951bcabcff12e2fce88143b45d929d3237088))
+* use direct API for image (#2465) ([8a1a82f7a0fd224f2b075c68ab116d1f580d1d82](https://github.com/googleapis/python-bigquery-dataframes/commit/8a1a82f7a0fd224f2b075c68ab116d1f580d1d82))
+* add bigframes default connection warning (#2471) ([f1bbba23667f01d3b8e7c51b18fe64641a4b135f](https://github.com/googleapis/python-bigquery-dataframes/commit/f1bbba23667f01d3b8e7c51b18fe64641a4b135f))
+* Move readme content to new User Guide section (#2464) ([61a948451baeb1caa323e721ad88b31c7cd0b3cb](https://github.com/googleapis/python-bigquery-dataframes/commit/61a948451baeb1caa323e721ad88b31c7cd0b3cb))
+* Skip inherited methods, use autosummary only for big classes (#2470) ([a9512498ef39b9d5260cad2ca0513c701a6d3592](https://github.com/googleapis/python-bigquery-dataframes/commit/a9512498ef39b9d5260cad2ca0513c701a6d3592))
+* Add code examples to configuration docstrings (#2352) ([3c21993e6fca474c32f3c2371c41ef2be146267e](https://github.com/googleapis/python-bigquery-dataframes/commit/3c21993e6fca474c32f3c2371c41ef2be146267e))
+
+
+### Features
+
+* Add cloud_function_cpus option to remote_function (#2475) ([4caf74ccaeb9608d91da864bb80eddf1148a1502](https://github.com/googleapis/python-bigquery-dataframes/commit/4caf74ccaeb9608d91da864bb80eddf1148a1502))
+* Support pd.col simple aggregates (#2480) ([cb00daabce49f067be8e16627166dda00d5d8134](https://github.com/googleapis/python-bigquery-dataframes/commit/cb00daabce49f067be8e16627166dda00d5d8134))
+* add display.render_mode to control DataFrame/Series visualization (#2413) ([7813eaa6fa2ae42943b90583e600c95beaf5d75e](https://github.com/googleapis/python-bigquery-dataframes/commit/7813eaa6fa2ae42943b90583e600c95beaf5d75e))
+* add support for Python 3.14 (#2232) ([c25a6d0151380dde74368a35e13deb7a930b494f](https://github.com/googleapis/python-bigquery-dataframes/commit/c25a6d0151380dde74368a35e13deb7a930b494f))
+* Support pd.col expressions with .loc and getitem (#2473) ([ae5c8b322765aef51eed016bfacaff5a7a917a7b](https://github.com/googleapis/python-bigquery-dataframes/commit/ae5c8b322765aef51eed016bfacaff5a7a917a7b))
+* add dt.tz_localize() (#2469) ([f70f93a1227add1627d522d7e55a37f42fc3549e](https://github.com/googleapis/python-bigquery-dataframes/commit/f70f93a1227add1627d522d7e55a37f42fc3549e))
+* Update bigquery.ai.generate_table output_schema to allow Mapping type (#2463) ([f7fd1895e64a133fe63eddeb90f57a42a35c29b2](https://github.com/googleapis/python-bigquery-dataframes/commit/f7fd1895e64a133fe63eddeb90f57a42a35c29b2))
+
+
+### Bug Fixes
+
+* upload local data through write API if nested JSONs detected (#2478) ([01dc5a34e09171351575d5cbdc9f301e505e1567](https://github.com/googleapis/python-bigquery-dataframes/commit/01dc5a34e09171351575d5cbdc9f301e505e1567))
+* allow IsInOp with same dtypes regardless nullable (#2466) ([1d81b414acbc964502ca624eae72cdb8c14e1576](https://github.com/googleapis/python-bigquery-dataframes/commit/1d81b414acbc964502ca624eae72cdb8c14e1576))
+
 ## [2.36.0](https://github.com/googleapis/python-bigquery-dataframes/compare/v2.35.0...v2.36.0) (2026-02-17)
 
 

GEMINI.md

@@ -480,6 +480,39 @@ def generate_text(
         return session.read_gbq_query(sql)
 
 
+@log_adapter.method_logger(custom_base_name="bigquery_ml")
+def get_insights(
+    model: Union[bigframes.ml.base.BaseEstimator, str, pd.Series],
+) -> dataframe.DataFrame:
+    """
+    Gets insights from a BigQuery ML model.
+
+    See the `BigQuery ML GET_INSIGHTS function syntax
+    <https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-get-insights>`_
+    for additional reference.
+
+    Args:
+        model (bigframes.ml.base.BaseEstimator, str, or pd.Series):
+            The model to get insights from.
+
+    Returns:
+        bigframes.pandas.DataFrame:
+            The insights.
+    """
+    import bigframes.pandas as bpd
+
+    model_name, session = utils.get_model_name_and_session(model)
+
+    sql = bigframes.core.sql.ml.get_insights(
+        model_name=model_name,
+    )
+
+    if session is None:
+        return bpd.read_gbq_query(sql)
+    else:
+        return session.read_gbq_query(sql)
+
+
 @log_adapter.method_logger(custom_base_name="bigquery_ml")
 def generate_embedding(
     model: Union[bigframes.ml.base.BaseEstimator, str, pd.Series],

bigframes/bigquery/_operations/ml.py

@@ -20,7 +20,7 @@
 
 import google.cloud.bigquery
 
-import bigframes.core.compile.sqlglot.sqlglot_ir as sqlglot_ir
+from bigframes.core.compile.sqlglot import sql
 import bigframes.dtypes
 import bigframes.operations
 import bigframes.series
@@ -68,10 +68,7 @@ def sql_scalar(
     # Another benefit of this is that if there is a syntax error in the SQL
     # template, then this will fail with an error earlier in the process,
     # aiding users in debugging.
-    literals_sql = [
-        sqlglot_ir._literal(None, column.dtype).sql(dialect="bigquery")
-        for column in columns
-    ]
+    literals_sql = [sql.to_sql(sql.literal(None, column.dtype)) for column in columns]
     select_sql = sql_template.format(*literals_sql)
     dry_run_sql = f"SELECT {select_sql}"
 

bigframes/bigquery/_operations/sql.py

@@ -25,6 +25,7 @@
     explain_predict,
     generate_embedding,
     generate_text,
+    get_insights,
     global_explain,
     predict,
     transform,
@@ -39,4 +40,5 @@
     "transform",
     "generate_text",
     "generate_embedding",
+    "get_insights",
 ]

bigframes/bigquery/ml.py

@@ -14,12 +14,18 @@
 from __future__ import annotations
 
 import dataclasses
-from typing import Any, Hashable
+from typing import Any, Hashable, Literal, TYPE_CHECKING
 
 import bigframes_vendored.pandas.core.col as pd_col
 
+from bigframes.core import agg_expressions, window_spec
 import bigframes.core.expression as bf_expression
 import bigframes.operations as bf_ops
+import bigframes.operations.aggregations as agg_ops
+
+if TYPE_CHECKING:
+    import bigframes.operations.datetimes as datetimes
+    import bigframes.operations.strings as strings
 
 
 # Not to be confused with the Expression class in `bigframes.core.expressions`
@@ -30,10 +36,26 @@ class Expression:
 
     _value: bf_expression.Expression
 
-    def _apply_unary(self, op: bf_ops.UnaryOp) -> Expression:
+    def _apply_unary_op(self, op: bf_ops.UnaryOp) -> Expression:
         return Expression(op.as_expr(self._value))
 
-    def _apply_binary(self, other: Any, op: bf_ops.BinaryOp, reverse: bool = False):
+    def _apply_unary_agg(self, op: agg_ops.UnaryAggregateOp) -> Expression:
+        # We probably shouldn't need to windowize here, but block apis expect pre-windowized expressions
+        # Later on, we will probably have col expressions in windowed context, so will need to defer windowization
+        # instead of automatically applying the default unbound window
+        agg_expr = op.as_expr(self._value)
+        return Expression(
+            agg_expressions.WindowExpression(agg_expr, window_spec.unbound())
+        )
+
+    # alignment is purely for series compatibility, and is ignored here
+    def _apply_binary_op(
+        self,
+        other: Any,
+        op: bf_ops.BinaryOp,
+        alignment: Literal["outer", "left"] = "outer",
+        reverse: bool = False,
+    ):
         if isinstance(other, Expression):
             other_value = other._value
         else:
@@ -44,79 +66,109 @@ def _apply_binary(self, other: Any, op: bf_ops.BinaryOp, reverse: bool = False):
             return Expression(op.as_expr(self._value, other_value))
 
     def __add__(self, other: Any) -> Expression:
-        return self._apply_binary(other, bf_ops.add_op)
+        return self._apply_binary_op(other, bf_ops.add_op)
 
     def __radd__(self, other: Any) -> Expression:
-        return self._apply_binary(other, bf_ops.add_op, reverse=True)
+        return self._apply_binary_op(other, bf_ops.add_op, reverse=True)
 
     def __sub__(self, other: Any) -> Expression:
-        return self._apply_binary(other, bf_ops.sub_op)
+        return self._apply_binary_op(other, bf_ops.sub_op)
 
     def __rsub__(self, other: Any) -> Expression:
-        return self._apply_binary(other, bf_ops.sub_op, reverse=True)
+        return self._apply_binary_op(other, bf_ops.sub_op, reverse=True)
 
     def __mul__(self, other: Any) -> Expression:
-        return self._apply_binary(other, bf_ops.mul_op)
+        return self._apply_binary_op(other, bf_ops.mul_op)
 
     def __rmul__(self, other: Any) -> Expression:
-        return self._apply_binary(other, bf_ops.mul_op, reverse=True)
+        return self._apply_binary_op(other, bf_ops.mul_op, reverse=True)
 
     def __truediv__(self, other: Any) -> Expression:
-        return self._apply_binary(other, bf_ops.div_op)
+        return self._apply_binary_op(other, bf_ops.div_op)
 
     def __rtruediv__(self, other: Any) -> Expression:
-        return self._apply_binary(other, bf_ops.div_op, reverse=True)
+        return self._apply_binary_op(other, bf_ops.div_op, reverse=True)
 
     def __floordiv__(self, other: Any) -> Expression:
-        return self._apply_binary(other, bf_ops.floordiv_op)
+        return self._apply_binary_op(other, bf_ops.floordiv_op)
 
     def __rfloordiv__(self, other: Any) -> Expression:
-        return self._apply_binary(other, bf_ops.floordiv_op, reverse=True)
+        return self._apply_binary_op(other, bf_ops.floordiv_op, reverse=True)
 
     def __ge__(self, other: Any) -> Expression:
-        return self._apply_binary(other, bf_ops.ge_op)
+        return self._apply_binary_op(other, bf_ops.ge_op)
 
     def __gt__(self, other: Any) -> Expression:
-        return self._apply_binary(other, bf_ops.gt_op)
+        return self._apply_binary_op(other, bf_ops.gt_op)
 
     def __le__(self, other: Any) -> Expression:
-        return self._apply_binary(other, bf_ops.le_op)
+        return self._apply_binary_op(other, bf_ops.le_op)
 
     def __lt__(self, other: Any) -> Expression:
-        return self._apply_binary(other, bf_ops.lt_op)
+        return self._apply_binary_op(other, bf_ops.lt_op)
 
     def __eq__(self, other: object) -> Expression:  # type: ignore
-        return self._apply_binary(other, bf_ops.eq_op)
+        return self._apply_binary_op(other, bf_ops.eq_op)
 
     def __ne__(self, other: object) -> Expression:  # type: ignore
-        return self._apply_binary(other, bf_ops.ne_op)
+        return self._apply_binary_op(other, bf_ops.ne_op)
 
     def __mod__(self, other: Any) -> Expression:
-        return self._apply_binary(other, bf_ops.mod_op)
+        return self._apply_binary_op(other, bf_ops.mod_op)
 
     def __rmod__(self, other: Any) -> Expression:
-        return self._apply_binary(other, bf_ops.mod_op, reverse=True)
+        return self._apply_binary_op(other, bf_ops.mod_op, reverse=True)
 
     def __and__(self, other: Any) -> Expression:
-        return self._apply_binary(other, bf_ops.and_op)
+        return self._apply_binary_op(other, bf_ops.and_op)
 
     def __rand__(self, other: Any) -> Expression:
-        return self._apply_binary(other, bf_ops.and_op, reverse=True)
+        return self._apply_binary_op(other, bf_ops.and_op, reverse=True)
 
     def __or__(self, other: Any) -> Expression:
-        return self._apply_binary(other, bf_ops.or_op)
+        return self._apply_binary_op(other, bf_ops.or_op)
 
     def __ror__(self, other: Any) -> Expression:
-        return self._apply_binary(other, bf_ops.or_op, reverse=True)
+        return self._apply_binary_op(other, bf_ops.or_op, reverse=True)
 
     def __xor__(self, other: Any) -> Expression:
-        return self._apply_binary(other, bf_ops.xor_op)
+        return self._apply_binary_op(other, bf_ops.xor_op)
 
     def __rxor__(self, other: Any) -> Expression:
-        return self._apply_binary(other, bf_ops.xor_op, reverse=True)
+        return self._apply_binary_op(other, bf_ops.xor_op, reverse=True)
 
     def __invert__(self) -> Expression:
-        return self._apply_unary(bf_ops.invert_op)
+        return self._apply_unary_op(bf_ops.invert_op)
+
+    def sum(self) -> Expression:
+        return self._apply_unary_agg(agg_ops.sum_op)
+
+    def mean(self) -> Expression:
+        return self._apply_unary_agg(agg_ops.mean_op)
+
+    def var(self) -> Expression:
+        return self._apply_unary_agg(agg_ops.var_op)
+
+    def std(self) -> Expression:
+        return self._apply_unary_agg(agg_ops.std_op)
+
+    def min(self) -> Expression:
+        return self._apply_unary_agg(agg_ops.min_op)
+
+    def max(self) -> Expression:
+        return self._apply_unary_agg(agg_ops.max_op)
+
+    @property
+    def dt(self) -> datetimes.DatetimeSimpleMethods:
+        import bigframes.operations.datetimes as datetimes
+
+        return datetimes.DatetimeSimpleMethods(self)
+
+    @property
+    def str(self) -> strings.StringMethods:
+        import bigframes.operations.strings as strings
+
+        return strings.StringMethods(self)
 
 
 def col(col_name: Hashable) -> Expression:

bigframes/core/col.py

@@ -23,12 +23,12 @@
 import bigframes_vendored.ibis.expr.datatypes as ibis_dtypes
 import bigframes_vendored.ibis.expr.operations as ibis_ops
 import bigframes_vendored.ibis.expr.types as ibis_types
+import bigframes_vendored.sqlglot.expressions as sge
 from google.cloud import bigquery
 import pyarrow as pa
 
 from bigframes.core import agg_expressions, rewrite
 import bigframes.core.agg_expressions as ex_types
-import bigframes.core.compile.googlesql
 import bigframes.core.compile.ibis_compiler.aggregate_compiler as agg_compiler
 import bigframes.core.compile.ibis_compiler.scalar_op_compiler as op_compilers
 import bigframes.core.compile.ibis_types
@@ -82,13 +82,21 @@ def to_sql(
         )
 
         if order_by or limit or not is_noop_selection:
-            sql = ibis_bigquery.Backend().compile(ibis_table)
-            sql = (
-                bigframes.core.compile.googlesql.Select()
-                .from_(sql)
-                .select(selection_strings)
-                .sql()
-            )
+            # selections are (ref.id.sql, name) where ref.id.sql is escaped identifier
+            to_select = [
+                sge.Alias(
+                    this=sge.to_identifier(src, quoted=True),
+                    alias=sge.to_identifier(alias, quoted=True),
+                )
+                if src != alias
+                else sge.to_identifier(src, quoted=True)
+                for src, alias in selection_strings
+            ]
+            # Use string formatting for FROM clause to avoid re-parsing potentially complex SQL (like ARRAY<STRUCT<...>>)
+            # that sqlglot might not handle perfectly when parsing BigQuery dialect strings.
+            select_sql = sge.Select().select(*to_select).sql(dialect="bigquery")
+            ibis_sql = ibis_bigquery.Backend().compile(ibis_table)
+            sql = f"{select_sql} FROM ({ibis_sql}) AS `t`"
 
             # Single row frames may not have any ordering columns
             if len(order_by) > 0:
@@ -99,7 +107,7 @@ def to_sql(
                     raise TypeError(f"Limit param: {limit} must be an int.")
                 sql += f"\nLIMIT {limit}"
         else:
-            sql = ibis_bigquery.Backend().compile(self._to_ibis_expr())
+            sql = ibis_bigquery.Backend().compile(ibis_table)
         return typing.cast(str, sql)
 
     @property