Merge branch 'main' into shuowei-fix-compiler-syntax-guards

Author: shuoweil

.github/workflows/docs.yml

@@ -36,6 +36,3 @@ jobs:
       run: |
         python -m pip install --upgrade setuptools pip wheel
         python -m pip install nox
-    - name: Run docfx
-      run: |
-        nox -s docfx

.librarian/state.yaml

@@ -1,7 +1,7 @@
 image: us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/python-librarian-generator@sha256:160860d189ff1c2f7515638478823712fa5b243e27ccc33a2728669fa1e2ed0c
 libraries:
   - id: bigframes
-    version: 2.37.0
+    version: 2.38.0
     last_generated_commit: ""
     apis: []
     source_roots:

CHANGELOG.md

@@ -4,6 +4,29 @@
 
 [1]: https://pypi.org/project/bigframes/#history
 
+## [2.38.0](https://github.com/googleapis/python-bigquery-dataframes/compare/v2.37.0...v2.38.0) (2026-03-16)
+
+
+### Documentation
+
+* add notebooks to user guide page (#2505) ([5cf37888bc0b4b1b0993dadd1e0fe5ee08341ef4](https://github.com/googleapis/python-bigquery-dataframes/commit/5cf37888bc0b4b1b0993dadd1e0fe5ee08341ef4))
+* Fix typo in ExperimentOptions class docstring (#2498) ([077cb2ebe515fc5e07bcbb5dc663edd28d3eaf00](https://github.com/googleapis/python-bigquery-dataframes/commit/077cb2ebe515fc5e07bcbb5dc663edd28d3eaf00))
+
+
+### Features
+
+* add `df.bigquery` pandas accessor (#2513) ([91b6c245521218bb78b543885e1b9424278ce2ab](https://github.com/googleapis/python-bigquery-dataframes/commit/91b6c245521218bb78b543885e1b9424278ce2ab))
+* use EUC for AI IF, CLASSIFY, and SCORE when connection is not provided (#2507) ([fe94910abff28e244dd79e1540a6c2184a12eb44](https://github.com/googleapis/python-bigquery-dataframes/commit/fe94910abff28e244dd79e1540a6c2184a12eb44))
+* Add `bigframes.bigquery.rand()` function (#2501) ([5c43efb745118f506ecc30196da68e9d6f4346dc](https://github.com/googleapis/python-bigquery-dataframes/commit/5c43efb745118f506ecc30196da68e9d6f4346dc))
+* add bigquery.ml.get_insights function (#2493) ([d29a60953ac989bb2c95e6eec3010620ac776a3c](https://github.com/googleapis/python-bigquery-dataframes/commit/d29a60953ac989bb2c95e6eec3010620ac776a3c))
+* Add str, dt accessors to pd.col Expression objects (#2488) ([ce5de57019449ca77d308946df72f04289343b51](https://github.com/googleapis/python-bigquery-dataframes/commit/ce5de57019449ca77d308946df72f04289343b51))
+
+
+### Bug Fixes
+
+* handle unsupported types and empty results in describe (#2506) ([2326ad6aec15c20a66756eff093b50be484b3ba8](https://github.com/googleapis/python-bigquery-dataframes/commit/2326ad6aec15c20a66756eff093b50be484b3ba8))
+* no longer automatically use anywidget in the `%%bqsql` magics (#2504) ([43353e2bc9ffbc38b7383c24ecaac80d3b8bab32](https://github.com/googleapis/python-bigquery-dataframes/commit/43353e2bc9ffbc38b7383c24ecaac80d3b8bab32))
+
 ## [2.37.0](https://github.com/googleapis/python-bigquery-dataframes/compare/v2.36.0...v2.37.0) (2026-03-03)
 
 

bigframes/__init__.py

@@ -32,6 +32,9 @@
 )
 import bigframes.enums as enums  # noqa: E402
 import bigframes.exceptions as exceptions  # noqa: E402
+
+# Register pandas extensions
+import bigframes.extensions.pandas.dataframe_accessor  # noqa: F401, E402
 from bigframes.session import connect, Session  # noqa: E402
 from bigframes.version import __version__  # noqa: E402
 

bigframes/bigquery/__init__.py

@@ -12,9 +12,38 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""This module integrates BigQuery built-in functions for use with DataFrame objects,
-such as array functions:
-https://cloud.google.com/bigquery/docs/reference/standard-sql/array_functions. """
+"""
+Access BigQuery-specific operations and namespaces within BigQuery DataFrames.
+
+This module provides specialized functions and sub-modules that expose BigQuery's
+advanced capabilities to DataFrames and Series. It acts as a bridge between the
+pandas-compatible API and the full power of BigQuery SQL.
+
+Key sub-modules include:
+
+* :mod:`bigframes.bigquery.ai`: Generative and predictive AI functions (Gemini, BQML).
+* :mod:`bigframes.bigquery.ml`: Direct access to BigQuery ML model operations.
+* :mod:`bigframes.bigquery.obj`: Support for BigQuery object tables.
+
+This module also provides direct access to optimized BigQuery functions for:
+
+* **JSON Processing:** High-performance functions like ``json_extract``, ``json_value``,
+  and ``parse_json`` for handling semi-structured data.
+* **Geospatial Analysis:** Comprehensive geographic functions such as ``st_area``,
+  ``st_distance``, and ``st_centroid`` (``ST_`` prefixed functions).
+* **Array Operations:** Tools for working with BigQuery arrays, including ``array_agg``
+  and ``array_length``.
+* **Vector Search:** Integration with BigQuery's vector search and indexing
+  capabilities for high-dimensional data.
+* **Custom SQL:** The ``sql_scalar`` function allows embedding raw SQL snippets for
+  advanced operations not yet directly mapped in the API.
+
+By using these functions, you can leverage BigQuery's high-performance engine for
+domain-specific tasks while maintaining a Python-centric development experience.
+
+For the full list of BigQuery standard SQL functions, see:
+https://cloud.google.com/bigquery/docs/reference/standard-sql/functions-reference
+"""
 
 import sys
 

bigframes/bigquery/_operations/ai.py

@@ -745,7 +745,7 @@ def if_(
             or pandas Series.
         connection_id (str, optional):
             Specifies the connection to use to communicate with the model. For example, `myproject.us.myconnection`.
-            If not provided, the connection from the current session will be used.
+            If not provided, the query uses your end-user credential.
 
     Returns:
         bigframes.series.Series: A new series of bools.
@@ -756,7 +756,7 @@ def if_(
 
     operator = ai_ops.AIIf(
         prompt_context=tuple(prompt_context),
-        connection_id=_resolve_connection_id(series_list[0], connection_id),
+        connection_id=connection_id,
     )
 
     return series_list[0]._apply_nary_op(operator, series_list[1:])
@@ -800,7 +800,7 @@ def classify(
             Categories to classify the input into.
         connection_id (str, optional):
             Specifies the connection to use to communicate with the model. For example, `myproject.us.myconnection`.
-            If not provided, the connection from the current session will be used.
+            If not provided, the query uses your end-user credential.
 
     Returns:
         bigframes.series.Series: A new series of strings.
@@ -812,7 +812,7 @@ def classify(
     operator = ai_ops.AIClassify(
         prompt_context=tuple(prompt_context),
         categories=tuple(categories),
-        connection_id=_resolve_connection_id(series_list[0], connection_id),
+        connection_id=connection_id,
     )
 
     return series_list[0]._apply_nary_op(operator, series_list[1:])
@@ -853,7 +853,7 @@ def score(
             or pandas Series.
         connection_id (str, optional):
             Specifies the connection to use to communicate with the model. For example, `myproject.us.myconnection`.
-            If not provided, the connection from the current session will be used.
+            If not provided, the query uses your end-user credential.
 
     Returns:
         bigframes.series.Series: A new series of double (float) values.
@@ -864,7 +864,7 @@ def score(
 
     operator = ai_ops.AIScore(
         prompt_context=tuple(prompt_context),
-        connection_id=_resolve_connection_id(series_list[0], connection_id),
+        connection_id=connection_id,
     )
 
     return series_list[0]._apply_nary_op(operator, series_list[1:])
@@ -880,6 +880,7 @@ def forecast(
     id_cols: Iterable[str] | None = None,
     horizon: int = 10,
     confidence_level: float = 0.95,
+    output_historical_time_series: bool = False,
     context_window: int | None = None,
 ) -> dataframe.DataFrame:
     """
@@ -914,6 +915,15 @@ def forecast(
         confidence_level (float, default 0.95):
             A FLOAT64 value that specifies the percentage of the future values that fall in the prediction interval.
             The default value is 0.95. The valid input range is [0, 1).
+        output_historical_time_series (bool, default False):
+            A BOOL value that determines whether the input data is returned
+            along with the forecasted data. Set this argument to TRUE to return
+            input data. The default value is FALSE.
+
+            Returning the input data along with the forecasted data lets you
+            compare the historical value of the data column with the forecasted
+            value of the data column, or chart the change in the data column
+            values over time.
         context_window (int, optional):
             An int value that specifies the context window length used by BigQuery ML's built-in TimesFM model.
             The context window length determines how many of the most recent data points from the input time series are use by the model.
@@ -945,6 +955,7 @@ def forecast(
         "timestamp_col": timestamp_col,
         "model": model,
         "horizon": horizon,
+        "output_historical_time_series": output_historical_time_series,
         "confidence_level": confidence_level,
     }
     if id_cols:

bigframes/bigquery/_operations/io.py

@@ -19,8 +19,8 @@
 import pandas as pd
 
 from bigframes.bigquery._operations.table import _get_table_metadata
+import bigframes.core.compile.sqlglot.sql as sql
 import bigframes.core.logging.log_adapter as log_adapter
-import bigframes.core.sql.io
 import bigframes.session
 
 
@@ -73,7 +73,7 @@ def load_data(
     """
     import bigframes.pandas as bpd
 
-    sql = bigframes.core.sql.io.load_data_ddl(
+    load_data_expr = sql.load_data(
         table_name=table_name,
         write_disposition=write_disposition,
         columns=columns,
@@ -84,11 +84,12 @@ def load_data(
         with_partition_columns=with_partition_columns,
         connection_name=connection_name,
     )
+    sql_text = sql.to_sql(load_data_expr)
 
     if session is None:
-        bpd.read_gbq_query(sql)
+        bpd.read_gbq_query(sql_text)
         session = bpd.get_global_session()
     else:
-        session.read_gbq_query(sql)
+        session.read_gbq_query(sql_text)
 
     return _get_table_metadata(bqclient=session.bqclient, table_name=table_name)

bigframes/bigquery/_operations/sql.py

@@ -16,19 +16,31 @@
 
 from __future__ import annotations
 
-from typing import Sequence
+from typing import cast, Optional, Sequence, Union
 
 import google.cloud.bigquery
 
 from bigframes.core.compile.sqlglot import sql
+import bigframes.dataframe
 import bigframes.dtypes
 import bigframes.operations
 import bigframes.series
 
 
+def _format_names(sql_template: str, dataframe: bigframes.dataframe.DataFrame):
+    """Turn sql_template from a template that uses names to one that uses
+    numbers.
+    """
+    names_to_numbers = {name: f"{{{i}}}" for i, name in enumerate(dataframe.columns)}
+    numbers = [f"{{{i}}}" for i in range(len(dataframe.columns))]
+    return sql_template.format(*numbers, **names_to_numbers)
+
+
 def sql_scalar(
     sql_template: str,
-    columns: Sequence[bigframes.series.Series],
+    columns: Union[bigframes.dataframe.DataFrame, Sequence[bigframes.series.Series]],
+    *,
+    output_dtype: Optional[bigframes.dtypes.Dtype] = None,
 ) -> bigframes.series.Series:
     """Create a Series from a SQL template.
 
@@ -37,6 +49,9 @@ def sql_scalar(
         >>> import bigframes.pandas as bpd
         >>> import bigframes.bigquery as bbq
 
+    Either pass in a sequence of series, in which case use  integers in the
+    format strings.
+
         >>> s = bpd.Series(["1.5", "2.5", "3.5"])
         >>> s = s.astype(pd.ArrowDtype(pa.decimal128(38, 9)))
         >>> bbq.sql_scalar("ROUND({0}, 0, 'ROUND_HALF_EVEN')", [s])
@@ -45,13 +60,29 @@ def sql_scalar(
         2    4.000000000
         dtype: decimal128(38, 9)[pyarrow]
 
+    Or pass in a DataFrame, in which case use the column names in the format
+    strings.
+
+        >>> df = bpd.DataFrame({"a": ["1.5", "2.5", "3.5"]})
+        >>> df = df.astype({"a": pd.ArrowDtype(pa.decimal128(38, 9))})
+        >>> bbq.sql_scalar("ROUND({a}, 0, 'ROUND_HALF_EVEN')", df)
+        0    2.000000000
+        1    2.000000000
+        2    4.000000000
+        dtype: decimal128(38, 9)[pyarrow]
+
     Args:
         sql_template (str):
             A SQL format string with Python-style {0} placeholders for each of
             the Series objects in ``columns``.
-        columns (Sequence[bigframes.pandas.Series]):
+        columns (
+            Sequence[bigframes.pandas.Series] | bigframes.pandas.DataFrame
+        ):
             Series objects representing the column inputs to the
             ``sql_template``. Must contain at least one Series.
+        output_dtype (a BigQuery DataFrames compatible dtype, optional):
+            If provided, BigQuery DataFrames uses this to determine the output
+            of the returned Series. This avoids a dry run query.
 
     Returns:
         bigframes.pandas.Series:
@@ -60,28 +91,38 @@ def sql_scalar(
     Raises:
         ValueError: If ``columns`` is empty.
     """
+    if isinstance(columns, bigframes.dataframe.DataFrame):
+        sql_template = _format_names(sql_template, columns)
+        columns = [
+            cast(bigframes.series.Series, columns[column]) for column in columns.columns
+        ]
+
     if len(columns) == 0:
         raise ValueError("Must provide at least one column in columns")
 
+    base_series = columns[0]
+
     # To integrate this into our expression trees, we need to get the output
     # type, so we do some manual compilation and a dry run query to get that.
     # Another benefit of this is that if there is a syntax error in the SQL
     # template, then this will fail with an error earlier in the process,
     # aiding users in debugging.
-    literals_sql = [sql.to_sql(sql.literal(None, column.dtype)) for column in columns]
-    select_sql = sql_template.format(*literals_sql)
-    dry_run_sql = f"SELECT {select_sql}"
-
-    # Use the executor directly, because we want the original column IDs, not
-    # the user-friendly column names that block.to_sql_query() would produce.
-    base_series = columns[0]
-    bqclient = base_series._session.bqclient
-    job = bqclient.query(
-        dry_run_sql, job_config=google.cloud.bigquery.QueryJobConfig(dry_run=True)
-    )
-    _, output_type = bigframes.dtypes.convert_schema_field(job.schema[0])
+    if output_dtype is None:
+        literals_sql = [
+            sql.to_sql(sql.literal(None, column.dtype)) for column in columns
+        ]
+        select_sql = sql_template.format(*literals_sql)
+        dry_run_sql = f"SELECT {select_sql}"
+
+        # Use the executor directly, because we want the original column IDs, not
+        # the user-friendly column names that block.to_sql_query() would produce.
+        bqclient = base_series._session.bqclient
+        job = bqclient.query(
+            dry_run_sql, job_config=google.cloud.bigquery.QueryJobConfig(dry_run=True)
+        )
+        _, output_dtype = bigframes.dtypes.convert_schema_field(job.schema[0])
 
     op = bigframes.operations.SqlScalarOp(
-        _output_type=output_type, sql_template=sql_template
+        _output_type=output_dtype, sql_template=sql_template
     )
     return base_series._apply_nary_op(op, columns[1:])

bigframes/bigquery/_operations/table.py

@@ -19,8 +19,8 @@
 import google.cloud.bigquery
 import pandas as pd
 
+import bigframes.core.compile.sqlglot.sql as sg_sql
 import bigframes.core.logging.log_adapter as log_adapter
-import bigframes.core.sql.table
 import bigframes.session
 
 
@@ -80,14 +80,16 @@ def create_external_table(
     """
     import bigframes.pandas as bpd
 
-    sql = bigframes.core.sql.table.create_external_table_ddl(
-        table_name=table_name,
-        replace=replace,
-        if_not_exists=if_not_exists,
-        columns=columns,
-        partition_columns=partition_columns,
-        connection_name=connection_name,
-        options=options,
+    sql = sg_sql.to_sql(
+        sg_sql.create_external_table(
+            table_name=table_name,
+            replace=replace,
+            if_not_exists=if_not_exists,
+            columns=columns,
+            partition_columns=partition_columns,
+            connection_name=connection_name,
+            options=options,
+        )
     )
 
     if session is None: