Merge branch 'main' into bigframes_magics

Author: TrevorBergeron

.librarian/state.yaml

@@ -1,7 +1,7 @@
 image: us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/python-librarian-generator@sha256:e7cc6823efb073a8a26e7cefdd869f12ec228abfbd2a44aa9a7eacc284023677
 libraries:
   - id: bigframes
-    version: 2.33.0
+    version: 2.34.0
     last_generated_commit: ""
     apis: []
     source_roots:

CHANGELOG.md

@@ -4,6 +4,22 @@
 
 [1]: https://pypi.org/project/bigframes/#history
 
+## [2.34.0](https://github.com/googleapis/python-bigquery-dataframes/compare/v2.33.0...v2.34.0) (2026-02-02)
+
+
+### Features
+
+* add `bigframes.pandas.options.experiments.sql_compiler` for switching the backend compiler (#2417) ([7eba6ee03f07938315d99e2aeaf72368c02074cf](https://github.com/googleapis/python-bigquery-dataframes/commit/7eba6ee03f07938315d99e2aeaf72368c02074cf))
+* add bigquery.ml.generate_embedding function (#2422) ([35f3f5e6f8c64b47e6e7214034f96f047785e647](https://github.com/googleapis/python-bigquery-dataframes/commit/35f3f5e6f8c64b47e6e7214034f96f047785e647))
+* add bigquery.create_external_table method (#2415) ([76db2956e505aec4f1055118ac7ca523facc10ff](https://github.com/googleapis/python-bigquery-dataframes/commit/76db2956e505aec4f1055118ac7ca523facc10ff))
+* add deprecation warnings for .blob accessor and read_gbq_object_table (#2408) ([7261a4ea5cdab6b30f5bc333501648c60e70be59](https://github.com/googleapis/python-bigquery-dataframes/commit/7261a4ea5cdab6b30f5bc333501648c60e70be59))
+* add bigquery.ml.generate_text function (#2403) ([5ac681028624de15e31f0c2ae360b47b2dcf1e8d](https://github.com/googleapis/python-bigquery-dataframes/commit/5ac681028624de15e31f0c2ae360b47b2dcf1e8d))
+
+
+### Bug Fixes
+
+* broken job url (#2411) ([fcb5bc1761c656e1aec61dbcf96a36d436833b7a](https://github.com/googleapis/python-bigquery-dataframes/commit/fcb5bc1761c656e1aec61dbcf96a36d436833b7a))
+
 ## [2.33.0](https://github.com/googleapis/python-bigquery-dataframes/compare/v2.32.0...v2.33.0) (2026-01-22)
 
 

bigframes/_config/experiment_options.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Optional
+from typing import Literal, Optional
 import warnings
 
 import bigframes
@@ -27,6 +27,7 @@ class ExperimentOptions:
     def __init__(self):
         self._semantic_operators: bool = False
         self._ai_operators: bool = False
+        self._sql_compiler: Literal["legacy", "stable", "experimental"] = "stable"
 
     @property
     def semantic_operators(self) -> bool:
@@ -55,6 +56,24 @@ def ai_operators(self, value: bool):
             warnings.warn(msg, category=bfe.PreviewWarning)
         self._ai_operators = value
 
+    @property
+    def sql_compiler(self) -> Literal["legacy", "stable", "experimental"]:
+        return self._sql_compiler
+
+    @sql_compiler.setter
+    def sql_compiler(self, value: Literal["legacy", "stable", "experimental"]):
+        if value not in ["legacy", "stable", "experimental"]:
+            raise ValueError(
+                "sql_compiler must be one of 'legacy', 'stable', or 'experimental'"
+            )
+        if value == "experimental":
+            msg = bfe.format_message(
+                "The experimental SQL compiler is still under experiments, and is subject "
+                "to change in the future."
+            )
+            warnings.warn(msg, category=FutureWarning)
+        self._sql_compiler = value
+
     @property
     def blob(self) -> bool:
         msg = bfe.format_message(

bigframes/bigquery/__init__.py

@@ -43,6 +43,7 @@
     st_regionstats,
     st_simplify,
 )
+from bigframes.bigquery._operations.io import load_data
 from bigframes.bigquery._operations.json import (
     json_extract,
     json_extract_array,
@@ -60,6 +61,7 @@
 from bigframes.bigquery._operations.search import create_vector_index, vector_search
 from bigframes.bigquery._operations.sql import sql_scalar
 from bigframes.bigquery._operations.struct import struct
+from bigframes.bigquery._operations.table import create_external_table
 from bigframes.core.logging import log_adapter
 
 _functions = [
@@ -104,6 +106,10 @@
     sql_scalar,
     # struct ops
     struct,
+    # table ops
+    create_external_table,
+    # io ops
+    load_data,
 ]
 
 _module = sys.modules[__name__]
@@ -155,6 +161,10 @@
     "sql_scalar",
     # struct ops
     "struct",
+    # table ops
+    "create_external_table",
+    # io ops
+    "load_data",
     # Modules / SQL namespaces
     "ai",
     "ml",

bigframes/bigquery/_operations/io.py

@@ -0,0 +1,94 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from typing import Mapping, Optional, Union
+
+import pandas as pd
+
+from bigframes.bigquery._operations.table import _get_table_metadata
+import bigframes.core.logging.log_adapter as log_adapter
+import bigframes.core.sql.io
+import bigframes.session
+
+
+@log_adapter.method_logger(custom_base_name="bigquery_io")
+def load_data(
+    table_name: str,
+    *,
+    write_disposition: str = "INTO",
+    columns: Optional[Mapping[str, str]] = None,
+    partition_by: Optional[list[str]] = None,
+    cluster_by: Optional[list[str]] = None,
+    table_options: Optional[Mapping[str, Union[str, int, float, bool, list]]] = None,
+    from_files_options: Mapping[str, Union[str, int, float, bool, list]],
+    with_partition_columns: Optional[Mapping[str, str]] = None,
+    connection_name: Optional[str] = None,
+    session: Optional[bigframes.session.Session] = None,
+) -> pd.Series:
+    """
+    Loads data into a BigQuery table.
+    See the `BigQuery LOAD DATA DDL syntax
+    <https://docs.cloud.google.com/bigquery/docs/reference/standard-sql/load-statements>`_
+    for additional reference.
+    Args:
+        table_name (str):
+            The name of the table in BigQuery.
+        write_disposition (str, default "INTO"):
+            Whether to replace the table if it already exists ("OVERWRITE") or append to it ("INTO").
+        columns (Mapping[str, str], optional):
+            The table's schema.
+        partition_by (list[str], optional):
+            A list of partition expressions to partition the table by. See https://docs.cloud.google.com/bigquery/docs/reference/standard-sql/load-statements#partition_expression.
+        cluster_by (list[str], optional):
+            A list of columns to cluster the table by.
+        table_options (Mapping[str, Union[str, int, float, bool, list]], optional):
+            The table options.
+        from_files_options (Mapping[str, Union[str, int, float, bool, list]]):
+            The options for loading data from files.
+        with_partition_columns (Mapping[str, str], optional):
+            The table's partition columns.
+        connection_name (str, optional):
+            The connection to use for the table.
+        session (bigframes.session.Session, optional):
+            The session to use. If not provided, the default session is used.
+    Returns:
+        pandas.Series:
+            A Series with object dtype containing the table metadata. Reference
+            the `BigQuery Table REST API reference
+            <https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#Table>`_
+            for available fields.
+    """
+    import bigframes.pandas as bpd
+
+    sql = bigframes.core.sql.io.load_data_ddl(
+        table_name=table_name,
+        write_disposition=write_disposition,
+        columns=columns,
+        partition_by=partition_by,
+        cluster_by=cluster_by,
+        table_options=table_options,
+        from_files_options=from_files_options,
+        with_partition_columns=with_partition_columns,
+        connection_name=connection_name,
+    )
+
+    if session is None:
+        bpd.read_gbq_query(sql)
+        session = bpd.get_global_session()
+    else:
+        session.read_gbq_query(sql)
+
+    return _get_table_metadata(bqclient=session.bqclient, table_name=table_name)

bigframes/bigquery/_operations/ml.py

@@ -520,3 +520,63 @@ def generate_text(
         return bpd.read_gbq_query(sql)
     else:
         return session.read_gbq_query(sql)
+
+
+@log_adapter.method_logger(custom_base_name="bigquery_ml")
+def generate_embedding(
+    model: Union[bigframes.ml.base.BaseEstimator, str, pd.Series],
+    input_: Union[pd.DataFrame, dataframe.DataFrame, str],
+    *,
+    flatten_json_output: Optional[bool] = None,
+    task_type: Optional[str] = None,
+    output_dimensionality: Optional[int] = None,
+) -> dataframe.DataFrame:
+    """
+    Generates text embedding using a BigQuery ML model.
+
+    See the `BigQuery ML GENERATE_EMBEDDING function syntax
+    <https://docs.cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-generate-embedding>`_
+    for additional reference.
+
+    Args:
+        model (bigframes.ml.base.BaseEstimator or str):
+            The model to use for text embedding.
+        input_ (Union[bigframes.pandas.DataFrame, str]):
+            The DataFrame or query to use for text embedding.
+        flatten_json_output (bool, optional):
+            A BOOL value that determines the content of the generated JSON column.
+        task_type (str, optional):
+            A STRING value that specifies the intended downstream application task.
+            Supported values are:
+            - `RETRIEVAL_QUERY`
+            - `RETRIEVAL_DOCUMENT`
+            - `SEMANTIC_SIMILARITY`
+            - `CLASSIFICATION`
+            - `CLUSTERING`
+            - `QUESTION_ANSWERING`
+            - `FACT_VERIFICATION`
+            - `CODE_RETRIEVAL_QUERY`
+        output_dimensionality (int, optional):
+            An INT64 value that specifies the size of the output embedding.
+
+    Returns:
+        bigframes.pandas.DataFrame:
+            The generated text embedding.
+    """
+    import bigframes.pandas as bpd
+
+    model_name, session = _get_model_name_and_session(model, input_)
+    table_sql = _to_sql(input_)
+
+    sql = bigframes.core.sql.ml.generate_embedding(
+        model_name=model_name,
+        table=table_sql,
+        flatten_json_output=flatten_json_output,
+        task_type=task_type,
+        output_dimensionality=output_dimensionality,
+    )
+
+    if session is None:
+        return bpd.read_gbq_query(sql)
+    else:
+        return session.read_gbq_query(sql)

bigframes/bigquery/_operations/table.py

@@ -0,0 +1,99 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from typing import Mapping, Optional, Union
+
+import google.cloud.bigquery
+import pandas as pd
+
+import bigframes.core.logging.log_adapter as log_adapter
+import bigframes.core.sql.table
+import bigframes.session
+
+
+def _get_table_metadata(
+    *,
+    bqclient: google.cloud.bigquery.Client,
+    table_name: str,
+) -> pd.Series:
+    table_metadata = bqclient.get_table(table_name)
+    table_dict = table_metadata.to_api_repr()
+    return pd.Series(table_dict)
+
+
+@log_adapter.method_logger(custom_base_name="bigquery_table")
+def create_external_table(
+    table_name: str,
+    *,
+    replace: bool = False,
+    if_not_exists: bool = False,
+    columns: Optional[Mapping[str, str]] = None,
+    partition_columns: Optional[Mapping[str, str]] = None,
+    connection_name: Optional[str] = None,
+    options: Mapping[str, Union[str, int, float, bool, list]],
+    session: Optional[bigframes.session.Session] = None,
+) -> pd.Series:
+    """
+    Creates a BigQuery external table.
+
+    See the `BigQuery CREATE EXTERNAL TABLE DDL syntax
+    <https://docs.cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_external_table_statement>`_
+    for additional reference.
+
+    Args:
+        table_name (str):
+            The name of the table in BigQuery.
+        replace (bool, default False):
+            Whether to replace the table if it already exists.
+        if_not_exists (bool, default False):
+            Whether to ignore the error if the table already exists.
+        columns (Mapping[str, str], optional):
+            The table's schema.
+        partition_columns (Mapping[str, str], optional):
+            The table's partition columns.
+        connection_name (str, optional):
+            The connection to use for the table.
+        options (Mapping[str, Union[str, int, float, bool, list]]):
+            The OPTIONS clause, which specifies the table options.
+        session (bigframes.session.Session, optional):
+            The session to use. If not provided, the default session is used.
+
+    Returns:
+        pandas.Series:
+            A Series with object dtype containing the table metadata. Reference
+            the `BigQuery Table REST API reference
+            <https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#Table>`_
+            for available fields.
+    """
+    import bigframes.pandas as bpd
+
+    sql = bigframes.core.sql.table.create_external_table_ddl(
+        table_name=table_name,
+        replace=replace,
+        if_not_exists=if_not_exists,
+        columns=columns,
+        partition_columns=partition_columns,
+        connection_name=connection_name,
+        options=options,
+    )
+
+    if session is None:
+        bpd.read_gbq_query(sql)
+        session = bpd.get_global_session()
+    else:
+        session.read_gbq_query(sql)
+
+    return _get_table_metadata(bqclient=session.bqclient, table_name=table_name)

bigframes/bigquery/ml.py

@@ -23,6 +23,7 @@
     create_model,
     evaluate,
     explain_predict,
+    generate_embedding,
     generate_text,
     global_explain,
     predict,
@@ -37,4 +38,5 @@
     "global_explain",
     "transform",
     "generate_text",
+    "generate_embedding",
 ]