Merge remote-tracking branch 'github/main' into garrettwu-load

Author: GarrettWu

.librarian/state.yaml

@@ -1,7 +1,7 @@
 image: us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/python-librarian-generator@sha256:e7cc6823efb073a8a26e7cefdd869f12ec228abfbd2a44aa9a7eacc284023677
 libraries:
   - id: bigframes
-    version: 2.33.0
+    version: 2.34.0
     last_generated_commit: ""
     apis: []
     source_roots:

CHANGELOG.md

@@ -4,6 +4,22 @@
 
 [1]: https://pypi.org/project/bigframes/#history
 
+## [2.34.0](https://github.com/googleapis/python-bigquery-dataframes/compare/v2.33.0...v2.34.0) (2026-02-02)
+
+
+### Features
+
+* add `bigframes.pandas.options.experiments.sql_compiler` for switching the backend compiler (#2417) ([7eba6ee03f07938315d99e2aeaf72368c02074cf](https://github.com/googleapis/python-bigquery-dataframes/commit/7eba6ee03f07938315d99e2aeaf72368c02074cf))
+* add bigquery.ml.generate_embedding function (#2422) ([35f3f5e6f8c64b47e6e7214034f96f047785e647](https://github.com/googleapis/python-bigquery-dataframes/commit/35f3f5e6f8c64b47e6e7214034f96f047785e647))
+* add bigquery.create_external_table method (#2415) ([76db2956e505aec4f1055118ac7ca523facc10ff](https://github.com/googleapis/python-bigquery-dataframes/commit/76db2956e505aec4f1055118ac7ca523facc10ff))
+* add deprecation warnings for .blob accessor and read_gbq_object_table (#2408) ([7261a4ea5cdab6b30f5bc333501648c60e70be59](https://github.com/googleapis/python-bigquery-dataframes/commit/7261a4ea5cdab6b30f5bc333501648c60e70be59))
+* add bigquery.ml.generate_text function (#2403) ([5ac681028624de15e31f0c2ae360b47b2dcf1e8d](https://github.com/googleapis/python-bigquery-dataframes/commit/5ac681028624de15e31f0c2ae360b47b2dcf1e8d))
+
+
+### Bug Fixes
+
+* broken job url (#2411) ([fcb5bc1761c656e1aec61dbcf96a36d436833b7a](https://github.com/googleapis/python-bigquery-dataframes/commit/fcb5bc1761c656e1aec61dbcf96a36d436833b7a))
+
 ## [2.33.0](https://github.com/googleapis/python-bigquery-dataframes/compare/v2.32.0...v2.33.0) (2026-01-22)
 
 

bigframes/_config/experiment_options.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Optional
+from typing import Literal, Optional
 import warnings
 
 import bigframes
@@ -27,6 +27,7 @@ class ExperimentOptions:
     def __init__(self):
         self._semantic_operators: bool = False
         self._ai_operators: bool = False
+        self._sql_compiler: Literal["legacy", "stable", "experimental"] = "stable"
 
     @property
     def semantic_operators(self) -> bool:
@@ -55,6 +56,24 @@ def ai_operators(self, value: bool):
             warnings.warn(msg, category=bfe.PreviewWarning)
         self._ai_operators = value
 
+    @property
+    def sql_compiler(self) -> Literal["legacy", "stable", "experimental"]:
+        return self._sql_compiler
+
+    @sql_compiler.setter
+    def sql_compiler(self, value: Literal["legacy", "stable", "experimental"]):
+        if value not in ["legacy", "stable", "experimental"]:
+            raise ValueError(
+                "sql_compiler must be one of 'legacy', 'stable', or 'experimental'"
+            )
+        if value == "experimental":
+            msg = bfe.format_message(
+                "The experimental SQL compiler is still under experiments, and is subject "
+                "to change in the future."
+            )
+            warnings.warn(msg, category=FutureWarning)
+        self._sql_compiler = value
+
     @property
     def blob(self) -> bool:
         msg = bfe.format_message(

bigframes/bigquery/_operations/table.py

@@ -16,7 +16,6 @@
 
 from typing import Mapping, Optional, Union
 
-import bigframes_vendored.constants
 import google.cloud.bigquery
 import pandas as pd
 
@@ -94,9 +93,6 @@ def create_external_table(
     if session is None:
         bpd.read_gbq_query(sql)
         session = bpd.get_global_session()
-        assert (
-            session is not None
-        ), f"Missing connection to BigQuery. Please report how you encountered this error at {bigframes_vendored.constants.FEEDBACK_LINK}."
     else:
         session.read_gbq_query(sql)
 

bigframes/core/compile/__init__.py

@@ -13,13 +13,28 @@
 # limitations under the License.
 from __future__ import annotations
 
+from typing import Any
+
+from bigframes import options
 from bigframes.core.compile.api import test_only_ibis_inferred_schema
 from bigframes.core.compile.configs import CompileRequest, CompileResult
-from bigframes.core.compile.ibis_compiler.ibis_compiler import compile_sql
+
+
+def compiler() -> Any:
+    """Returns the appropriate compiler module based on session options."""
+    if options.experiments.sql_compiler == "experimental":
+        import bigframes.core.compile.sqlglot.compiler as sqlglot_compiler
+
+        return sqlglot_compiler
+    else:
+        import bigframes.core.compile.ibis_compiler.ibis_compiler as ibis_compiler
+
+        return ibis_compiler
+
 
 __all__ = [
     "test_only_ibis_inferred_schema",
-    "compile_sql",
     "CompileRequest",
     "CompileResult",
+    "compiler",
 ]

bigframes/core/compile/configs.py

@@ -34,3 +34,4 @@ class CompileResult:
     sql: str
     sql_schema: typing.Sequence[google.cloud.bigquery.SchemaField]
     row_order: typing.Optional[ordering.RowOrdering]
+    encoded_type_refs: str

bigframes/core/compile/ibis_compiler/ibis_compiler.py

@@ -29,6 +29,7 @@
 import bigframes.core.compile.concat as concat_impl
 import bigframes.core.compile.configs as configs
 import bigframes.core.compile.explode
+from bigframes.core.logging import data_types as data_type_logger
 import bigframes.core.nodes as nodes
 import bigframes.core.ordering as bf_ordering
 import bigframes.core.rewrite as rewrites
@@ -56,23 +57,30 @@ def compile_sql(request: configs.CompileRequest) -> configs.CompileResult:
     )
     if request.sort_rows:
         result_node = cast(nodes.ResultNode, rewrites.column_pruning(result_node))
+        encoded_type_refs = data_type_logger.encode_type_refs(result_node)
         sql = compile_result_node(result_node)
         return configs.CompileResult(
-            sql, result_node.schema.to_bigquery(), result_node.order_by
+            sql,
+            result_node.schema.to_bigquery(),
+            result_node.order_by,
+            encoded_type_refs,
         )
 
     ordering: Optional[bf_ordering.RowOrdering] = result_node.order_by
     result_node = dataclasses.replace(result_node, order_by=None)
     result_node = cast(nodes.ResultNode, rewrites.column_pruning(result_node))
     result_node = cast(nodes.ResultNode, rewrites.defer_selection(result_node))
+    encoded_type_refs = data_type_logger.encode_type_refs(result_node)
     sql = compile_result_node(result_node)
     # Return the ordering iff no extra columns are needed to define the row order
     if ordering is not None:
         output_order = (
             ordering if ordering.referenced_columns.issubset(result_node.ids) else None
         )
     assert (not request.materialize_all_order_keys) or (output_order is not None)
-    return configs.CompileResult(sql, result_node.schema.to_bigquery(), output_order)
+    return configs.CompileResult(
+        sql, result_node.schema.to_bigquery(), output_order, encoded_type_refs
+    )
 
 
 def _replace_unsupported_ops(node: nodes.BigFrameNode):

bigframes/core/compile/sqlglot/compiler.py

@@ -34,6 +34,7 @@
 from bigframes.core.compile.sqlglot.expressions import typed_expr
 import bigframes.core.compile.sqlglot.scalar_compiler as scalar_compiler
 import bigframes.core.compile.sqlglot.sqlglot_ir as ir
+from bigframes.core.logging import data_types as data_type_logger
 import bigframes.core.ordering as bf_ordering
 from bigframes.core.rewrite import schema_binding
 
@@ -59,23 +60,29 @@ def compile_sql(request: configs.CompileRequest) -> configs.CompileResult:
     )
     if request.sort_rows:
         result_node = typing.cast(nodes.ResultNode, rewrite.column_pruning(result_node))
+        encoded_type_refs = data_type_logger.encode_type_refs(result_node)
         sql = _compile_result_node(result_node)
         return configs.CompileResult(
-            sql, result_node.schema.to_bigquery(), result_node.order_by
+            sql,
+            result_node.schema.to_bigquery(),
+            result_node.order_by,
+            encoded_type_refs,
         )
 
     ordering: typing.Optional[bf_ordering.RowOrdering] = result_node.order_by
     result_node = dataclasses.replace(result_node, order_by=None)
     result_node = typing.cast(nodes.ResultNode, rewrite.column_pruning(result_node))
+    encoded_type_refs = data_type_logger.encode_type_refs(result_node)
     sql = _compile_result_node(result_node)
-
     # Return the ordering iff no extra columns are needed to define the row order
     if ordering is not None:
         output_order = (
             ordering if ordering.referenced_columns.issubset(result_node.ids) else None
         )
     assert (not request.materialize_all_order_keys) or (output_order is not None)
-    return configs.CompileResult(sql, result_node.schema.to_bigquery(), output_order)
+    return configs.CompileResult(
+        sql, result_node.schema.to_bigquery(), output_order, encoded_type_refs
+    )
 
 
 def _remap_variables(

bigframes/formatting_helpers.py

@@ -200,10 +200,12 @@ def progress_callback(
                 display_id=current_display_id,
             )
         elif isinstance(event, bigframes.core.events.ExecutionFinished):
-            display.update_display(
-                display.HTML(f"✅ Completed. {previous_display_html}"),
-                display_id=current_display_id,
-            )
+            if previous_display_html:
+                display.update_display(
+                    display.HTML(f"✅ Completed. {previous_display_html}"),
+                    display_id=current_display_id,
+                )
+
         elif isinstance(event, bigframes.core.events.SessionClosed):
             display.update_display(
                 display.HTML(f"Session {event.session_id} closed."),

bigframes/session/bq_caching_executor.py

@@ -174,7 +174,9 @@ def to_sql(
             else array_value.node
         )
         node = self._substitute_large_local_sources(node)
-        compiled = compile.compile_sql(compile.CompileRequest(node, sort_rows=ordered))
+        compiled = compile.compiler().compile_sql(
+            compile.CompileRequest(node, sort_rows=ordered)
+        )
         return compiled.sql
 
     def execute(
@@ -290,7 +292,9 @@ def _export_gbq(
         # validate destination table
         existing_table = self._maybe_find_existing_table(spec)
 
-        compiled = compile.compile_sql(compile.CompileRequest(plan, sort_rows=False))
+        compiled = compile.compiler().compile_sql(
+            compile.CompileRequest(plan, sort_rows=False)
+        )
         sql = compiled.sql
 
         if (existing_table is not None) and _if_schema_match(
@@ -318,6 +322,8 @@ def _export_gbq(
                 clustering_fields=spec.cluster_cols if spec.cluster_cols else None,
             )
 
+        # Attach data type usage to the job labels
+        job_config.labels["bigframes-dtypes"] = compiled.encoded_type_refs
         # TODO(swast): plumb through the api_name of the user-facing api that
         # caused this query.
         iterator, job = self._run_execute_query(
@@ -641,7 +647,7 @@ def _execute_plan_gbq(
                 ]
                 cluster_cols = cluster_cols[:_MAX_CLUSTER_COLUMNS]
 
-        compiled = compile.compile_sql(
+        compiled = compile.compiler().compile_sql(
             compile.CompileRequest(
                 plan,
                 sort_rows=ordered,
@@ -661,6 +667,8 @@ def _execute_plan_gbq(
             )
             job_config.destination = destination_table
 
+        # Attach data type usage to the job labels
+        job_config.labels["bigframes-dtypes"] = compiled.encoded_type_refs
         iterator, query_job = self._run_execute_query(
             sql=compiled.sql,
             job_config=job_config,