Merge remote-tracking branch 'github/main' into session_simplify

Author: TrevorBergeron

CHANGELOG.md

@@ -4,6 +4,41 @@
 
 [1]: https://pypi.org/project/bigframes/#history
 
+## [2.16.0](https://github.com/googleapis/python-bigquery-dataframes/compare/v2.15.0...v2.16.0) (2025-08-20)
+
+
+### Features
+
+* Add `bigframes.pandas.options.display.precision` option ([#1979](https://github.com/googleapis/python-bigquery-dataframes/issues/1979)) ([15e6175](https://github.com/googleapis/python-bigquery-dataframes/commit/15e6175ec0aeb1b7b02d0bba9e8e1e018bd11c31))
+* Add level, inplace params to reset_index ([#1988](https://github.com/googleapis/python-bigquery-dataframes/issues/1988)) ([3446950](https://github.com/googleapis/python-bigquery-dataframes/commit/34469504b79a082d3380f9f25c597483aef2068a))
+* Add ML code samples from dbt blog post ([#1978](https://github.com/googleapis/python-bigquery-dataframes/issues/1978)) ([ebaa244](https://github.com/googleapis/python-bigquery-dataframes/commit/ebaa244a9eb7b87f7f9fd9c3bebe5c7db24cd013))
+* Add where, coalesce, fillna, casewhen, invert local impl ([#1976](https://github.com/googleapis/python-bigquery-dataframes/issues/1976)) ([f7f686c](https://github.com/googleapis/python-bigquery-dataframes/commit/f7f686cf85ab7e265d9c07ebc7f0cd59babc5357))
+* Adjust anywidget CSS to prevent overflow ([#1981](https://github.com/googleapis/python-bigquery-dataframes/issues/1981)) ([204f083](https://github.com/googleapis/python-bigquery-dataframes/commit/204f083a2f00fcc9fd1500dcd7a738eda3904d2f))
+* Format page number in table widget ([#1992](https://github.com/googleapis/python-bigquery-dataframes/issues/1992)) ([e83836e](https://github.com/googleapis/python-bigquery-dataframes/commit/e83836e8e1357f009f3f95666f1661bdbe0d3751))
+* Or, And, Xor can execute locally ([#1994](https://github.com/googleapis/python-bigquery-dataframes/issues/1994)) ([59c52a5](https://github.com/googleapis/python-bigquery-dataframes/commit/59c52a55ebea697855eb4c70529e226cc077141f))
+* Support callable bigframes function for dataframe where ([#1990](https://github.com/googleapis/python-bigquery-dataframes/issues/1990)) ([44c1ec4](https://github.com/googleapis/python-bigquery-dataframes/commit/44c1ec48cc4db1c4c9c15ec1fab43d4ef0758e56))
+* Support callable for series where method ([#2005](https://github.com/googleapis/python-bigquery-dataframes/issues/2005)) ([768b82a](https://github.com/googleapis/python-bigquery-dataframes/commit/768b82af96a5dd0c434edcb171036eb42cfb9b41))
+* When using `repr_mode = "anywidget"`, numeric values align right ([15e6175](https://github.com/googleapis/python-bigquery-dataframes/commit/15e6175ec0aeb1b7b02d0bba9e8e1e018bd11c31))
+
+
+### Bug Fixes
+
+* Address the packages issue for bigframes function ([#1991](https://github.com/googleapis/python-bigquery-dataframes/issues/1991)) ([68f1d22](https://github.com/googleapis/python-bigquery-dataframes/commit/68f1d22d5ed8457a5cabc7751ed1d178063dd63e))
+* Correct pypdf dependency specifier for remote PDF functions ([#1980](https://github.com/googleapis/python-bigquery-dataframes/issues/1980)) ([0bd5e1b](https://github.com/googleapis/python-bigquery-dataframes/commit/0bd5e1b3c004124d2100c3fbec2fbe1e965d1e96))
+* Enable default retries in calls to BQ Storage Read API ([#1985](https://github.com/googleapis/python-bigquery-dataframes/issues/1985)) ([f25d7bd](https://github.com/googleapis/python-bigquery-dataframes/commit/f25d7bd30800dffa65b6c31b0b7ac711a13d790f))
+* Fix the copyright year in dbt sample files ([#1996](https://github.com/googleapis/python-bigquery-dataframes/issues/1996)) ([fad5722](https://github.com/googleapis/python-bigquery-dataframes/commit/fad57223d129f0c95d0c6a066179bb66880edd06))
+
+
+### Performance Improvements
+
+* Faster session startup by defering anon dataset fetch ([#1982](https://github.com/googleapis/python-bigquery-dataframes/issues/1982)) ([2720c4c](https://github.com/googleapis/python-bigquery-dataframes/commit/2720c4cf070bf57a0930d7623bfc41d89cc053ee))
+
+
+### Documentation
+
+* Add examples of running bigframes in kaggle ([#2002](https://github.com/googleapis/python-bigquery-dataframes/issues/2002)) ([7d89d76](https://github.com/googleapis/python-bigquery-dataframes/commit/7d89d76976595b75cb0105fbe7b4f7ca2fdf49f2))
+* Remove preview warning from partial ordering mode sample notebook ([#1986](https://github.com/googleapis/python-bigquery-dataframes/issues/1986)) ([132e0ed](https://github.com/googleapis/python-bigquery-dataframes/commit/132e0edfe9f96c15753649d77fcb6edd0b0708a3))
+
 ## [2.15.0](https://github.com/googleapis/python-bigquery-dataframes/compare/v2.14.0...v2.15.0) (2025-08-11)
 
 

bigframes/core/compile/ibis_compiler/scalar_op_registry.py

@@ -1062,7 +1062,7 @@ def isin_op_impl(x: ibis_types.Value, op: ops.IsInOp):
     if op.match_nulls and contains_nulls:
         return x.isnull() | x.isin(matchable_ibis_values)
     else:
-        return x.isin(matchable_ibis_values)
+        return x.isin(matchable_ibis_values).fillna(False)
 
 
 @scalar_op_compiler.register_unary_op(ops.ToDatetimeOp, pass_op=True)

bigframes/core/compile/polars/compiler.py

@@ -198,6 +198,10 @@ def _(self, op: ops.ScalarOp, l_input: pl.Expr, r_input: pl.Expr) -> pl.Expr:
         def _(self, op: ops.ScalarOp, l_input: pl.Expr, r_input: pl.Expr) -> pl.Expr:
             return l_input | r_input
 
+        @compile_op.register(bool_ops.XorOp)
+        def _(self, op: ops.ScalarOp, l_input: pl.Expr, r_input: pl.Expr) -> pl.Expr:
+            return l_input ^ r_input
+
         @compile_op.register(num_ops.AddOp)
         def _(self, op: ops.ScalarOp, l_input: pl.Expr, r_input: pl.Expr) -> pl.Expr:
             return l_input + r_input
@@ -259,11 +263,9 @@ def _(self, op: ops.ScalarOp, l_input: pl.Expr, r_input: pl.Expr) -> pl.Expr:
         def _(self, op: ops.ScalarOp, input: pl.Expr) -> pl.Expr:
             # TODO: Filter out types that can't be coerced to right type
             assert isinstance(op, gen_ops.IsInOp)
-            if op.match_nulls or not any(map(pd.isna, op.values)):
-                # newer polars version have nulls_equal arg
-                return input.is_in(op.values)
-            else:
-                return input.is_in(op.values) or input.is_null()
+            assert not op.match_nulls  # should be stripped by a lowering step rn
+            values = pl.Series(op.values, strict=False)
+            return input.is_in(values)
 
         @compile_op.register(gen_ops.FillNaOp)
         @compile_op.register(gen_ops.CoalesceOp)

bigframes/core/compile/polars/lowering.py

@@ -13,8 +13,10 @@
 # limitations under the License.
 
 import dataclasses
+from typing import cast
 
 import numpy as np
+import pandas as pd
 
 from bigframes import dtypes
 from bigframes.core import bigframe_node, expression
@@ -316,6 +318,35 @@ def lower(self, expr: expression.OpExpression) -> expression.Expression:
         return expr
 
 
+class LowerIsinOp(op_lowering.OpLoweringRule):
+    @property
+    def op(self) -> type[ops.ScalarOp]:
+        return generic_ops.IsInOp
+
+    def lower(self, expr: expression.OpExpression) -> expression.Expression:
+        assert isinstance(expr.op, generic_ops.IsInOp)
+        arg = expr.children[0]
+        new_values = []
+        match_nulls = False
+        for val in expr.op.values:
+            # coercible, non-coercible
+            # float NaN/inf should be treated as distinct from 'true' null values
+            if cast(bool, pd.isna(val)) and not isinstance(val, float):
+                if expr.op.match_nulls:
+                    match_nulls = True
+            elif dtypes.is_compatible(val, arg.output_type):
+                new_values.append(val)
+            else:
+                pass
+
+        new_isin = ops.IsInOp(tuple(new_values), match_nulls=False).as_expr(arg)
+        if match_nulls:
+            return ops.coalesce_op.as_expr(new_isin, expression.const(True))
+        else:
+            # polars propagates nulls, so need to coalesce to false
+            return ops.coalesce_op.as_expr(new_isin, expression.const(False))
+
+
 def _coerce_comparables(
     expr1: expression.Expression,
     expr2: expression.Expression,
@@ -414,6 +445,7 @@ def _lower_cast(cast_op: ops.AsTypeOp, arg: expression.Expression):
     LowerModRule(),
     LowerAsTypeRule(),
     LowerInvertOp(),
+    LowerIsinOp(),
 )
 
 

bigframes/core/compile/sqlglot/aggregations/unary_compiler.py

@@ -37,6 +37,15 @@ def compile(
     return UNARY_OP_REGISTRATION[op](op, column, window=window)
 
 
+@UNARY_OP_REGISTRATION.register(agg_ops.CountOp)
+def _(
+    op: agg_ops.CountOp,
+    column: typed_expr.TypedExpr,
+    window: typing.Optional[window_spec.WindowSpec] = None,
+) -> sge.Expression:
+    return apply_window_if_present(sge.func("COUNT", column.expr), window)
+
+
 @UNARY_OP_REGISTRATION.register(agg_ops.SumOp)
 def _(
     op: agg_ops.SumOp,

bigframes/core/compile/sqlglot/compiler.py

@@ -336,6 +336,9 @@ def compile_window(
                             this=is_observation_expr, expression=expr
                         )
                 is_observation = ir._cast(is_observation_expr, "INT64")
+                observation_count = windows.apply_window_if_present(
+                    sge.func("SUM", is_observation), window_spec
+                )
             else:
                 # Operations like count treat even NULLs as valid observations
                 # for the sake of min_periods notnull is just used to convert
@@ -344,10 +347,10 @@ def compile_window(
                     sge.Not(this=sge.Is(this=inputs[0], expression=sge.Null())),
                     "INT64",
                 )
+                observation_count = windows.apply_window_if_present(
+                    sge.func("COUNT", is_observation), window_spec
+                )
 
-            observation_count = windows.apply_window_if_present(
-                sge.func("SUM", is_observation), window_spec
-            )
             clauses.append(
                 (
                     observation_count < sge.convert(window_spec.min_periods),

bigframes/core/compile/sqlglot/expressions/binary_compiler.py

@@ -14,11 +14,12 @@
 
 from __future__ import annotations
 
-import bigframes_vendored.constants as constants
+import bigframes_vendored.constants as bf_constants
 import sqlglot.expressions as sge
 
 from bigframes import dtypes
 from bigframes import operations as ops
+import bigframes.core.compile.sqlglot.expressions.constants as constants
 from bigframes.core.compile.sqlglot.expressions.op_registration import OpRegistration
 from bigframes.core.compile.sqlglot.expressions.typed_expr import TypedExpr
 
@@ -37,50 +38,65 @@ def _(op, left: TypedExpr, right: TypedExpr) -> sge.Expression:
         return sge.Concat(expressions=[left.expr, right.expr])
 
     if dtypes.is_numeric(left.dtype) and dtypes.is_numeric(right.dtype):
-        left_expr = left.expr
-        if left.dtype == dtypes.BOOL_DTYPE:
-            left_expr = sge.Cast(this=left_expr, to="INT64")
-        right_expr = right.expr
-        if right.dtype == dtypes.BOOL_DTYPE:
-            right_expr = sge.Cast(this=right_expr, to="INT64")
+        left_expr = _coerce_bool_to_int(left)
+        right_expr = _coerce_bool_to_int(right)
         return sge.Add(this=left_expr, expression=right_expr)
 
     if (
         dtypes.is_time_or_date_like(left.dtype)
         and right.dtype == dtypes.TIMEDELTA_DTYPE
     ):
-        left_expr = left.expr
-        if left.dtype == dtypes.DATE_DTYPE:
-            left_expr = sge.Cast(this=left_expr, to="DATETIME")
+        left_expr = _coerce_date_to_datetime(left)
         return sge.TimestampAdd(
             this=left_expr, expression=right.expr, unit=sge.Var(this="MICROSECOND")
         )
     if (
         dtypes.is_time_or_date_like(right.dtype)
         and left.dtype == dtypes.TIMEDELTA_DTYPE
     ):
-        right_expr = right.expr
-        if right.dtype == dtypes.DATE_DTYPE:
-            right_expr = sge.Cast(this=right_expr, to="DATETIME")
+        right_expr = _coerce_date_to_datetime(right)
         return sge.TimestampAdd(
             this=right_expr, expression=left.expr, unit=sge.Var(this="MICROSECOND")
         )
     if left.dtype == dtypes.TIMEDELTA_DTYPE and right.dtype == dtypes.TIMEDELTA_DTYPE:
         return sge.Add(this=left.expr, expression=right.expr)
 
     raise TypeError(
-        f"Cannot add type {left.dtype} and {right.dtype}. {constants.FEEDBACK_LINK}"
+        f"Cannot add type {left.dtype} and {right.dtype}. {bf_constants.FEEDBACK_LINK}"
     )
 
 
-@BINARY_OP_REGISTRATION.register(ops.div_op)
+@BINARY_OP_REGISTRATION.register(ops.eq_op)
+def _(op, left: TypedExpr, right: TypedExpr) -> sge.Expression:
+    left_expr = _coerce_bool_to_int(left)
+    right_expr = _coerce_bool_to_int(right)
+    return sge.EQ(this=left_expr, expression=right_expr)
+
+
+@BINARY_OP_REGISTRATION.register(ops.eq_null_match_op)
 def _(op, left: TypedExpr, right: TypedExpr) -> sge.Expression:
     left_expr = left.expr
-    if left.dtype == dtypes.BOOL_DTYPE:
-        left_expr = sge.Cast(this=left_expr, to="INT64")
+    if right.dtype != dtypes.BOOL_DTYPE:
+        left_expr = _coerce_bool_to_int(left)
+
     right_expr = right.expr
-    if right.dtype == dtypes.BOOL_DTYPE:
-        right_expr = sge.Cast(this=right_expr, to="INT64")
+    if left.dtype != dtypes.BOOL_DTYPE:
+        right_expr = _coerce_bool_to_int(right)
+
+    sentinel = sge.convert("$NULL_SENTINEL$")
+    left_coalesce = sge.Coalesce(
+        this=sge.Cast(this=left_expr, to="STRING"), expressions=[sentinel]
+    )
+    right_coalesce = sge.Coalesce(
+        this=sge.Cast(this=right_expr, to="STRING"), expressions=[sentinel]
+    )
+    return sge.EQ(this=left_coalesce, expression=right_coalesce)
+
+
+@BINARY_OP_REGISTRATION.register(ops.div_op)
+def _(op, left: TypedExpr, right: TypedExpr) -> sge.Expression:
+    left_expr = _coerce_bool_to_int(left)
+    right_expr = _coerce_bool_to_int(right)
 
     result = sge.func("IEEE_DIVIDE", left_expr, right_expr)
     if left.dtype == dtypes.TIMEDELTA_DTYPE and dtypes.is_numeric(right.dtype):
@@ -89,6 +105,39 @@ def _(op, left: TypedExpr, right: TypedExpr) -> sge.Expression:
         return result
 
 
+@BINARY_OP_REGISTRATION.register(ops.floordiv_op)
+def _(op, left: TypedExpr, right: TypedExpr) -> sge.Expression:
+    left_expr = _coerce_bool_to_int(left)
+    right_expr = _coerce_bool_to_int(right)
+
+    result: sge.Expression = sge.Cast(
+        this=sge.Floor(this=sge.func("IEEE_DIVIDE", left_expr, right_expr)), to="INT64"
+    )
+
+    # DIV(N, 0) will error in bigquery, but needs to return `0` for int, and
+    # `inf`` for float in BQ so we short-circuit in this case.
+    # Multiplying left by zero propogates nulls.
+    zero_result = (
+        constants._INF
+        if (left.dtype == dtypes.FLOAT_DTYPE or right.dtype == dtypes.FLOAT_DTYPE)
+        else constants._ZERO
+    )
+    result = sge.Case(
+        ifs=[
+            sge.If(
+                this=sge.EQ(this=right_expr, expression=constants._ZERO),
+                true=zero_result * left_expr,
+            )
+        ],
+        default=result,
+    )
+
+    if dtypes.is_numeric(right.dtype) and left.dtype == dtypes.TIMEDELTA_DTYPE:
+        result = sge.Cast(this=sge.Floor(this=result), to="INT64")
+
+    return result
+
+
 @BINARY_OP_REGISTRATION.register(ops.ge_op)
 def _(op, left: TypedExpr, right: TypedExpr) -> sge.Expression:
     return sge.GTE(this=left.expr, expression=right.expr)
@@ -101,12 +150,8 @@ def _(op, left: TypedExpr, right: TypedExpr) -> sge.Expression:
 
 @BINARY_OP_REGISTRATION.register(ops.mul_op)
 def _(op, left: TypedExpr, right: TypedExpr) -> sge.Expression:
-    left_expr = left.expr
-    if left.dtype == dtypes.BOOL_DTYPE:
-        left_expr = sge.Cast(this=left_expr, to="INT64")
-    right_expr = right.expr
-    if right.dtype == dtypes.BOOL_DTYPE:
-        right_expr = sge.Cast(this=right_expr, to="INT64")
+    left_expr = _coerce_bool_to_int(left)
+    right_expr = _coerce_bool_to_int(right)
 
     result = sge.Mul(this=left_expr, expression=right_expr)
 
@@ -118,36 +163,33 @@ def _(op, left: TypedExpr, right: TypedExpr) -> sge.Expression:
         return result
 
 
+@BINARY_OP_REGISTRATION.register(ops.ne_op)
+def _(op, left: TypedExpr, right: TypedExpr) -> sge.Expression:
+    left_expr = _coerce_bool_to_int(left)
+    right_expr = _coerce_bool_to_int(right)
+    return sge.NEQ(this=left_expr, expression=right_expr)
+
+
 @BINARY_OP_REGISTRATION.register(ops.sub_op)
 def _(op, left: TypedExpr, right: TypedExpr) -> sge.Expression:
     if dtypes.is_numeric(left.dtype) and dtypes.is_numeric(right.dtype):
-        left_expr = left.expr
-        if left.dtype == dtypes.BOOL_DTYPE:
-            left_expr = sge.Cast(this=left_expr, to="INT64")
-        right_expr = right.expr
-        if right.dtype == dtypes.BOOL_DTYPE:
-            right_expr = sge.Cast(this=right_expr, to="INT64")
+        left_expr = _coerce_bool_to_int(left)
+        right_expr = _coerce_bool_to_int(right)
         return sge.Sub(this=left_expr, expression=right_expr)
 
     if (
         dtypes.is_time_or_date_like(left.dtype)
         and right.dtype == dtypes.TIMEDELTA_DTYPE
     ):
-        left_expr = left.expr
-        if left.dtype == dtypes.DATE_DTYPE:
-            left_expr = sge.Cast(this=left_expr, to="DATETIME")
+        left_expr = _coerce_date_to_datetime(left)
         return sge.TimestampSub(
             this=left_expr, expression=right.expr, unit=sge.Var(this="MICROSECOND")
         )
     if dtypes.is_time_or_date_like(left.dtype) and dtypes.is_time_or_date_like(
         right.dtype
     ):
-        left_expr = left.expr
-        if left.dtype == dtypes.DATE_DTYPE:
-            left_expr = sge.Cast(this=left_expr, to="DATETIME")
-        right_expr = right.expr
-        if right.dtype == dtypes.DATE_DTYPE:
-            right_expr = sge.Cast(this=right_expr, to="DATETIME")
+        left_expr = _coerce_date_to_datetime(left)
+        right_expr = _coerce_date_to_datetime(right)
         return sge.TimestampDiff(
             this=left_expr, expression=right_expr, unit=sge.Var(this="MICROSECOND")
         )
@@ -156,10 +198,24 @@ def _(op, left: TypedExpr, right: TypedExpr) -> sge.Expression:
         return sge.Sub(this=left.expr, expression=right.expr)
 
     raise TypeError(
-        f"Cannot subtract type {left.dtype} and {right.dtype}. {constants.FEEDBACK_LINK}"
+        f"Cannot subtract type {left.dtype} and {right.dtype}. {bf_constants.FEEDBACK_LINK}"
     )
 
 
 @BINARY_OP_REGISTRATION.register(ops.obj_make_ref_op)
 def _(op, left: TypedExpr, right: TypedExpr) -> sge.Expression:
     return sge.func("OBJ.MAKE_REF", left.expr, right.expr)
+
+
+def _coerce_bool_to_int(typed_expr: TypedExpr) -> sge.Expression:
+    """Coerce boolean expression to integer."""
+    if typed_expr.dtype == dtypes.BOOL_DTYPE:
+        return sge.Cast(this=typed_expr.expr, to="INT64")
+    return typed_expr.expr
+
+
+def _coerce_date_to_datetime(typed_expr: TypedExpr) -> sge.Expression:
+    """Coerce date expression to datetime."""
+    if typed_expr.dtype == dtypes.DATE_DTYPE:
+        return sge.Cast(this=typed_expr.expr, to="DATETIME")
+    return typed_expr.expr