setup deps for new engines

Author: TrevorBergeron

packages/bigframes/noxfile.py

@@ -72,12 +72,12 @@
 UNIT_TEST_DEPENDENCIES: List[str] = []
 UNIT_TEST_EXTRAS: List[str] = ["tests"]
 UNIT_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = {
-    "3.10": ["tests", "scikit-learn", "anywidget"],
-    "3.11": ["tests", "polars", "scikit-learn", "anywidget"],
+    "3.10": ["tests", "polars", "datafusion", "substrait", "scikit-learn", "anywidget"],
+    "3.11": ["tests", "polars", "datafusion", "substrait", "scikit-learn", "anywidget"],
     # Make sure we leave some versions without "extras" so we know those
     # dependencies are actually optional.
-    "3.13": ["tests", "polars", "scikit-learn", "anywidget"],
-    "3.14": ["tests", "polars", "scikit-learn", "anywidget"],
+    "3.13": ["tests", "polars", "datafusion", "substrait", "scikit-learn", "anywidget"],
+    "3.14": ["tests", "polars", "datafusion", "substrait", "scikit-learn", "anywidget"],
 }
 
 # 3.11 is used by colab.
@@ -106,9 +106,9 @@
     # Make sure we leave some versions without "extras" so we know those
     # dependencies are actually optional.
     "3.10": ["tests", "scikit-learn", "anywidget"],
-    "3.12": ["tests", "scikit-learn", "polars", "anywidget"],
-    "3.13": ["tests", "polars", "anywidget"],
-    "3.14": ["tests", "polars", "anywidget"],
+    "3.12": ["tests", "scikit-learn", "polars", "datafusion", "substrait", "anywidget"],
+    "3.13": ["tests", "polars", "datafusion", "substrait", "anywidget"],
+    "3.14": ["tests", "polars", "datafusion", "substrait", "anywidget"],
 }
 
 LOGGING_NAME_ENV_VAR = "BIGFRAMES_PERFORMANCE_LOG_NAME"

packages/bigframes/setup.py

@@ -79,6 +79,8 @@
     ],
     # used for local engine
     "polars": ["polars >= 1.21.0"],
+    "datafusion": ["datafusion >= 45.0.0", "substrait >= 0.29"],
+    "substrait": ["substrait >= 0.29"],
     "scikit-learn": ["scikit-learn>=1.2.2"],
     # Packages required for basic development flow.
     "dev": [

packages/bigframes/testing/constraints-3.10.txt

@@ -16,6 +16,9 @@ grpc-google-iam-v1==0.14.2
 numpy==1.24.0
 pandas==1.5.3
 pandas-gbq==0.26.1
+polars==1.21.0
+substrait==0.29.0
+datafusion==45.0.0
 pyarrow==23.0.1
 pydata-google-auth==1.8.2
 pyiceberg==0.7.1

packages/bigframes/tests/system/small/engines/conftest.py

@@ -17,6 +17,18 @@
 import google.cloud.bigquery_storage_v1
 import pandas as pd
 import pytest
+
+# Skip the entire engines test directory if required libraries are missing
+try:
+    import datafusion  # noqa: F401
+    import polars  # noqa: F401
+    import substrait  # noqa: F401
+except ImportError as e:
+    pytest.skip(
+        f"Skipping engines tests because dependencies are missing: {e}",
+        allow_module_level=True,
+    )
+
 from google.cloud import bigquery
 
 import bigframes