Address review feedback on V2 PR

- Fix NameError in tests/conftest.py setup_pyspark shim
- Drop duplicate PyDeequSession entry in __getattr__ tuple
- Widen pyspark pin to >=3.5.0,<4.0.0; remove empty extras table
- Bump black target_version to py39 to match python requirement
- Raise ValueError on Check.where() before any constraint
- Replace fixed sleep with port-wait for Spark Connect server in CI

Author: chenliu0831

.github/workflows/base.yml

@@ -56,8 +56,8 @@ jobs:
             --packages org.apache.spark:spark-connect_2.12:3.5.0 \
             --jars $PWD/deequ_2.12-2.1.0b-spark-3.5.jar \
             --conf spark.connect.extensions.relation.classes=com.amazon.deequ.connect.DeequRelationPlugin
-          # Wait for server to start
-          sleep 20
+          # Wait for the gRPC port to accept connections (fail fast if startup broke)
+          timeout 60 bash -c 'until (echo > /dev/tcp/localhost/15002) >/dev/null 2>&1; do sleep 1; done'
           # Verify server is running
           ps aux | grep SparkConnectServer | grep -v grep
 

pydeequ/__init__.py

@@ -45,7 +45,7 @@ def __getattr__(name):
         return _f2j_maven_coord
 
     if name in ("AnalysisRunner", "Check", "CheckLevel", "ColumnProfilerRunner",
-                "PyDeequSession", "DEEQU_MAVEN_COORD"):
+                "DEEQU_MAVEN_COORD"):
         # Import legacy modules on demand
         if name == "AnalysisRunner":
             from pydeequ.analyzers import AnalysisRunner

pydeequ/v2/checks.py

@@ -901,8 +901,9 @@ def where(self, filter_condition: str) -> "Check":
         Example:
             check.isComplete("email").where("status = 'active'")
         """
-        if self._constraints:
-            self._constraints[-1].where = filter_condition
+        if not self._constraints:
+            raise ValueError("where() called before any constraint was added")
+        self._constraints[-1].where = filter_condition
         return self
 
     # ========================================================================

pyproject.toml

@@ -33,7 +33,7 @@ numpy = ">=1.23.0"
 pandas = ">=1.5.0"
 protobuf = ">=4.21.0"
 setuptools = ">=69.0.0"  # Required for Python 3.12+ (distutils removed)
-pyspark = {version = "3.5.0", extras = ["connect"]}
+pyspark = {version = ">=3.5.0,<4.0.0", extras = ["connect"]}
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.0.0"
@@ -43,8 +43,6 @@ black = "^24.0.0"
 pre-commit = "^3.6.0"
 pytest-rerunfailures = "^14.0"
 
-[tool.poetry.extras]
-
 [build-system]
 requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
@@ -57,7 +55,7 @@ include_trailing_comma = true
 force_grid_wrap = 0
 use_parentheses = true
 ensure_newline_before_comments = true
-target_version = ['py38']
+target_version = ['py39']
 include = '\.pyi?$'
 exclude = '''
 /(

tests/conftest.py

@@ -75,6 +75,6 @@ def config(self, key, value):
             return self
 
         def getOrCreate(self):
-            return get_spark_connect_session()
+            return create_spark_connect_session()
 
     return SparkConnectBuilder()