fix: guard PythonUtils.toScalaMap, normalize Spark 4 CI version, rename PANDAS_VERSION_SPEC

- to_scala_map: add CollectionConverters fallback in case PythonUtils is
  removed in a future Spark/PySpark version
- base.yml: use "4.0" (two-part) for PYSPARK_VERSION to match other matrix
  entries
- base.yml: rename PANDAS_VERSION → PANDAS_VERSION_SPEC to reflect that the
  value is a version specifier (e.g. ">=2.0.0"), not a bare version number

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: m-aciek

.github/workflows/base.yml

@@ -25,10 +25,10 @@ jobs:
           - PYSPARK_VERSION: "3.5"
             PYTHON_VERSION: "3.9"
             JAVA_VERSION: "17"
-          - PYSPARK_VERSION: "4.0.0"
+          - PYSPARK_VERSION: "4.0"
             PYTHON_VERSION: "3.9"
             JAVA_VERSION: "17"
-            PANDAS_VERSION: ">=2.0.0"
+            PANDAS_VERSION_SPEC: ">=2.0.0"
 
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -45,11 +45,11 @@ jobs:
       - name: Running tests with pyspark==${{matrix.PYSPARK_VERSION}}
         env:
           SPARK_VERSION: ${{matrix.PYSPARK_VERSION}}
-          PANDAS_VERSION: ${{matrix.PANDAS_VERSION}}
+          PANDAS_VERSION_SPEC: ${{matrix.PANDAS_VERSION_SPEC}}
         run: |
           pip install --upgrade pip
           pip install poetry==1.7.1
           poetry install
           poetry run pip install pyspark==$SPARK_VERSION
-          if [ -n "$PANDAS_VERSION" ]; then poetry run pip install "pandas$PANDAS_VERSION"; fi
+          if [ -n "$PANDAS_VERSION_SPEC" ]; then poetry run pip install "pandas$PANDAS_VERSION_SPEC"; fi
           poetry run python -m pytest -s tests --ignore=tests/test_bot.py

pydeequ/scala_utils.py

@@ -124,7 +124,15 @@ def to_scala_map(spark_session, d):
     Returns:
         Scala map
     """
-    return spark_session._jvm.PythonUtils.toScalaMap(d)
+    jvm = spark_session._jvm
+    try:
+        # PythonUtils.toScalaMap is a PySpark internal that may be removed in future versions.
+        return jvm.PythonUtils.toScalaMap(d)
+    except Exception:
+        style, converters = _get_converters(jvm)
+        if style == "jdk":
+            return converters.asScala(d).toMap()
+        return converters.mapAsScalaMapConverter(d).asScala().toMap()
 
 
 def scala_map_to_dict(jvm, scala_map):