[pyspark] Add support for Spark Connect ML (#11970)

- Support Spark Connect
- Drop support for old Spark 3.x.

Author: medb

dmlc-core

@@ -26,7 +26,7 @@ dependencies:
 - llvmlite
 - loky>=3.5.1
 - pyarrow
-- pyspark>=3.4.0
+- pyspark>=4.0.0
 - cloudpickle
 - pip:
   - awscli

ops/conda_env/aarch64_test.yml

@@ -38,4 +38,4 @@ dependencies:
 - protobuf
 - cloudpickle
 - modin
-- pyspark>=3.4.0
+- pyspark>=4.0.0

ops/conda_env/linux_cpu_test.yml

@@ -20,4 +20,4 @@ dependencies:
 - pytest
 - hypothesis
 - hatchling
-- pyspark>=3.4.0
+- pyspark>=4.0.0

ops/conda_env/python_lint.yml

@@ -46,7 +46,7 @@ pandas = ["pandas>=1.2"]
 scikit-learn = ["scikit-learn"]
 dask = ["dask", "pandas", "distributed"]
 plotting = ["graphviz", "matplotlib"]
-pyspark = ["pyspark>=3.4", "scikit-learn", "cloudpickle"]
+pyspark = ["pyspark>=4.0", "scikit-learn", "cloudpickle"]
 
 [tool.hatch.build.targets.wheel.hooks.custom]
 

python-package/pyproject.toml

@@ -45,7 +45,7 @@ pandas = ["pandas>=1.2"]
 scikit-learn = ["scikit-learn"]
 dask = ["dask", "pandas", "distributed"]
 plotting = ["graphviz", "matplotlib"]
-pyspark = ["pyspark>=3.4", "scikit-learn", "cloudpickle"]
+pyspark = ["pyspark>=4.0", "scikit-learn", "cloudpickle"]
 
 [tool.hatch.build.targets.wheel.hooks.custom]