[SEDONA-725] restructure spark python package (#1930)

* SEDONA-725 Add pyflink to Sedona.

* SEDONA-725 rearrange the spark module

* SEDONA-725 rearrange the spark module

* SEDONA-725 rearrange the spark module

* SEDONA-725 rearrange the spark module

* SEDONA-725 rearrange the spark module

Author: Imbruced

.pre-commit-config.yaml

@@ -125,6 +125,15 @@ repos:
           - --license-filepath
           - .github/workflows/license-templates/LICENSE.txt
           - --fuzzy-match-generates-todo
+      - id: insert-license
+        name: add license for all Python files
+        files: \.py$
+        args:
+          - --comment-style
+          - '|# |'
+          - --license-filepath
+          - .github/workflows/license-templates/LICENSE.txt
+          - --fuzzy-match-generates-todo
   - repo: https://github.com/asottile/pyupgrade
     rev: v3.19.1
     hooks:

docs/api/sql/Raster-visualizer.md

@@ -77,7 +77,7 @@ Output:
 Example:
 
 ```python
-from sedona.raster_utils.SedonaUtils import SedonaUtils
+from sedona.spark import SedonaUtils
 
 # Or from sedona.spark import *
 

docs/api/sql/Visualization_SedonaKepler.md

@@ -28,7 +28,7 @@ from sedona.spark import *
 Alternatively it can also be imported using:
 
 ```python
-from sedona.maps.SedonaKepler import SedonaKepler
+from sedona.spark import SedonaKepler
 ```
 
 Following are details on all the APIs exposed via SedonaKepler:

docs/api/sql/Visualization_SedonaPyDeck.md

@@ -28,7 +28,7 @@ from sedona.spark import *
 Alternatively it can also be imported using:
 
 ```python
-from sedona.maps.SedonaPyDeck import SedonaPyDeck
+from sedona.spark import SedonaPyDeck
 ```
 
 !!!Note

docs/setup/install-python.md

@@ -85,8 +85,8 @@ SedonaRegistrator is deprecated in Sedona 1.4.1 and later versions. Please use t
 
 ```python
 from pyspark.sql import SparkSession
-from sedona.register import SedonaRegistrator
-from sedona.utils import SedonaKryoRegistrator, KryoSerializer
+from sedona.spark import SedonaRegistrator
+from sedona.spark import SedonaKryoRegistrator, KryoSerializer
 
 spark = (
     SparkSession.builder.appName("appName")

docs/tutorial/concepts/clustering-algorithms.md

@@ -96,7 +96,7 @@ Here are the contents of the DataFrame:
 Here’s how to run the DBSCAN algorithm:
 
 ```python
-from sedona.stats.clustering.dbscan import dbscan
+from sedona.spark.stats import dbscan
 
 dbscan(df, 1.0, 3).orderBy("id").show()
 ```

docs/tutorial/files/stac-sedona-spark.md

@@ -204,7 +204,7 @@ The Python API allows you to interact with a SpatioTemporal Asset Catalog (STAC)
 #### Initialize the Client
 
 ```python
-from sedona.stac.client import Client
+from sedona.spark.stac import Client
 
 # Initialize the client
 client = Client.open("https://planetarycomputer.microsoft.com/api/stac/v1")

docs/tutorial/geopandas-shapely.md

@@ -81,7 +81,7 @@ def create_spatial_dataframe(spark: SparkSession, gdf: gpd.GeoDataFrame) -> Data
 Example:
 
 ```python
-from sedona.utils.geoarrow import create_spatial_dataframe
+from sedona.spark.geoarrow import create_spatial_dataframe
 
 create_spatial_dataframe(spark, gdf)
 ```

docs/tutorial/rdd.md

@@ -51,7 +51,7 @@ Please refer to [Create a Geometry type column](sql.md#create-a-geometry-type-co
 === "Python"
 
 	```python
-	from sedona.utils.structured_adapter import StructuredAdapter
+	from sedona.spark import StructuredAdapter
 
 	spatialRDD = StructuredAdapter.toSpatialRdd(spatialDf, "usacounty")
 	```
@@ -165,8 +165,8 @@ Assume you now have a SpatialRDD (typed or generic). You can use the following c
 === "Python"
 
 	```python
-	from sedona.core.geom.envelope import Envelope
-	from sedona.core.spatialOperator import RangeQuery
+	from sedona.spark import Envelope
+	from sedona.spark import RangeQuery
 
 	range_query_window = Envelope(-90.01, -80.01, 30.01, 40.01)
 	consider_boundary_intersection = False  ## Only return gemeotries fully covered by the window
@@ -179,9 +179,9 @@ Assume you now have a SpatialRDD (typed or generic). You can use the following c
 
     Example:
     ```python
-    from sedona.core.geom.envelope import Envelope
-    from sedona.core.spatialOperator import RangeQueryRaw
-    from sedona.utils.adapter import Adapter
+    from sedona.spark import Envelope
+    from sedona.spark import RangeQueryRaw
+    from sedona.spark import Adapter
 
     range_query_window = Envelope(-90.01, -80.01, 30.01, 40.01)
     consider_boundary_intersection = False  ## Only return gemeotries fully covered by the window
@@ -283,9 +283,9 @@ To utilize a spatial index in a spatial range query, use the following code:
 === "Python"
 
 	```python
-	from sedona.core.geom.envelope import Envelope
-	from sedona.core.enums import IndexType
-	from sedona.core.spatialOperator import RangeQuery
+	from sedona.spark import Envelope
+	from sedona.spark import IndexType
+	from sedona.spark import RangeQuery
 
 	range_query_window = Envelope(-90.01, -80.01, 30.01, 40.01)
 	consider_boundary_intersection = False ## Only return gemeotries fully covered by the window
@@ -379,7 +379,7 @@ Assume you now have a SpatialRDD (typed or generic). You can use the following c
 === "Python"
 
 	```python
-	from sedona.core.spatialOperator import KNNQuery
+	from sedona.spark import KNNQuery
 	from shapely.geometry import Point
 
 	point = Point(-84.01, 34.01)
@@ -446,8 +446,8 @@ To utilize a spatial index in a spatial KNN query, use the following code:
 === "Python"
 
 	```python
-	from sedona.core.spatialOperator import KNNQuery
-	from sedona.core.enums import IndexType
+	from sedona.spark import KNNQuery
+	from sedona.spark import IndexType
 	from shapely.geometry import Point
 
 	point = Point(-84.01, 34.01)
@@ -518,8 +518,8 @@ Assume you now have two SpatialRDDs (typed or generic). You can use the followin
 === "Python"
 
 	```python
-	from sedona.core.enums import GridType
-	from sedona.core.spatialOperator import JoinQuery
+	from sedona.spark import GridType
+	from sedona.spark import JoinQuery
 
 	consider_boundary_intersection = False ## Only return geometries fully covered by each query window in queryWindowRDD
 	using_index = False
@@ -610,9 +610,9 @@ To utilize a spatial index in a spatial join query, use the following code:
 === "Python"
 
 	```python
-	from sedona.core.enums import GridType
-	from sedona.core.enums import IndexType
-	from sedona.core.spatialOperator import JoinQuery
+	from sedona.spark import GridType
+	from sedona.spark import IndexType
+	from sedona.spark import JoinQuery
 
 	object_rdd.spatialPartitioning(GridType.KDBTREE)
 	query_window_rdd.spatialPartitioning(object_rdd.getPartitioner())
@@ -676,10 +676,10 @@ The index should be built on either one of two SpatialRDDs. In general, you shou
 
     Example:
     ```python
-    from sedona.core.SpatialRDD import CircleRDD
-    from sedona.core.enums import GridType
-    from sedona.core.spatialOperator import JoinQueryRaw
-	from sedona.utils.structured_adapter import StructuredAdapter
+    from sedona.spark import CircleRDD
+    from sedona.spark import GridType
+    from sedona.spark import JoinQueryRaw
+	from sedona.spark import StructuredAdapter
 
     object_rdd.analyze()
 
@@ -743,9 +743,9 @@ Assume you now have two SpatialRDDs (typed or generic). You can use the followin
 === "Python"
 
 	```python
-	from sedona.core.SpatialRDD import CircleRDD
-	from sedona.core.enums import GridType
-	from sedona.core.spatialOperator import JoinQuery
+	from sedona.spark import CircleRDD
+	from sedona.spark import GridType
+	from sedona.spark import JoinQuery
 
 	object_rdd.analyze()
 

docs/tutorial/sql.md

@@ -614,7 +614,7 @@ The first parameter is the dataframe, the next two are the epsilon and min_point
 === "Python"
 
 	```python
-	from sedona.stats.clustering.dbscan import dbscan
+	from sedona.spark.stats import dbscan
 
 	dbscan(df, 0.1, 5).show()
 	```
@@ -670,7 +670,7 @@ The first parameter is the dataframe, the next is the number of nearest neighbor
 === "Python"
 
 	```python
-	from sedona.stats.outlier_detection.local_outlier_factor import local_outlier_factor
+	from sedona.spark.stats import local_outlier_factor
 
 	local_outlier_factor(df, 20).show()
 	```
@@ -737,8 +737,8 @@ Using Gi involves first generating the neighbors list for each record, then call
 === "Python"
 
 	```python
-	from sedona.stats.weighting import add_binary_distance_band_column
-	from sedona.stats.hotspot_detection.getis_ord import g_local
+	from sedona.spark.stats import add_binary_distance_band_column
+	from sedona.spark.stats import g_local
 
 	distance_radius = 1.0
 	weighted_df = addBinaryDistanceBandColumn(df, distance_radius)
@@ -966,7 +966,7 @@ This UDF example takes a geometry type input and returns a primitive type output
 === "Python"
 
 	```python
-	from sedona.sql.types import GeometryType
+	from sedona.spark.sql.types import GeometryType
 	from pyspark.sql.types import DoubleType
 
 	def lengthPoly(geom: GeometryType()):
@@ -1025,7 +1025,7 @@ This UDF example takes a geometry type input and returns a geometry type output:
 === "Python"
 
 	```python
-	from sedona.sql.types import GeometryType
+	from sedona.spark import GeometryType
 	from pyspark.sql.types import DoubleType
 
 	def bufferFixed(geom: GeometryType()):
@@ -1083,7 +1083,7 @@ This UDF example takes a geometry type input and a primitive type input and retu
 === "Python"
 
 	```python
-	from sedona.sql.types import GeometryType
+	from sedona.spark import GeometryType
 	from pyspark.sql.types import DoubleType
 
 	def bufferIt(geom: GeometryType(), distance: DoubleType()):
@@ -1165,7 +1165,7 @@ This UDF example takes a geometry type input and a primitive type input and retu
 === "Python"
 
 	```python
-	from sedona.sql.types import GeometryType
+	from sedona.spark import GeometryType
 	from pyspark.sql.types import *
 
 	schemaUDF = StructType([
@@ -1230,7 +1230,7 @@ a given geometry.
 
 ```python
 import shapely.geometry.base as b
-from sedona.sql.functions import sedona_vectorized_udf
+from sedona.spark import sedona_vectorized_udf
 
 @sedona_vectorized_udf(return_type=GeometryType())
 def vectorized_buffer(geom: b.BaseGeometry) -> b.BaseGeometry:
@@ -1241,8 +1241,8 @@ def vectorized_buffer(geom: b.BaseGeometry) -> b.BaseGeometry:
 
 ```python
 import geopandas as gpd
-from sedona.sql.functions import sedona_vectorized_udf, SedonaUDFType
-from sedona.sql.types import GeometryType
+from sedona.spark import sedona_vectorized_udf, SedonaUDFType
+from sedona.spark import GeometryType
 
 
 @sedona_vectorized_udf(udf_type=SedonaUDFType.GEO_SERIES, return_type=GeometryType())
@@ -1339,7 +1339,7 @@ Use SedonaSQL DataFrame-RDD Adapter to convert a DataFrame to an SpatialRDD.
 === "Python"
 
 	```python
-	from sedona.utils.structured_adapter import StructuredAdapter
+	from sedona.spark import StructuredAdapter
 
 	spatialRDD = StructuredAdapter.toSpatialRdd(spatialDf, "usacounty")
 	```
@@ -1365,7 +1365,7 @@ Use SedonaSQL DataFrame-RDD Adapter to convert a DataFrame to an SpatialRDD. Ple
 === "Python"
 
 	```python
-	from sedona.utils.adapter import StructuredAdapter
+	from sedona.spark import StructuredAdapter
 
 	spatialDf = StructuredAdapter.toDf(spatialRDD, sedona)
 	```
@@ -1401,7 +1401,7 @@ You can use `StructuredAdapter` and the `spatialRDD.spatialPartitioningWithoutDu
 === "Python"
 
 	```python
-	from sedona.utils.structured_adapter import StructuredAdapter
+	from sedona.spark import StructuredAdapter
 
 	spatialRDD.spatialPartitioningWithoutDuplicates(GridType.KDBTREE)
 	# Specify the desired number of partitions as 10, though the actual number may vary
@@ -1427,7 +1427,7 @@ PairRDD is the result of a spatial join query or distance join query. SedonaSQL
 === "Python"
 
 	```python
-	from sedona.utils.adapter import StructuredAdapter
+	from sedona.spark import StructuredAdapter
 
 	joinResultDf = StructuredAdapter.pairRddToDf(result_pair_rdd, leftDf.schema, rightDf.schema, spark)
 	```