diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d99385aa3a3..f7a1c51d93e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -125,6 +125,15 @@ repos: - --license-filepath - .github/workflows/license-templates/LICENSE.txt - --fuzzy-match-generates-todo + - id: insert-license + name: add license for all Python files + files: \.py$ + args: + - --comment-style + - '|# |' + - --license-filepath + - .github/workflows/license-templates/LICENSE.txt + - --fuzzy-match-generates-todo - repo: https://github.com/asottile/pyupgrade rev: v3.19.1 hooks: diff --git a/docs/api/sql/Raster-visualizer.md b/docs/api/sql/Raster-visualizer.md index f0b83cb1650..ce58eb986c6 100644 --- a/docs/api/sql/Raster-visualizer.md +++ b/docs/api/sql/Raster-visualizer.md @@ -77,7 +77,7 @@ Output: Example: ```python -from sedona.raster_utils.SedonaUtils import SedonaUtils +from sedona.spark import SedonaUtils # Or from sedona.spark import * diff --git a/docs/api/sql/Visualization_SedonaKepler.md b/docs/api/sql/Visualization_SedonaKepler.md index 48891095294..fa4fb4b3029 100644 --- a/docs/api/sql/Visualization_SedonaKepler.md +++ b/docs/api/sql/Visualization_SedonaKepler.md @@ -28,7 +28,7 @@ from sedona.spark import * Alternatively it can also be imported using: ```python -from sedona.maps.SedonaKepler import SedonaKepler +from sedona.spark import SedonaKepler ``` Following are details on all the APIs exposed via SedonaKepler: diff --git a/docs/api/sql/Visualization_SedonaPyDeck.md b/docs/api/sql/Visualization_SedonaPyDeck.md index a9b0f5a1c80..62a4f061a2e 100644 --- a/docs/api/sql/Visualization_SedonaPyDeck.md +++ b/docs/api/sql/Visualization_SedonaPyDeck.md @@ -28,7 +28,7 @@ from sedona.spark import * Alternatively it can also be imported using: ```python -from sedona.maps.SedonaPyDeck import SedonaPyDeck +from sedona.spark import SedonaPyDeck ``` !!!Note diff --git a/docs/setup/install-python.md b/docs/setup/install-python.md index 0ea1e2cc8a5..02cb51d4755 100644 --- a/docs/setup/install-python.md +++ b/docs/setup/install-python.md @@ -85,8 +85,8 @@ SedonaRegistrator is deprecated in Sedona 1.4.1 and later versions. Please use t ```python from pyspark.sql import SparkSession -from sedona.register import SedonaRegistrator -from sedona.utils import SedonaKryoRegistrator, KryoSerializer +from sedona.spark import SedonaRegistrator +from sedona.spark import SedonaKryoRegistrator, KryoSerializer spark = ( SparkSession.builder.appName("appName") diff --git a/docs/tutorial/concepts/clustering-algorithms.md b/docs/tutorial/concepts/clustering-algorithms.md index 79cb72cd8f9..460e252ad69 100644 --- a/docs/tutorial/concepts/clustering-algorithms.md +++ b/docs/tutorial/concepts/clustering-algorithms.md @@ -96,7 +96,7 @@ Here are the contents of the DataFrame: Here’s how to run the DBSCAN algorithm: ```python -from sedona.stats.clustering.dbscan import dbscan +from sedona.spark.stats import dbscan dbscan(df, 1.0, 3).orderBy("id").show() ``` diff --git a/docs/tutorial/files/stac-sedona-spark.md b/docs/tutorial/files/stac-sedona-spark.md index 864b6ef9d66..78b8616f836 100644 --- a/docs/tutorial/files/stac-sedona-spark.md +++ b/docs/tutorial/files/stac-sedona-spark.md @@ -204,7 +204,7 @@ The Python API allows you to interact with a SpatioTemporal Asset Catalog (STAC) #### Initialize the Client ```python -from sedona.stac.client import Client +from sedona.spark.stac import Client # Initialize the client client = Client.open("https://planetarycomputer.microsoft.com/api/stac/v1") diff --git a/docs/tutorial/geopandas-shapely.md b/docs/tutorial/geopandas-shapely.md index c3fcbf2dc3b..66c4c068f38 100644 --- a/docs/tutorial/geopandas-shapely.md +++ b/docs/tutorial/geopandas-shapely.md @@ -81,7 +81,7 @@ def create_spatial_dataframe(spark: SparkSession, gdf: gpd.GeoDataFrame) -> Data Example: ```python -from sedona.utils.geoarrow import create_spatial_dataframe +from sedona.spark.geoarrow import create_spatial_dataframe create_spatial_dataframe(spark, gdf) ``` diff --git a/docs/tutorial/rdd.md b/docs/tutorial/rdd.md index d3b27896f7c..23b94d519ff 100644 --- a/docs/tutorial/rdd.md +++ b/docs/tutorial/rdd.md @@ -51,7 +51,7 @@ Please refer to [Create a Geometry type column](sql.md#create-a-geometry-type-co === "Python" ```python - from sedona.utils.structured_adapter import StructuredAdapter + from sedona.spark import StructuredAdapter spatialRDD = StructuredAdapter.toSpatialRdd(spatialDf, "usacounty") ``` @@ -165,8 +165,8 @@ Assume you now have a SpatialRDD (typed or generic). You can use the following c === "Python" ```python - from sedona.core.geom.envelope import Envelope - from sedona.core.spatialOperator import RangeQuery + from sedona.spark import Envelope + from sedona.spark import RangeQuery range_query_window = Envelope(-90.01, -80.01, 30.01, 40.01) consider_boundary_intersection = False ## Only return gemeotries fully covered by the window @@ -179,9 +179,9 @@ Assume you now have a SpatialRDD (typed or generic). You can use the following c Example: ```python - from sedona.core.geom.envelope import Envelope - from sedona.core.spatialOperator import RangeQueryRaw - from sedona.utils.adapter import Adapter + from sedona.spark import Envelope + from sedona.spark import RangeQueryRaw + from sedona.spark import Adapter range_query_window = Envelope(-90.01, -80.01, 30.01, 40.01) consider_boundary_intersection = False ## Only return gemeotries fully covered by the window @@ -283,9 +283,9 @@ To utilize a spatial index in a spatial range query, use the following code: === "Python" ```python - from sedona.core.geom.envelope import Envelope - from sedona.core.enums import IndexType - from sedona.core.spatialOperator import RangeQuery + from sedona.spark import Envelope + from sedona.spark import IndexType + from sedona.spark import RangeQuery range_query_window = Envelope(-90.01, -80.01, 30.01, 40.01) consider_boundary_intersection = False ## Only return gemeotries fully covered by the window @@ -379,7 +379,7 @@ Assume you now have a SpatialRDD (typed or generic). You can use the following c === "Python" ```python - from sedona.core.spatialOperator import KNNQuery + from sedona.spark import KNNQuery from shapely.geometry import Point point = Point(-84.01, 34.01) @@ -446,8 +446,8 @@ To utilize a spatial index in a spatial KNN query, use the following code: === "Python" ```python - from sedona.core.spatialOperator import KNNQuery - from sedona.core.enums import IndexType + from sedona.spark import KNNQuery + from sedona.spark import IndexType from shapely.geometry import Point point = Point(-84.01, 34.01) @@ -518,8 +518,8 @@ Assume you now have two SpatialRDDs (typed or generic). You can use the followin === "Python" ```python - from sedona.core.enums import GridType - from sedona.core.spatialOperator import JoinQuery + from sedona.spark import GridType + from sedona.spark import JoinQuery consider_boundary_intersection = False ## Only return geometries fully covered by each query window in queryWindowRDD using_index = False @@ -610,9 +610,9 @@ To utilize a spatial index in a spatial join query, use the following code: === "Python" ```python - from sedona.core.enums import GridType - from sedona.core.enums import IndexType - from sedona.core.spatialOperator import JoinQuery + from sedona.spark import GridType + from sedona.spark import IndexType + from sedona.spark import JoinQuery object_rdd.spatialPartitioning(GridType.KDBTREE) query_window_rdd.spatialPartitioning(object_rdd.getPartitioner()) @@ -676,10 +676,10 @@ The index should be built on either one of two SpatialRDDs. In general, you shou Example: ```python - from sedona.core.SpatialRDD import CircleRDD - from sedona.core.enums import GridType - from sedona.core.spatialOperator import JoinQueryRaw - from sedona.utils.structured_adapter import StructuredAdapter + from sedona.spark import CircleRDD + from sedona.spark import GridType + from sedona.spark import JoinQueryRaw + from sedona.spark import StructuredAdapter object_rdd.analyze() @@ -743,9 +743,9 @@ Assume you now have two SpatialRDDs (typed or generic). You can use the followin === "Python" ```python - from sedona.core.SpatialRDD import CircleRDD - from sedona.core.enums import GridType - from sedona.core.spatialOperator import JoinQuery + from sedona.spark import CircleRDD + from sedona.spark import GridType + from sedona.spark import JoinQuery object_rdd.analyze() diff --git a/docs/tutorial/sql.md b/docs/tutorial/sql.md index 71e8e7d8a16..3cd3b5d734b 100644 --- a/docs/tutorial/sql.md +++ b/docs/tutorial/sql.md @@ -614,7 +614,7 @@ The first parameter is the dataframe, the next two are the epsilon and min_point === "Python" ```python - from sedona.stats.clustering.dbscan import dbscan + from sedona.spark.stats import dbscan dbscan(df, 0.1, 5).show() ``` @@ -670,7 +670,7 @@ The first parameter is the dataframe, the next is the number of nearest neighbor === "Python" ```python - from sedona.stats.outlier_detection.local_outlier_factor import local_outlier_factor + from sedona.spark.stats import local_outlier_factor local_outlier_factor(df, 20).show() ``` @@ -737,8 +737,8 @@ Using Gi involves first generating the neighbors list for each record, then call === "Python" ```python - from sedona.stats.weighting import add_binary_distance_band_column - from sedona.stats.hotspot_detection.getis_ord import g_local + from sedona.spark.stats import add_binary_distance_band_column + from sedona.spark.stats import g_local distance_radius = 1.0 weighted_df = addBinaryDistanceBandColumn(df, distance_radius) @@ -966,7 +966,7 @@ This UDF example takes a geometry type input and returns a primitive type output === "Python" ```python - from sedona.sql.types import GeometryType + from sedona.spark.sql.types import GeometryType from pyspark.sql.types import DoubleType def lengthPoly(geom: GeometryType()): @@ -1025,7 +1025,7 @@ This UDF example takes a geometry type input and returns a geometry type output: === "Python" ```python - from sedona.sql.types import GeometryType + from sedona.spark import GeometryType from pyspark.sql.types import DoubleType def bufferFixed(geom: GeometryType()): @@ -1083,7 +1083,7 @@ This UDF example takes a geometry type input and a primitive type input and retu === "Python" ```python - from sedona.sql.types import GeometryType + from sedona.spark import GeometryType from pyspark.sql.types import DoubleType def bufferIt(geom: GeometryType(), distance: DoubleType()): @@ -1165,7 +1165,7 @@ This UDF example takes a geometry type input and a primitive type input and retu === "Python" ```python - from sedona.sql.types import GeometryType + from sedona.spark import GeometryType from pyspark.sql.types import * schemaUDF = StructType([ @@ -1230,7 +1230,7 @@ a given geometry. ```python import shapely.geometry.base as b -from sedona.sql.functions import sedona_vectorized_udf +from sedona.spark import sedona_vectorized_udf @sedona_vectorized_udf(return_type=GeometryType()) def vectorized_buffer(geom: b.BaseGeometry) -> b.BaseGeometry: @@ -1241,8 +1241,8 @@ def vectorized_buffer(geom: b.BaseGeometry) -> b.BaseGeometry: ```python import geopandas as gpd -from sedona.sql.functions import sedona_vectorized_udf, SedonaUDFType -from sedona.sql.types import GeometryType +from sedona.spark import sedona_vectorized_udf, SedonaUDFType +from sedona.spark import GeometryType @sedona_vectorized_udf(udf_type=SedonaUDFType.GEO_SERIES, return_type=GeometryType()) @@ -1339,7 +1339,7 @@ Use SedonaSQL DataFrame-RDD Adapter to convert a DataFrame to an SpatialRDD. === "Python" ```python - from sedona.utils.structured_adapter import StructuredAdapter + from sedona.spark import StructuredAdapter spatialRDD = StructuredAdapter.toSpatialRdd(spatialDf, "usacounty") ``` @@ -1365,7 +1365,7 @@ Use SedonaSQL DataFrame-RDD Adapter to convert a DataFrame to an SpatialRDD. Ple === "Python" ```python - from sedona.utils.adapter import StructuredAdapter + from sedona.spark import StructuredAdapter spatialDf = StructuredAdapter.toDf(spatialRDD, sedona) ``` @@ -1401,7 +1401,7 @@ You can use `StructuredAdapter` and the `spatialRDD.spatialPartitioningWithoutDu === "Python" ```python - from sedona.utils.structured_adapter import StructuredAdapter + from sedona.spark import StructuredAdapter spatialRDD.spatialPartitioningWithoutDuplicates(GridType.KDBTREE) # Specify the desired number of partitions as 10, though the actual number may vary @@ -1427,7 +1427,7 @@ PairRDD is the result of a spatial join query or distance join query. SedonaSQL === "Python" ```python - from sedona.utils.adapter import StructuredAdapter + from sedona.spark import StructuredAdapter joinResultDf = StructuredAdapter.pairRddToDf(result_pair_rdd, leftDf.schema, rightDf.schema, spark) ``` diff --git a/docs/usecases/ApacheSedonaCore.ipynb b/docs/usecases/ApacheSedonaCore.ipynb index 3f04021844b..50270af244c 100644 --- a/docs/usecases/ApacheSedonaCore.ipynb +++ b/docs/usecases/ApacheSedonaCore.ipynb @@ -40,7 +40,7 @@ "from shapely.geometry import Polygon\n", "\n", "from sedona.spark import *\n", - "from sedona.core.geom.envelope import Envelope" + "from sedona.spark import Envelope" ] }, { @@ -601,9 +601,7 @@ "|testattribute0|testattribute1|testattribute2|POINT (-88.388954 32.357073)|\n", "|testattribute0|testattribute1|testattribute2|POINT (-88.221102 32.35078) |\n", "|testattribute0|testattribute1|testattribute2|POINT (-88.323995 32.950671)|\n", - "+--------------+--------------+--------------+----------------------------+\n", - "only showing top 5 rows\n", - "\n" + "+--------------+--------------+--------------+----------------------------+\n" ] }, { @@ -1207,9 +1205,7 @@ "|POLYGON ((-87.229...|POINT (-87.19351 ...|\n", "|POLYGON ((-87.285...|POINT (-87.28468 ...|\n", "|POLYGON ((-87.285...|POINT (-87.215491...|\n", - "+--------------------+--------------------+\n", - "only showing top 5 rows\n", - "\n" + "+--------------------+--------------------+\n" ] }, { @@ -1246,8 +1242,7 @@ "text": [ "root\n", " |-- geom_left: geometry (nullable = false)\n", - " |-- geom_right: geometry (nullable = false)\n", - "\n" + " |-- geom_right: geometry (nullable = false)\n" ] } ], @@ -1286,9 +1281,7 @@ "|POLYGON ((-87.229...| |POINT (-87.19351 ...| |\n", "|POLYGON ((-87.285...| |POINT (-87.28468 ...| |\n", "|POLYGON ((-87.285...| |POINT (-87.215491...| |\n", - "+--------------------+-----+--------------------+-----+\n", - "only showing top 5 rows\n", - "\n" + "+--------------------+-----+--------------------+-----+\n" ] }, { @@ -1330,8 +1323,7 @@ " |-- geom_1: geometry (nullable = true)\n", " |-- attr1: string (nullable = true)\n", " |-- geom_2: geometry (nullable = true)\n", - " |-- attr2: string (nullable = true)\n", - "\n" + " |-- attr2: string (nullable = true)\n" ] }, { @@ -1437,8 +1429,7 @@ "|POLYGON ((-87.105...| 15|\n", "|POLYGON ((-86.860...| 12|\n", "|POLYGON ((-87.092...| 5|\n", - "+--------------------+----------------+\n", - "\n" + "+--------------------+----------------+\n" ] }, { @@ -1616,7 +1607,7 @@ "metadata": {}, "outputs": [], "source": [ - "from sedona.core.geom.envelope import Envelope" + "from sedona.spark.core.geom.envelope import Envelope" ] }, { @@ -1712,9 +1703,7 @@ "|LINESTRING (-72.4...|\n", "|LINESTRING (-73.4...|\n", "|LINESTRING (-73.6...|\n", - "+--------------------+\n", - "only showing top 5 rows\n", - "\n" + "+--------------------+\n" ] }, { @@ -1804,9 +1793,7 @@ "|MULTIPOLYGON (((1...|\n", "|POLYGON ((118.362...|\n", "|MULTIPOLYGON (((-...|\n", - "+--------------------+\n", - "only showing top 5 rows\n", - "\n" + "+--------------------+\n" ] } ], @@ -1885,9 +1872,7 @@ "|POLYGON ((-86.000...| 01| 055| 001300| 3|1500000US01055001...|010550013003| 3| BG| 1378742|\n", "|POLYGON ((-86.574...| 01| 089| 001700| 2|1500000US01089001...|010890017002| 2| BG| 1040641|\n", "|POLYGON ((-85.382...| 01| 069| 041400| 1|1500000US01069041...|010690414001| 1| BG| 8243574|\n", - "+--------------------+-------+--------+-------+--------+--------------------+------------+----+----+--------+\n", - "only showing top 5 rows\n", - "\n" + "+--------------------+-------+--------+-------+--------+--------------------+------------+----+----+--------+\n" ] }, { @@ -1950,8 +1935,7 @@ "output_type": "stream", "text": [ "root\n", - " |-- geometry: geometry (nullable = true)\n", - "\n" + " |-- geometry: geometry (nullable = true)\n" ] } ], @@ -1983,9 +1967,7 @@ "|POLYGON ((-104.56...|\n", "|POLYGON ((-96.910...|\n", "|POLYGON ((-98.273...|\n", - "+--------------------+\n", - "only showing top 5 rows\n", - "\n" + "+--------------------+\n" ] }, { @@ -2035,9 +2017,7 @@ "|POLYGON ((-104.56...|\n", "|POLYGON ((-96.910...|\n", "|POLYGON ((-98.273...|\n", - "+--------------------+\n", - "only showing top 5 rows\n", - "\n" + "+--------------------+\n" ] }, { @@ -2094,8 +2074,7 @@ "text": [ "root\n", " |-- leftgeometry: geometry (nullable = true)\n", - " |-- rightgeometry: geometry (nullable = true)\n", - "\n" + " |-- rightgeometry: geometry (nullable = true)\n" ] } ], @@ -2127,9 +2106,7 @@ "|POLYGON ((-86.749...|POINT (-86.735506...|\n", "|POLYGON ((-86.749...|POINT (-86.68645 ...|\n", "|POLYGON ((-86.749...|POINT (-86.675405...|\n", - "+--------------------+--------------------+\n", - "only showing top 5 rows\n", - "\n" + "+--------------------+--------------------+\n" ] } ], @@ -2190,9 +2167,7 @@ "|POLYGON ((-86.749...| |POINT (-86.735506...| null|\n", "|POLYGON ((-86.749...| |POINT (-86.68645 ...| null|\n", "|POLYGON ((-86.749...| |POINT (-86.675405...| null|\n", - "+--------------------+--------------+--------------------+---------------+\n", - "only showing top 5 rows\n", - "\n" + "+--------------------+--------------+--------------------+---------------+\n" ] } ], @@ -2247,9 +2222,7 @@ "|LINESTRING (-72.4...|\n", "|LINESTRING (-73.4...|\n", "|LINESTRING (-73.6...|\n", - "+--------------------+\n", - "only showing top 5 rows\n", - "\n" + "+--------------------+\n" ] } ], @@ -2267,8 +2240,7 @@ "output_type": "stream", "text": [ "root\n", - " |-- geometry: geometry (nullable = true)\n", - "\n" + " |-- geometry: geometry (nullable = true)\n" ] } ], @@ -2304,9 +2276,7 @@ "|LINESTRING (-72.4...| |\n", "|LINESTRING (-73.4...| |\n", "|LINESTRING (-73.6...| |\n", - "+--------------------+---+\n", - "only showing top 5 rows\n", - "\n" + "+--------------------+---+\n" ] } ], @@ -2325,8 +2295,7 @@ "text": [ "root\n", " |-- geometry: geometry (nullable = true)\n", - " |-- _c1: string (nullable = true)\n", - "\n" + " |-- _c1: string (nullable = true)\n" ] } ], diff --git a/docs/usecases/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb b/docs/usecases/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb index 1472c2bab34..464bd481de8 100644 --- a/docs/usecases/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb +++ b/docs/usecases/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb @@ -138,7 +138,7 @@ "metadata": {}, "outputs": [], "source": [ - "from sedona.utils.structured_adapter import StructuredAdapter\n", + "from sedona.spark import StructuredAdapter\n", "\n", "airports_rdd = StructuredAdapter.toSpatialRdd(airports_df, \"geometry\")\n", "# Drop the duplicate name column in countries_df\n", diff --git a/docs/usecases/contrib/ApacheSedonaImageFilter.ipynb b/docs/usecases/contrib/ApacheSedonaImageFilter.ipynb index 13d4ea7f72d..2c03dd11f20 100644 --- a/docs/usecases/contrib/ApacheSedonaImageFilter.ipynb +++ b/docs/usecases/contrib/ApacheSedonaImageFilter.ipynb @@ -62,11 +62,11 @@ " ArrayType,\n", ")\n", "from pyspark.sql.functions import regexp_replace\n", - "from sedona.register import SedonaRegistrator\n", - "from sedona.utils import SedonaKryoRegistrator, KryoSerializer\n", + "from sedona.spark import SedonaRegistrator\n", + "from sedona.spark import SedonaKryoRegistrator, KryoSerializer\n", "from pyspark.sql.functions import col, split, expr\n", "from pyspark.sql.functions import udf, lit\n", - "from sedona.utils import SedonaKryoRegistrator, KryoSerializer\n", + "from sedona.spark import SedonaKryoRegistrator, KryoSerializer\n", "from pyspark.sql.functions import col, split, expr\n", "from pyspark.sql.functions import udf, lit" ] @@ -181,8 +181,7 @@ " | |-- width: integer (nullable = true)\n", " | |-- nBands: integer (nullable = true)\n", " | |-- data: array (nullable = true)\n", - " | | |-- element: double (containsNull = true)\n", - "\n" + " | | |-- element: double (containsNull = true)\n" ] } ], diff --git a/docs/usecases/contrib/NdviSentinelApacheSedona.ipynb b/docs/usecases/contrib/NdviSentinelApacheSedona.ipynb index c1614ae75d4..d5594059eea 100644 --- a/docs/usecases/contrib/NdviSentinelApacheSedona.ipynb +++ b/docs/usecases/contrib/NdviSentinelApacheSedona.ipynb @@ -56,11 +56,11 @@ " ArrayType,\n", ")\n", "from pyspark.sql.functions import regexp_replace\n", - "from sedona.register import SedonaRegistrator\n", - "from sedona.utils import SedonaKryoRegistrator, KryoSerializer\n", + "from sedona.spark import SedonaRegistrator\n", + "from sedona.spark import SedonaKryoRegistrator, KryoSerializer\n", "from pyspark.sql.functions import col, split, expr\n", "from pyspark.sql.functions import udf, lit\n", - "from sedona.utils import SedonaKryoRegistrator, KryoSerializer\n", + "from sedona.spark import SedonaKryoRegistrator, KryoSerializer\n", "from pyspark.sql.functions import col, split, expr\n", "from pyspark.sql.functions import udf, lit, flatten\n", "from pywebhdfs.webhdfs import PyWebHdfsClient\n", @@ -237,8 +237,7 @@ " | |-- width: integer (nullable = true)\n", " | |-- nBands: integer (nullable = true)\n", " | |-- data: array (nullable = true)\n", - " | | |-- element: double (containsNull = true)\n", - "\n" + " | | |-- element: double (containsNull = true)\n" ] }, { @@ -269,7 +268,6 @@ "+- InMemoryTableScan [image#14]\n", " +- InMemoryRelation [image#14], StorageLevel(disk, memory, deserialized, 1 replicas)\n", " +- FileScan geotiff [image#14] Batched: false, DataFilters: [], Format: org.apache.spark.sql.sedona_sql.io.GeotiffFileFormat@5f9f8a31, Location: InMemoryFileIndex[hdfs://776faf4d6a1e:8020/sentinel2_tmp/1, hdfs://776faf4d6a1e:8020/sentinel2_tm..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct>>\n", - "\n", "\n" ] }, @@ -292,9 +290,7 @@ "|[hdfs://776faf4d6...| 2|\n", "|[hdfs://776faf4d6...| 3|\n", "|[hdfs://776faf4d6...| 4|\n", - "+--------------------+---+\n", - "only showing top 5 rows\n", - "\n" + "+--------------------+---+\n" ] }, { @@ -331,7 +327,6 @@ " +- InMemoryTableScan [image#14]\n", " +- InMemoryRelation [image#14], StorageLevel(disk, memory, deserialized, 1 replicas)\n", " +- FileScan geotiff [image#14] Batched: false, DataFilters: [], Format: org.apache.spark.sql.sedona_sql.io.GeotiffFileFormat@5f9f8a31, Location: InMemoryFileIndex[hdfs://776faf4d6a1e:8020/sentinel2_tmp/1, hdfs://776faf4d6a1e:8020/sentinel2_tm..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct>>\n", - "\n", "\n" ] } @@ -428,7 +423,6 @@ " +- InMemoryTableScan [image#14]\n", " +- InMemoryRelation [image#14], StorageLevel(disk, memory, deserialized, 1 replicas)\n", " +- FileScan geotiff [image#14] Batched: false, DataFilters: [], Format: org.apache.spark.sql.sedona_sql.io.GeotiffFileFormat@5f9f8a31, Location: InMemoryFileIndex[hdfs://776faf4d6a1e:8020/sentinel2_tmp/1, hdfs://776faf4d6a1e:8020/sentinel2_tm..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct>>\n", - "\n", "\n" ] } @@ -473,9 +467,7 @@ "|hdfs://776faf4d6a...|POLYGON ((-54.274...|[931.0, 971.0, 95...|[1282.0, 1356.0, ...|[0.0, 0.0, 0.0, 0...|[0.0, 0.0, 0.0, 0...|[2.4, 2.4, 2.4, 2...|[2.5, 2.5, 2.5, 2...|[1.0, 1.0, 1.0, 1...|[-0.5, -0.5, -0.5...|[120.0, 120.0, 12...|[0.001, 0.001, 0....|\n", "|hdfs://776faf4d6a...|POLYGON ((-54.274...|[957.0, 995.0, 97...|[1282.0, 1354.0, ...|[0.0, 0.0, 0.0, 0...|[0.0, 0.0, 0.0, 0...|[2.4, 2.4, 2.4, 2...|[2.5, 2.5, 2.5, 2...|[1.0, 1.0, 1.0, 1...|[-0.5, -0.5, -0.5...|[120.0, 120.0, 12...|[0.001, 0.001, 0....|\n", "|hdfs://776faf4d6a...|POLYGON ((-54.274...|[428.0, 428.0, 43...|[880.0, 874.0, 79...|[0.0, 0.0, 0.0, 0...|[0.0, 0.0, 0.0, 0...|[2.4, 2.4, 2.4, 2...|[2.5, 2.5, 2.5, 2...|[1.0, 1.0, 1.0, 1...|[-0.5, -0.5, -0.5...|[120.0, 120.0, 12...|[0.001, 0.001, 0....|\n", - "+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+\n", - "only showing top 5 rows\n", - "\n" + "+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+\n" ] }, { @@ -530,9 +522,7 @@ "|2021-12-26 13:42:12| 3|hdfs://776faf4d6a...|POLYGON ((-54.274...|[931.0, 971.0, 95...|[1282.0, 1356.0, ...|[0.0, 0.0, 0.0, 0...|[0.0, 0.0, 0.0, 0...|[2.4, 2.4, 2.4, 2...|[2.5, 2.5, 2.5, 2...|[1.0, 1.0, 1.0, 1...|[-0.5, -0.5, -0.5...|[120.0, 120.0, 12...|[0.001, 0.001, 0....|\n", "|2021-12-26 13:42:12| 3|hdfs://776faf4d6a...|POLYGON ((-54.274...|[957.0, 995.0, 97...|[1282.0, 1354.0, ...|[0.0, 0.0, 0.0, 0...|[0.0, 0.0, 0.0, 0...|[2.4, 2.4, 2.4, 2...|[2.5, 2.5, 2.5, 2...|[1.0, 1.0, 1.0, 1...|[-0.5, -0.5, -0.5...|[120.0, 120.0, 12...|[0.001, 0.001, 0....|\n", "|2021-12-16 13:42:11| 3|hdfs://776faf4d6a...|POLYGON ((-54.274...|[428.0, 428.0, 43...|[880.0, 874.0, 79...|[0.0, 0.0, 0.0, 0...|[0.0, 0.0, 0.0, 0...|[2.4, 2.4, 2.4, 2...|[2.5, 2.5, 2.5, 2...|[1.0, 1.0, 1.0, 1...|[-0.5, -0.5, -0.5...|[120.0, 120.0, 12...|[0.001, 0.001, 0....|\n", - "+-------------------+------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+\n", - "only showing top 5 rows\n", - "\n" + "+-------------------+------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+\n" ] }, { @@ -662,8 +652,7 @@ " | |-- element: double (containsNull = true)\n", " |-- origin: string (nullable = true)\n", " |-- image_date: timestamp (nullable = true)\n", - " |-- feature_name: string (nullable = true)\n", - "\n" + " |-- feature_name: string (nullable = true)\n" ] }, { @@ -747,8 +736,7 @@ " |-- tgi: double (nullable = false)\n", " |-- origin: string (nullable = true)\n", " |-- image_date: timestamp (nullable = true)\n", - " |-- feature_name: string (nullable = true)\n", - "\n" + " |-- feature_name: string (nullable = true)\n" ] } ], @@ -883,8 +871,7 @@ "| 0.0| 0.0|0.0|-50.85|1.0| 3|(5,[3,4],[-50.85,...|\n", "| 0.0| 0.0|0.0| -23.8|1.0| 3|(5,[3,4],[-23.8,1...|\n", "+----+-----+---+------+---+------+--------------------+\n", - "only showing top 5 rows\n", - "\n" + "only showing top 5 rows\n" ] } ], diff --git a/docs/usecases/contrib/VectorAnalisisApacheSedona.ipynb b/docs/usecases/contrib/VectorAnalisisApacheSedona.ipynb index 98679873952..4042c9f5c1a 100644 --- a/docs/usecases/contrib/VectorAnalisisApacheSedona.ipynb +++ b/docs/usecases/contrib/VectorAnalisisApacheSedona.ipynb @@ -44,11 +44,11 @@ " ArrayType,\n", ")\n", "from pyspark.sql.functions import regexp_replace\n", - "from sedona.register import SedonaRegistrator\n", - "from sedona.utils import SedonaKryoRegistrator, KryoSerializer\n", + "from sedona.spark import SedonaRegistrator\n", + "from sedona.spark import SedonaKryoRegistrator, KryoSerializer\n", "from pyspark.sql.functions import col, split, expr\n", "from pyspark.sql.functions import udf, lit\n", - "from sedona.utils import SedonaKryoRegistrator, KryoSerializer\n", + "from sedona.spark import SedonaKryoRegistrator, KryoSerializer\n", "from pyspark.sql.functions import col, split, expr\n", "from pyspark.sql.functions import udf, lit, flatten\n", "from pywebhdfs.webhdfs import PyWebHdfsClient\n", @@ -314,8 +314,7 @@ " | |-- copyright: string (nullable = true)\n", " | |-- timestamp_areas_base: timestamp (nullable = true)\n", " | |-- timestamp_osm_base: timestamp (nullable = true)\n", - " |-- version: double (nullable = true)\n", - "\n" + " |-- version: double (nullable = true)\n" ] } ], @@ -369,9 +368,7 @@ "|[[[-25.5267745, -...|Overpass API 0.7....|[The data include...| 0.6| 36560|26122631|\n", "|[[[-25.5267745, -...|Overpass API 0.7....|[The data include...| 0.6| 36560|26122645|\n", "|[[[-25.5267745, -...|Overpass API 0.7....|[The data include...| 0.6| 36560|26122801|\n", - "+--------------------+--------------------+--------------------+-------+-----------+--------+\n", - "only showing top 5 rows\n", - "\n" + "+--------------------+--------------------+--------------------+-------+-----------+--------+\n" ] }, { @@ -442,9 +439,7 @@ "|LINESTRING (-25.5...|26122631| 60| null|concrete|Ponte Internacion...|[285975617, 69310...| 36560|\n", "|LINESTRING (-25.6...|26122645| null| null| paved|Rodovia das Catar...|[307420349, 16982...| 36560|\n", "|LINESTRING (-25.5...|26122801| null| null| null| Rua Natal Graciotin|[5514364259, 5514...| 36560|\n", - "+--------------------+--------+--------+-------+--------+--------------------+--------------------+-----------+\n", - "only showing top 5 rows\n", - "\n" + "+--------------------+--------+--------+-------+--------+--------------------+--------------------+-----------+\n" ] } ], @@ -616,9 +611,7 @@ "|[285975617, 69310...| 118.8| 0.00| 0.02| 60| 0%|concrete|Ponte Internacion...|26122631|LINESTRING (-25.5...|1.9538901610218935|0.12057253982300521|0.10941553855472455| 36560|\n", "|[307420349, 16982...| 39.4| 0.00| 0.03| 20| 0%| paved|Rodovia das Catar...|26122645|LINESTRING (-25.6...|11.873439986567648| 0.2619862588434014|0.24791409977758808| 36560|\n", "|[5514364259, 5514...| 39.2| 0.00| 0.04| 20| 0%| soil| Rua Natal Graciotin|26122801|LINESTRING (-25.5...| 6.558866347858073|0.08873043404159503| 0.079155366559698| 36560|\n", - "+--------------------+------+-------------+-------------+--------+-------+--------+--------------------+--------+--------------------+------------------+-------------------+-------------------+-----------+\n", - "only showing top 5 rows\n", - "\n" + "+--------------------+------+-------------+-------------+--------+-------+--------+--------------------+--------+--------------------+------------------+-------------------+-------------------+-----------+\n" ] } ], @@ -687,8 +680,7 @@ "|min(weight)|\n", "+-----------+\n", "| 39.0|\n", - "+-----------+\n", - "\n" + "+-----------+\n" ] } ], @@ -752,8 +744,7 @@ "| id| distance|\n", "+--------+--------------------+\n", "|32347167|8.330276767022637E-5|\n", - "+--------+--------------------+\n", - "\n" + "+--------+--------------------+\n" ] }, { @@ -848,9 +839,7 @@ "|918520467|LINESTRING (-25.4...|\n", "| 32345349|LINESTRING (-25.4...|\n", "|964560305|LINESTRING (-25.4...|\n", - "+---------+--------------------+\n", - "only showing top 5 rows\n", - "\n" + "+---------+--------------------+\n" ] } ], diff --git a/docs/usecases/contrib/foot-traffic.ipynb b/docs/usecases/contrib/foot-traffic.ipynb index 200da0dc729..f1ee331c3a0 100644 --- a/docs/usecases/contrib/foot-traffic.ipynb +++ b/docs/usecases/contrib/foot-traffic.ipynb @@ -111,8 +111,8 @@ ], "source": [ "from pyspark.sql import SparkSession\n", - "from sedona.register import SedonaRegistrator\n", - "from sedona.utils import SedonaKryoRegistrator, KryoSerializer\n", + "from sedona.spark import SedonaRegistrator\n", + "from sedona.spark import SedonaKryoRegistrator, KryoSerializer\n", "\n", "spark = (\n", " SparkSession.builder.appName(\"sigspatial2021\")\n", @@ -794,9 +794,7 @@ "|zzw-22d@5x4-4b7-bkz|Eltana |305 Harrison St |294.919 |\n", "|zzw-224@5x4-4b5-f75|MarketSpice |85A Pike St |263.49886363636364 |\n", "|22b-222@5x4-4vs-qzz|Cherry Street Coffee House|700 1st Ave |210.19386666666668 |\n", - "+-------------------+--------------------------+-----------------------------------+-------------------------+\n", - "only showing top 20 rows\n", - "\n" + "+-------------------+--------------------------+-----------------------------------+-------------------------+\n" ] }, { @@ -898,8 +896,8 @@ "# but `distance_from_home` takes into account ALL visitors to the coffee shop, which as we saw above can distort values. When selecting a site for a coffee shop, we likely care more about how far visitors traveled from within Seattle\n", "# we can compute this using Sedona\n", "import sedona\n", - "from sedona.register import SedonaRegistrator\n", - "from sedona.utils import SedonaKryoRegistrator, KryoSerializer\n", + "from sedona.spark import SedonaRegistrator\n", + "from sedona.spark import SedonaKryoRegistrator, KryoSerializer\n", "\n", "SedonaRegistrator.registerAll(spark)" ] diff --git a/python/sedona/spark/SedonaContext.py b/python/sedona/spark/SedonaContext.py index 49db2e47aab..5a8dc97330f 100644 --- a/python/sedona/spark/SedonaContext.py +++ b/python/sedona/spark/SedonaContext.py @@ -18,8 +18,8 @@ import attr from pyspark.sql import SparkSession -from sedona.register.geo_registrator import PackageImporter -from sedona.utils import KryoSerializer, SedonaKryoRegistrator +from sedona.spark.register.geo_registrator import PackageImporter +from sedona.spark.utils import KryoSerializer, SedonaKryoRegistrator try: from pyspark.sql.utils import is_remote diff --git a/python/sedona/spark/__init__.py b/python/sedona/spark/__init__.py index 8f4c4f24ed3..15358904db2 100644 --- a/python/sedona/spark/__init__.py +++ b/python/sedona/spark/__init__.py @@ -14,18 +14,27 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from sedona.spark.core import Envelope +from sedona.spark.sql.functions import sedona_vectorized_udf -from sedona.core.enums import FileDataSplitter, GridType, IndexType -from sedona.core.formatMapper import GeoJsonReader, WkbReader, WktReader -from sedona.core.formatMapper.shapefileParser import ShapefileReader -from sedona.core.spatialOperator import ( +try: + import pyspark +except ImportError: + raise ImportError( + "Apache Sedona requires PySpark. Please install PySpark before using Sedona spark." + ) + +from sedona.spark.core.enums import FileDataSplitter, GridType, IndexType +from sedona.spark.core.formatMapper import GeoJsonReader, WkbReader, WktReader +from sedona.spark.core.formatMapper.shapefileParser import ShapefileReader +from sedona.spark.core.spatialOperator import ( JoinQuery, JoinQueryRaw, KNNQuery, RangeQuery, RangeQueryRaw, ) -from sedona.core.SpatialRDD import ( +from sedona.spark.core.SpatialRDD import ( CircleRDD, LineStringRDD, PointRDD, @@ -33,16 +42,17 @@ RectangleRDD, SpatialRDD, ) -from sedona.maps.SedonaKepler import SedonaKepler -from sedona.maps.SedonaPyDeck import SedonaPyDeck -from sedona.raster_utils.SedonaUtils import SedonaUtils -from sedona.register import SedonaRegistrator +from sedona.spark.maps.SedonaKepler import SedonaKepler +from sedona.spark.maps.SedonaPyDeck import SedonaPyDeck +from sedona.spark.raster_utils.SedonaUtils import SedonaUtils +from sedona.spark.register import SedonaRegistrator from sedona.spark.SedonaContext import SedonaContext -from sedona.sql.st_aggregates import * -from sedona.sql.st_constructors import * -from sedona.sql.st_functions import * -from sedona.sql.st_predicates import * -from sedona.sql.types import GeometryType, GeographyType, RasterType -from sedona.utils import KryoSerializer, SedonaKryoRegistrator -from sedona.utils.adapter import Adapter -from sedona.utils.geoarrow import dataframe_to_arrow +from sedona.spark.sql.st_aggregates import * +from sedona.spark.sql.st_constructors import * +from sedona.spark.sql.st_functions import * +from sedona.spark.sql.st_predicates import * +from sedona.spark.sql.types import GeometryType, GeographyType, RasterType +from sedona.spark.utils import KryoSerializer, SedonaKryoRegistrator +from sedona.spark.utils.adapter import Adapter +from sedona.spark.utils.structured_adapter import StructuredAdapter +from sedona.spark.geoarrow.geoarrow import dataframe_to_arrow diff --git a/python/sedona/core/SpatialRDD/__init__.py b/python/sedona/spark/core/SpatialRDD/__init__.py similarity index 100% rename from python/sedona/core/SpatialRDD/__init__.py rename to python/sedona/spark/core/SpatialRDD/__init__.py diff --git a/python/sedona/core/SpatialRDD/circle_rdd.py b/python/sedona/spark/core/SpatialRDD/circle_rdd.py similarity index 84% rename from python/sedona/core/SpatialRDD/circle_rdd.py rename to python/sedona/spark/core/SpatialRDD/circle_rdd.py index 48142a52a85..6cd13a10007 100644 --- a/python/sedona/core/SpatialRDD/circle_rdd.py +++ b/python/sedona/spark/core/SpatialRDD/circle_rdd.py @@ -15,9 +15,9 @@ # specific language governing permissions and limitations # under the License. -from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD -from sedona.core.SpatialRDD.spatial_rdd_factory import SpatialRDDFactory -from sedona.utils.meta import MultipleMeta +from sedona.spark.core.SpatialRDD.spatial_rdd import SpatialRDD +from sedona.spark.core.SpatialRDD.spatial_rdd_factory import SpatialRDDFactory +from sedona.spark.utils.meta import MultipleMeta class CircleRDD(SpatialRDD, metaclass=MultipleMeta): @@ -32,7 +32,7 @@ def __init__(self, spatialRDD: SpatialRDD, Radius: float): self._srdd = self._jvm_spatial_rdd(spatialRDD._srdd, Radius) def getCenterPointAsSpatialRDD(self) -> "PointRDD": - from sedona.core.SpatialRDD import PointRDD + from sedona.spark.core.SpatialRDD import PointRDD srdd = self._srdd.getCenterPointAsSpatialRDD() point_rdd = PointRDD() @@ -40,7 +40,7 @@ def getCenterPointAsSpatialRDD(self) -> "PointRDD": return point_rdd def getCenterPolygonAsSpatialRDD(self) -> "PolygonRDD": - from sedona.core.SpatialRDD import PolygonRDD + from sedona.spark.core.SpatialRDD import PolygonRDD srdd = self._srdd.getCenterPolygonAsSpatialRDD() polygon_rdd = PolygonRDD() @@ -48,7 +48,7 @@ def getCenterPolygonAsSpatialRDD(self) -> "PolygonRDD": return polygon_rdd def getCenterLineStringRDDAsSpatialRDD(self) -> "LineStringRDD": - from sedona.core.SpatialRDD import LineStringRDD + from sedona.spark.core.SpatialRDD import LineStringRDD srdd = self._srdd.getCenterPolygonAsSpatialRDD() linestring_rdd = LineStringRDD() @@ -56,7 +56,7 @@ def getCenterLineStringRDDAsSpatialRDD(self) -> "LineStringRDD": return linestring_rdd def getCenterRectangleRDDAsSpatialRDD(self) -> "RectangleRDD": - from sedona.core.SpatialRDD import RectangleRDD + from sedona.spark.core.SpatialRDD import RectangleRDD srdd = self._srdd.getCenterLineStringRDDAsSpatialRDD() rectangle_rdd = RectangleRDD() diff --git a/python/sedona/core/SpatialRDD/linestring_rdd.py b/python/sedona/spark/core/SpatialRDD/linestring_rdd.py similarity index 91% rename from python/sedona/core/SpatialRDD/linestring_rdd.py rename to python/sedona/spark/core/SpatialRDD/linestring_rdd.py index fcdf9450ec4..7d5e55fdda5 100644 --- a/python/sedona/core/SpatialRDD/linestring_rdd.py +++ b/python/sedona/spark/core/SpatialRDD/linestring_rdd.py @@ -15,14 +15,14 @@ # specific language governing permissions and limitations # under the License. -from pyspark import RDD, SparkContext, StorageLevel +from pyspark import RDD, SparkContext -from sedona.core.enums import FileDataSplitter -from sedona.core.enums.file_data_splitter import FileSplitterJvm -from sedona.core.jvm.translate import PythonRddToJavaRDDAdapter -from sedona.core.SpatialRDD.spatial_rdd import JvmSpatialRDD, SpatialRDD -from sedona.core.SpatialRDD.spatial_rdd_factory import SpatialRDDFactory -from sedona.utils.meta import MultipleMeta +from sedona.spark.core.enums import FileDataSplitter +from sedona.spark.core.enums.file_data_splitter import FileSplitterJvm +from sedona.spark.core.jvm.translate import PythonRddToJavaRDDAdapter +from sedona.spark.core.SpatialRDD.spatial_rdd import JvmSpatialRDD, SpatialRDD +from sedona.spark.core.SpatialRDD.spatial_rdd_factory import SpatialRDDFactory +from sedona.spark.utils.meta import MultipleMeta class LineStringRDD(SpatialRDD, metaclass=MultipleMeta): @@ -166,7 +166,7 @@ def _jvm_spatial_rdd(self): return spatial_factory.create_linestring_rdd() def MinimumBoundingRectangle(self): - from sedona.core.SpatialRDD import RectangleRDD + from sedona.spark.core.SpatialRDD import RectangleRDD rectangle_rdd = RectangleRDD() srdd = self._srdd.MinimumBoundingRectangle() diff --git a/python/sedona/core/SpatialRDD/point_rdd.py b/python/sedona/spark/core/SpatialRDD/point_rdd.py similarity index 92% rename from python/sedona/core/SpatialRDD/point_rdd.py rename to python/sedona/spark/core/SpatialRDD/point_rdd.py index 54205cad60c..fade34efec4 100644 --- a/python/sedona/core/SpatialRDD/point_rdd.py +++ b/python/sedona/spark/core/SpatialRDD/point_rdd.py @@ -15,13 +15,13 @@ # specific language governing permissions and limitations # under the License. -from pyspark import RDD, SparkContext, StorageLevel +from pyspark import RDD, SparkContext -from sedona.core.enums.file_data_splitter import FileDataSplitter, FileSplitterJvm -from sedona.core.jvm.translate import PythonRddToJavaRDDAdapter -from sedona.core.SpatialRDD.spatial_rdd import JvmSpatialRDD, SpatialRDD -from sedona.core.SpatialRDD.spatial_rdd_factory import SpatialRDDFactory -from sedona.utils.meta import MultipleMeta +from sedona.spark.core.enums.file_data_splitter import FileDataSplitter, FileSplitterJvm +from sedona.spark.core.jvm.translate import PythonRddToJavaRDDAdapter +from sedona.spark.core.SpatialRDD.spatial_rdd import JvmSpatialRDD, SpatialRDD +from sedona.spark.core.SpatialRDD.spatial_rdd_factory import SpatialRDDFactory +from sedona.spark.utils.meta import MultipleMeta class PointRDD(SpatialRDD, metaclass=MultipleMeta): diff --git a/python/sedona/core/SpatialRDD/polygon_rdd.py b/python/sedona/spark/core/SpatialRDD/polygon_rdd.py similarity index 91% rename from python/sedona/core/SpatialRDD/polygon_rdd.py rename to python/sedona/spark/core/SpatialRDD/polygon_rdd.py index ab6dfa46378..e4f7ae40208 100644 --- a/python/sedona/core/SpatialRDD/polygon_rdd.py +++ b/python/sedona/spark/core/SpatialRDD/polygon_rdd.py @@ -15,13 +15,13 @@ # specific language governing permissions and limitations # under the License. -from pyspark import RDD, SparkContext, StorageLevel +from pyspark import RDD, SparkContext -from sedona.core.enums.file_data_splitter import FileDataSplitter, FileSplitterJvm -from sedona.core.jvm.translate import PythonRddToJavaRDDAdapter -from sedona.core.SpatialRDD.spatial_rdd import JvmSpatialRDD, SpatialRDD -from sedona.core.SpatialRDD.spatial_rdd_factory import SpatialRDDFactory -from sedona.utils.meta import MultipleMeta +from sedona.spark.core.enums.file_data_splitter import FileDataSplitter, FileSplitterJvm +from sedona.spark.core.jvm.translate import PythonRddToJavaRDDAdapter +from sedona.spark.core.SpatialRDD.spatial_rdd import JvmSpatialRDD, SpatialRDD +from sedona.spark.core.SpatialRDD.spatial_rdd_factory import SpatialRDDFactory +from sedona.spark.utils.meta import MultipleMeta class PolygonRDD(SpatialRDD, metaclass=MultipleMeta): @@ -161,7 +161,7 @@ def __init__( ) def MinimumBoundingRectangle(self): - from sedona.core.SpatialRDD import RectangleRDD + from sedona.spark.core.SpatialRDD import RectangleRDD rectangle_rdd = RectangleRDD() srdd = self._srdd.MinimumBoundingRectangle() diff --git a/python/sedona/core/SpatialRDD/rectangle_rdd.py b/python/sedona/spark/core/SpatialRDD/rectangle_rdd.py similarity index 92% rename from python/sedona/core/SpatialRDD/rectangle_rdd.py rename to python/sedona/spark/core/SpatialRDD/rectangle_rdd.py index 3d8e457ef27..397cdee4534 100644 --- a/python/sedona/core/SpatialRDD/rectangle_rdd.py +++ b/python/sedona/spark/core/SpatialRDD/rectangle_rdd.py @@ -15,13 +15,12 @@ # specific language governing permissions and limitations # under the License. -from pyspark import RDD, SparkContext, StorageLevel +from pyspark import SparkContext -from sedona.core.enums.file_data_splitter import FileDataSplitter, FileSplitterJvm -from sedona.core.SpatialRDD.spatial_rdd import JvmSpatialRDD, SpatialRDD -from sedona.core.SpatialRDD.spatial_rdd_factory import SpatialRDDFactory -from sedona.utils.jvm import JvmStorageLevel -from sedona.utils.meta import MultipleMeta +from sedona.spark.core.enums.file_data_splitter import FileDataSplitter, FileSplitterJvm +from sedona.spark.core.SpatialRDD.spatial_rdd import JvmSpatialRDD, SpatialRDD +from sedona.spark.core.SpatialRDD.spatial_rdd_factory import SpatialRDDFactory +from sedona.spark.utils.meta import MultipleMeta class RectangleRDD(SpatialRDD, metaclass=MultipleMeta): diff --git a/python/sedona/core/SpatialRDD/spatial_rdd.py b/python/sedona/spark/core/SpatialRDD/spatial_rdd.py similarity index 95% rename from python/sedona/core/SpatialRDD/spatial_rdd.py rename to python/sedona/spark/core/SpatialRDD/spatial_rdd.py index 266676c0d51..f091c490030 100644 --- a/python/sedona/core/SpatialRDD/spatial_rdd.py +++ b/python/sedona/spark/core/SpatialRDD/spatial_rdd.py @@ -15,7 +15,6 @@ # specific language governing permissions and limitations # under the License. -import pickle from typing import List, Optional, Union import attr @@ -23,16 +22,19 @@ from pyspark import RDD, SparkContext, StorageLevel from pyspark.sql import SparkSession -from sedona.core.enums.grid_type import GridType, GridTypeJvm -from sedona.core.enums.index_type import IndexType, IndexTypeJvm -from sedona.core.enums.spatial import SpatialType -from sedona.core.geom.envelope import Envelope -from sedona.core.jvm.translate import JvmSedonaPythonConverter, SedonaPythonConverter -from sedona.core.SpatialRDD.spatial_rdd_factory import SpatialRDDFactory -from sedona.utils.decorators import require -from sedona.utils.jvm import JvmStorageLevel -from sedona.utils.spatial_rdd_parser import SedonaPickler -from sedona.utils.types import crs +from sedona.spark.core.enums.grid_type import GridType, GridTypeJvm +from sedona.spark.core.enums.index_type import IndexType, IndexTypeJvm +from sedona.spark.core.enums.spatial import SpatialType +from sedona.spark.core.geom.envelope import Envelope +from sedona.spark.core.jvm.translate import ( + JvmSedonaPythonConverter, + SedonaPythonConverter, +) +from sedona.spark.core.SpatialRDD.spatial_rdd_factory import SpatialRDDFactory +from sedona.spark.utils.decorators import require +from sedona.spark.utils.jvm import JvmStorageLevel +from sedona.spark.utils.spatial_rdd_parser import SedonaPickler +from sedona.spark.utils.types import crs @attr.s @@ -404,6 +406,8 @@ def setSampleNumber(self, sampleNumber: int) -> bool: @property def spatialPartitionedRDD(self): + from sedona.spark.utils.spatial_rdd_parser import SedonaPickler + """ :return: diff --git a/python/sedona/core/SpatialRDD/spatial_rdd_factory.py b/python/sedona/spark/core/SpatialRDD/spatial_rdd_factory.py similarity index 97% rename from python/sedona/core/SpatialRDD/spatial_rdd_factory.py rename to python/sedona/spark/core/SpatialRDD/spatial_rdd_factory.py index f44611ea37f..ad504ab9e51 100644 --- a/python/sedona/core/SpatialRDD/spatial_rdd_factory.py +++ b/python/sedona/spark/core/SpatialRDD/spatial_rdd_factory.py @@ -20,7 +20,7 @@ import attr from pyspark import SparkContext -from sedona.utils.decorators import require +from sedona.spark.utils.decorators import require @attr.s diff --git a/python/sedona/spark/core/__init__.py b/python/sedona/spark/core/__init__.py new file mode 100644 index 00000000000..54cae81f096 --- /dev/null +++ b/python/sedona/spark/core/__init__.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from sedona.spark.core.geom.envelope import Envelope +from sedona.spark.core.spatialOperator import RangeQueryRaw + +__all__ = ["Envelope", "RangeQueryRaw"] diff --git a/python/sedona/core/enums/__init__.py b/python/sedona/spark/core/enums/__init__.py similarity index 100% rename from python/sedona/core/enums/__init__.py rename to python/sedona/spark/core/enums/__init__.py diff --git a/python/sedona/core/enums/file_data_splitter.py b/python/sedona/spark/core/enums/file_data_splitter.py similarity index 94% rename from python/sedona/core/enums/file_data_splitter.py rename to python/sedona/spark/core/enums/file_data_splitter.py index 81b402b3f37..a2470922909 100644 --- a/python/sedona/core/enums/file_data_splitter.py +++ b/python/sedona/spark/core/enums/file_data_splitter.py @@ -19,8 +19,8 @@ import attr -from sedona.core.jvm.abstract import JvmObject -from sedona.utils.decorators import require +from sedona.spark.core.jvm.abstract import JvmObject +from sedona.spark.utils.decorators import require class FileDataSplitter(Enum): diff --git a/python/sedona/core/enums/grid_type.py b/python/sedona/spark/core/enums/grid_type.py similarity index 93% rename from python/sedona/core/enums/grid_type.py rename to python/sedona/spark/core/enums/grid_type.py index 89deeee9648..8d43c44737e 100644 --- a/python/sedona/core/enums/grid_type.py +++ b/python/sedona/spark/core/enums/grid_type.py @@ -19,8 +19,8 @@ import attr -from sedona.core.jvm.abstract import JvmObject -from sedona.utils.decorators import require +from sedona.spark.core.jvm.abstract import JvmObject +from sedona.spark.utils.decorators import require class GridType(Enum): diff --git a/python/sedona/core/enums/index_type.py b/python/sedona/spark/core/enums/index_type.py similarity index 93% rename from python/sedona/core/enums/index_type.py rename to python/sedona/spark/core/enums/index_type.py index 3a00004d84d..dc29d696a2b 100644 --- a/python/sedona/core/enums/index_type.py +++ b/python/sedona/spark/core/enums/index_type.py @@ -19,8 +19,8 @@ import attr -from sedona.core.jvm.abstract import JvmObject -from sedona.utils.decorators import require +from sedona.spark.core.jvm.abstract import JvmObject +from sedona.spark.utils.decorators import require class IndexType(Enum): diff --git a/python/sedona/core/enums/join_build_side.py b/python/sedona/spark/core/enums/join_build_side.py similarity index 94% rename from python/sedona/core/enums/join_build_side.py rename to python/sedona/spark/core/enums/join_build_side.py index f79c1596118..d0c5ed63126 100644 --- a/python/sedona/core/enums/join_build_side.py +++ b/python/sedona/spark/core/enums/join_build_side.py @@ -14,8 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - -from sedona.utils.decorators import classproperty +from sedona.spark.utils.decorators import classproperty class JoinBuildSide: diff --git a/python/sedona/core/enums/spatial.py b/python/sedona/spark/core/enums/spatial.py similarity index 100% rename from python/sedona/core/enums/spatial.py rename to python/sedona/spark/core/enums/spatial.py diff --git a/python/sedona/core/formatMapper/__init__.py b/python/sedona/spark/core/formatMapper/__init__.py similarity index 100% rename from python/sedona/core/formatMapper/__init__.py rename to python/sedona/spark/core/formatMapper/__init__.py diff --git a/python/sedona/core/formatMapper/disc_utils.py b/python/sedona/spark/core/formatMapper/disc_utils.py similarity index 94% rename from python/sedona/core/formatMapper/disc_utils.py rename to python/sedona/spark/core/formatMapper/disc_utils.py index e5b8256d35d..459e864ad44 100644 --- a/python/sedona/core/formatMapper/disc_utils.py +++ b/python/sedona/spark/core/formatMapper/disc_utils.py @@ -19,9 +19,9 @@ from pyspark import SparkContext -from sedona.core.jvm.translate import SpatialObjectLoaderAdapter -from sedona.core.SpatialRDD import LineStringRDD, PointRDD, PolygonRDD, SpatialRDD -from sedona.utils.decorators import require +from sedona.spark.core.jvm.translate import SpatialObjectLoaderAdapter +from sedona.spark.core.SpatialRDD import LineStringRDD, PointRDD, PolygonRDD, SpatialRDD +from sedona.spark.utils.decorators import require class DiscLoader: diff --git a/python/sedona/core/formatMapper/geo_json_reader.py b/python/sedona/spark/core/formatMapper/geo_json_reader.py similarity index 94% rename from python/sedona/core/formatMapper/geo_json_reader.py rename to python/sedona/spark/core/formatMapper/geo_json_reader.py index 7b1784f0cf0..23a090cd0ca 100644 --- a/python/sedona/core/formatMapper/geo_json_reader.py +++ b/python/sedona/spark/core/formatMapper/geo_json_reader.py @@ -17,9 +17,9 @@ from pyspark import RDD, SparkContext -from sedona.core.formatMapper.geo_reader import GeoDataReader -from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD -from sedona.utils.meta import MultipleMeta +from sedona.spark.core.formatMapper.geo_reader import GeoDataReader +from sedona.spark.core.SpatialRDD.spatial_rdd import SpatialRDD +from sedona.spark.utils.meta import MultipleMeta class GeoJsonReader(GeoDataReader, metaclass=MultipleMeta): diff --git a/python/sedona/core/formatMapper/geo_reader.py b/python/sedona/spark/core/formatMapper/geo_reader.py similarity index 95% rename from python/sedona/core/formatMapper/geo_reader.py rename to python/sedona/spark/core/formatMapper/geo_reader.py index 1cfb373a979..96fdee509f0 100644 --- a/python/sedona/core/formatMapper/geo_reader.py +++ b/python/sedona/spark/core/formatMapper/geo_reader.py @@ -19,7 +19,7 @@ import attr -from sedona.utils.meta import MultipleMeta +from sedona.spark.utils.meta import MultipleMeta @attr.s diff --git a/python/sedona/core/formatMapper/shapefileParser/__init__.py b/python/sedona/spark/core/formatMapper/shapefileParser/__init__.py similarity index 100% rename from python/sedona/core/formatMapper/shapefileParser/__init__.py rename to python/sedona/spark/core/formatMapper/shapefileParser/__init__.py diff --git a/python/sedona/core/formatMapper/shapefileParser/shape_file_reader.py b/python/sedona/spark/core/formatMapper/shapefileParser/shape_file_reader.py similarity index 90% rename from python/sedona/core/formatMapper/shapefileParser/shape_file_reader.py rename to python/sedona/spark/core/formatMapper/shapefileParser/shape_file_reader.py index 4c456096552..936ef5da22f 100644 --- a/python/sedona/core/formatMapper/shapefileParser/shape_file_reader.py +++ b/python/sedona/spark/core/formatMapper/shapefileParser/shape_file_reader.py @@ -18,10 +18,10 @@ import attr from pyspark import SparkContext -from sedona.core.formatMapper.geo_reader import GeoDataReader -from sedona.core.SpatialRDD import LineStringRDD, PointRDD, PolygonRDD -from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD -from sedona.utils.meta import MultipleMeta +from sedona.spark.core.formatMapper.geo_reader import GeoDataReader +from sedona.spark.core.SpatialRDD import LineStringRDD, PointRDD, PolygonRDD +from sedona.spark.core.SpatialRDD.spatial_rdd import SpatialRDD +from sedona.spark.utils.meta import MultipleMeta @attr.s diff --git a/python/sedona/core/formatMapper/wkb_reader.py b/python/sedona/spark/core/formatMapper/wkb_reader.py similarity index 93% rename from python/sedona/core/formatMapper/wkb_reader.py rename to python/sedona/spark/core/formatMapper/wkb_reader.py index 45bbc5acdb3..2fd373b0966 100644 --- a/python/sedona/core/formatMapper/wkb_reader.py +++ b/python/sedona/spark/core/formatMapper/wkb_reader.py @@ -17,9 +17,9 @@ from pyspark import RDD, SparkContext -from sedona.core.formatMapper.geo_reader import GeoDataReader -from sedona.core.SpatialRDD import SpatialRDD -from sedona.utils.meta import MultipleMeta +from sedona.spark.core.formatMapper.geo_reader import GeoDataReader +from sedona.spark.core.SpatialRDD import SpatialRDD +from sedona.spark.utils.meta import MultipleMeta class WkbReader(GeoDataReader, metaclass=MultipleMeta): diff --git a/python/sedona/core/formatMapper/wkt_reader.py b/python/sedona/spark/core/formatMapper/wkt_reader.py similarity index 93% rename from python/sedona/core/formatMapper/wkt_reader.py rename to python/sedona/spark/core/formatMapper/wkt_reader.py index 7d5689eceea..6e8ac588d78 100644 --- a/python/sedona/core/formatMapper/wkt_reader.py +++ b/python/sedona/spark/core/formatMapper/wkt_reader.py @@ -17,9 +17,9 @@ from pyspark import RDD, SparkContext -from sedona.core.formatMapper.geo_reader import GeoDataReader -from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD -from sedona.utils.meta import MultipleMeta +from sedona.spark.core.formatMapper.geo_reader import GeoDataReader +from sedona.spark.core.SpatialRDD.spatial_rdd import SpatialRDD +from sedona.spark.utils.meta import MultipleMeta class WktReader(GeoDataReader, metaclass=MultipleMeta): diff --git a/python/sedona/core/__init__.py b/python/sedona/spark/core/geom/__init__.py similarity index 100% rename from python/sedona/core/__init__.py rename to python/sedona/spark/core/geom/__init__.py diff --git a/python/sedona/core/geom/circle.py b/python/sedona/spark/core/geom/circle.py similarity index 100% rename from python/sedona/core/geom/circle.py rename to python/sedona/spark/core/geom/circle.py diff --git a/python/sedona/core/geom/envelope.py b/python/sedona/spark/core/geom/envelope.py similarity index 100% rename from python/sedona/core/geom/envelope.py rename to python/sedona/spark/core/geom/envelope.py diff --git a/python/sedona/core/geom/geography.py b/python/sedona/spark/core/geom/geography.py similarity index 100% rename from python/sedona/core/geom/geography.py rename to python/sedona/spark/core/geom/geography.py diff --git a/python/sedona/core/geom/__init__.py b/python/sedona/spark/core/geom/shapely1/__init__.py similarity index 100% rename from python/sedona/core/geom/__init__.py rename to python/sedona/spark/core/geom/shapely1/__init__.py diff --git a/python/sedona/core/geom/shapely1/circle.py b/python/sedona/spark/core/geom/shapely1/circle.py similarity index 98% rename from python/sedona/core/geom/shapely1/circle.py rename to python/sedona/spark/core/geom/shapely1/circle.py index f227e748c71..bce29c703a6 100644 --- a/python/sedona/core/geom/shapely1/circle.py +++ b/python/sedona/spark/core/geom/shapely1/circle.py @@ -27,7 +27,7 @@ ) from shapely.geometry.base import BaseGeometry -from sedona.core.geom.envelope import Envelope +from sedona.spark.core.geom.envelope import Envelope class Circle(Polygon): diff --git a/python/sedona/core/geom/shapely1/envelope.py b/python/sedona/spark/core/geom/shapely1/envelope.py similarity index 97% rename from python/sedona/core/geom/shapely1/envelope.py rename to python/sedona/spark/core/geom/shapely1/envelope.py index df49e615054..20fa45ff96a 100644 --- a/python/sedona/core/geom/shapely1/envelope.py +++ b/python/sedona/spark/core/geom/shapely1/envelope.py @@ -18,11 +18,10 @@ import math import pickle +from sedona.spark.utils.decorators import require from shapely.geometry import Point, Polygon from shapely.geometry.base import BaseGeometry -from sedona.utils.decorators import require - class Envelope(Polygon): @@ -65,7 +64,7 @@ def from_jvm_instance(cls, java_obj): ) def to_bytes(self): - from sedona.utils.binary_parser import BinaryBuffer + from sedona.spark.utils.binary_parser import BinaryBuffer bin_buffer = BinaryBuffer() bin_buffer.put_double(self.minx) diff --git a/python/sedona/core/geom/shapely1/__init__.py b/python/sedona/spark/core/geom/shapely2/__init__.py similarity index 100% rename from python/sedona/core/geom/shapely1/__init__.py rename to python/sedona/spark/core/geom/shapely2/__init__.py diff --git a/python/sedona/core/geom/shapely2/circle.py b/python/sedona/spark/core/geom/shapely2/circle.py similarity index 98% rename from python/sedona/core/geom/shapely2/circle.py rename to python/sedona/spark/core/geom/shapely2/circle.py index 859d1ef3b29..9d916825543 100644 --- a/python/sedona/core/geom/shapely2/circle.py +++ b/python/sedona/spark/core/geom/shapely2/circle.py @@ -24,11 +24,10 @@ MultiPolygon, Point, Polygon, - box, ) from shapely.geometry.base import BaseGeometry -from sedona.core.geom.envelope import Envelope +from sedona.spark.core.geom.envelope import Envelope class Circle(Polygon): diff --git a/python/sedona/core/geom/shapely2/envelope.py b/python/sedona/spark/core/geom/shapely2/envelope.py similarity index 97% rename from python/sedona/core/geom/shapely2/envelope.py rename to python/sedona/spark/core/geom/shapely2/envelope.py index 6a5aeb0787e..3e17f826edc 100644 --- a/python/sedona/core/geom/shapely2/envelope.py +++ b/python/sedona/spark/core/geom/shapely2/envelope.py @@ -21,7 +21,7 @@ from shapely.geometry import Polygon, box from shapely.geometry.base import BaseGeometry -from sedona.utils.decorators import require +from sedona.spark.utils.decorators import require class Envelope(Polygon): @@ -77,7 +77,7 @@ def from_jvm_instance(cls, java_obj): ) def to_bytes(self): - from sedona.utils.binary_parser import BinaryBuffer + from sedona.spark.utils import BinaryBuffer minx, miny, maxx, maxy = self.bounds bin_buffer = BinaryBuffer() diff --git a/python/sedona/core/geom/shapely2/__init__.py b/python/sedona/spark/core/jvm/__init__.py similarity index 100% rename from python/sedona/core/geom/shapely2/__init__.py rename to python/sedona/spark/core/jvm/__init__.py diff --git a/python/sedona/core/jvm/abstract.py b/python/sedona/spark/core/jvm/abstract.py similarity index 100% rename from python/sedona/core/jvm/abstract.py rename to python/sedona/spark/core/jvm/abstract.py diff --git a/python/sedona/core/jvm/config.py b/python/sedona/spark/core/jvm/config.py similarity index 99% rename from python/sedona/core/jvm/config.py rename to python/sedona/spark/core/jvm/config.py index 09650ed51bd..fdbc371e4c0 100644 --- a/python/sedona/core/jvm/config.py +++ b/python/sedona/spark/core/jvm/config.py @@ -26,7 +26,7 @@ from py4j.protocol import Py4JJavaError from pyspark.sql import SparkSession -from sedona.utils.decorators import classproperty +from sedona.spark.utils.decorators import classproperty string_types = (bytes, str) diff --git a/python/sedona/core/jvm/translate.py b/python/sedona/spark/core/jvm/translate.py similarity index 100% rename from python/sedona/core/jvm/translate.py rename to python/sedona/spark/core/jvm/translate.py diff --git a/python/sedona/core/spatialOperator/__init__.py b/python/sedona/spark/core/spatialOperator/__init__.py similarity index 100% rename from python/sedona/core/spatialOperator/__init__.py rename to python/sedona/spark/core/spatialOperator/__init__.py diff --git a/python/sedona/core/spatialOperator/join_params.py b/python/sedona/spark/core/spatialOperator/join_params.py similarity index 92% rename from python/sedona/core/spatialOperator/join_params.py rename to python/sedona/spark/core/spatialOperator/join_params.py index c77224af7f7..f22177af3cb 100644 --- a/python/sedona/core/spatialOperator/join_params.py +++ b/python/sedona/spark/core/spatialOperator/join_params.py @@ -17,9 +17,9 @@ import attr -from sedona.core.enums import IndexType -from sedona.core.enums.join_build_side import JoinBuildSide -from sedona.core.jvm.abstract import JvmObject +from sedona.spark.core.enums import IndexType +from sedona.spark.core.enums.join_build_side import JoinBuildSide +from sedona.spark.core.jvm.abstract import JvmObject @attr.s diff --git a/python/sedona/core/spatialOperator/join_query.py b/python/sedona/spark/core/spatialOperator/join_query.py similarity index 94% rename from python/sedona/core/spatialOperator/join_query.py rename to python/sedona/spark/core/spatialOperator/join_query.py index 7e1f016536b..70830a800ea 100644 --- a/python/sedona/core/spatialOperator/join_query.py +++ b/python/sedona/spark/core/spatialOperator/join_query.py @@ -17,10 +17,10 @@ from pyspark import RDD -from sedona.core.spatialOperator.join_params import JoinParams -from sedona.core.spatialOperator.join_query_raw import JoinQueryRaw -from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD -from sedona.utils.decorators import require +from sedona.spark.core.spatialOperator.join_params import JoinParams +from sedona.spark.core.spatialOperator.join_query_raw import JoinQueryRaw +from sedona.spark.core.SpatialRDD.spatial_rdd import SpatialRDD +from sedona.spark.utils.decorators import require class JoinQuery: diff --git a/python/sedona/core/spatialOperator/join_query_raw.py b/python/sedona/spark/core/spatialOperator/join_query_raw.py similarity index 92% rename from python/sedona/core/spatialOperator/join_query_raw.py rename to python/sedona/spark/core/spatialOperator/join_query_raw.py index e97118215ae..fbaace1b2a7 100644 --- a/python/sedona/core/spatialOperator/join_query_raw.py +++ b/python/sedona/spark/core/spatialOperator/join_query_raw.py @@ -15,10 +15,10 @@ # specific language governing permissions and limitations # under the License. -from sedona.core.spatialOperator.join_params import JoinParams -from sedona.core.spatialOperator.rdd import SedonaPairRDD, SedonaPairRDDList -from sedona.core.SpatialRDD import SpatialRDD -from sedona.utils.decorators import require +from sedona.spark.core.spatialOperator.join_params import JoinParams +from sedona.spark.core.spatialOperator.rdd import SedonaPairRDD, SedonaPairRDDList +from sedona.spark.core.SpatialRDD import SpatialRDD +from sedona.spark.utils.decorators import require class JoinQueryRaw: diff --git a/python/sedona/core/spatialOperator/knn_query.py b/python/sedona/spark/core/spatialOperator/knn_query.py similarity index 83% rename from python/sedona/core/spatialOperator/knn_query.py rename to python/sedona/spark/core/spatialOperator/knn_query.py index c858cdc0f60..5f68943efbc 100644 --- a/python/sedona/core/spatialOperator/knn_query.py +++ b/python/sedona/spark/core/spatialOperator/knn_query.py @@ -18,12 +18,12 @@ import attr from shapely.geometry.base import BaseGeometry -from sedona.core.jvm.translate import JvmSedonaPythonConverter -from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD -from sedona.utils.binary_parser import BinaryParser -from sedona.utils.decorators import require -from sedona.utils.geometry_adapter import GeometryAdapter -from sedona.utils.spatial_rdd_parser import SpatialRDDParserData +from sedona.spark.core.jvm.translate import JvmSedonaPythonConverter +from sedona.spark.core.SpatialRDD.spatial_rdd import SpatialRDD +from sedona.spark.utils.binary_parser import BinaryParser +from sedona.spark.utils.decorators import require +from sedona.spark.utils.geometry_adapter import GeometryAdapter +from sedona.spark.utils.spatial_rdd_parser import SpatialRDDParserData @attr.s diff --git a/python/sedona/core/spatialOperator/range_query.py b/python/sedona/spark/core/spatialOperator/range_query.py similarity index 88% rename from python/sedona/core/spatialOperator/range_query.py rename to python/sedona/spark/core/spatialOperator/range_query.py index 4990f15ab71..2d557cdfe7a 100644 --- a/python/sedona/core/spatialOperator/range_query.py +++ b/python/sedona/spark/core/spatialOperator/range_query.py @@ -17,9 +17,9 @@ from shapely.geometry.base import BaseGeometry -from sedona.core.spatialOperator.range_query_raw import RangeQueryRaw -from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD -from sedona.utils.decorators import require +from sedona.spark.core.spatialOperator.range_query_raw import RangeQueryRaw +from sedona.spark.core.SpatialRDD.spatial_rdd import SpatialRDD +from sedona.spark.utils.decorators import require class RangeQuery: diff --git a/python/sedona/core/spatialOperator/range_query_raw.py b/python/sedona/spark/core/spatialOperator/range_query_raw.py similarity index 87% rename from python/sedona/core/spatialOperator/range_query_raw.py rename to python/sedona/spark/core/spatialOperator/range_query_raw.py index 1b03ef58bf8..c7481da8d5f 100644 --- a/python/sedona/core/spatialOperator/range_query_raw.py +++ b/python/sedona/spark/core/spatialOperator/range_query_raw.py @@ -17,10 +17,10 @@ from shapely.geometry.base import BaseGeometry -from sedona.core.spatialOperator.rdd import SedonaPairRDD, SedonaRDD -from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD -from sedona.utils.decorators import require -from sedona.utils.geometry_adapter import GeometryAdapter +from sedona.spark.core.spatialOperator.rdd import SedonaRDD +from sedona.spark.core.SpatialRDD.spatial_rdd import SpatialRDD +from sedona.spark.utils.decorators import require +from sedona.spark.utils.geometry_adapter import GeometryAdapter class RangeQueryRaw: diff --git a/python/sedona/core/spatialOperator/rdd.py b/python/sedona/spark/core/spatialOperator/rdd.py similarity index 93% rename from python/sedona/core/spatialOperator/rdd.py rename to python/sedona/spark/core/spatialOperator/rdd.py index 9576f731dbb..9f15fe416ef 100644 --- a/python/sedona/core/spatialOperator/rdd.py +++ b/python/sedona/spark/core/spatialOperator/rdd.py @@ -17,8 +17,8 @@ from pyspark import RDD, SparkContext -from sedona.core.jvm.translate import JvmSedonaPythonConverter -from sedona.utils.spatial_rdd_parser import SedonaPickler +from sedona.spark.core.jvm.translate import JvmSedonaPythonConverter +from sedona.spark.utils.spatial_rdd_parser import SedonaPickler class SedonaRDD: diff --git a/python/sedona/core/utils.py b/python/sedona/spark/core/utils.py similarity index 100% rename from python/sedona/core/utils.py rename to python/sedona/spark/core/utils.py diff --git a/python/sedona/spark/geoarrow/__init__.py b/python/sedona/spark/geoarrow/__init__.py new file mode 100644 index 00000000000..7c7ae431523 --- /dev/null +++ b/python/sedona/spark/geoarrow/__init__.py @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from sedona.spark.geoarrow.geoarrow import create_spatial_dataframe + +__all__ = ["create_spatial_dataframe"] diff --git a/python/sedona/utils/geoarrow.py b/python/sedona/spark/geoarrow/geoarrow.py similarity index 99% rename from python/sedona/utils/geoarrow.py rename to python/sedona/spark/geoarrow/geoarrow.py index 25ec012a8f7..a615f64e05d 100644 --- a/python/sedona/utils/geoarrow.py +++ b/python/sedona/spark/geoarrow/geoarrow.py @@ -21,12 +21,12 @@ # with the ArrowStreamSerializer (instead of the ArrowCollectSerializer) -from sedona.sql.st_functions import ST_AsEWKB +from sedona.spark.sql.st_functions import ST_AsEWKB from pyspark.sql import SparkSession from pyspark.sql import DataFrame from pyspark.sql.types import StructType, StructField, DataType, ArrayType, MapType -from sedona.sql.types import GeometryType +from sedona.spark.sql.types import GeometryType from pyspark.sql.pandas.types import ( from_arrow_type, ) diff --git a/python/sedona/maps/SedonaKepler.py b/python/sedona/spark/maps/SedonaKepler.py similarity index 95% rename from python/sedona/maps/SedonaKepler.py rename to python/sedona/spark/maps/SedonaKepler.py index 6822cb28ade..b3536f4cb54 100644 --- a/python/sedona/maps/SedonaKepler.py +++ b/python/sedona/spark/maps/SedonaKepler.py @@ -15,8 +15,8 @@ # specific language governing permissions and limitations # under the License. -from sedona.maps.SedonaMapUtils import SedonaMapUtils -from sedona.sql.types import RasterType +from sedona.spark.maps.SedonaMapUtils import SedonaMapUtils +from sedona.spark.sql.types import RasterType class SedonaKepler: diff --git a/python/sedona/maps/SedonaMapUtils.py b/python/sedona/spark/maps/SedonaMapUtils.py similarity index 97% rename from python/sedona/maps/SedonaMapUtils.py rename to python/sedona/spark/maps/SedonaMapUtils.py index d1385e8d985..a1d21088e3c 100644 --- a/python/sedona/maps/SedonaMapUtils.py +++ b/python/sedona/spark/maps/SedonaMapUtils.py @@ -17,8 +17,8 @@ import json -from sedona.sql.types import GeometryType -from sedona.utils.geoarrow import dataframe_to_arrow +from sedona.spark.sql.types import GeometryType +from sedona.spark.geoarrow.geoarrow import dataframe_to_arrow class SedonaMapUtils: diff --git a/python/sedona/maps/SedonaPyDeck.py b/python/sedona/spark/maps/SedonaPyDeck.py similarity index 99% rename from python/sedona/maps/SedonaPyDeck.py rename to python/sedona/spark/maps/SedonaPyDeck.py index a553817ec8f..d2da8368893 100644 --- a/python/sedona/maps/SedonaPyDeck.py +++ b/python/sedona/spark/maps/SedonaPyDeck.py @@ -27,7 +27,7 @@ ShortType, ) -from sedona.maps.SedonaMapUtils import SedonaMapUtils +from sedona.spark.maps.SedonaMapUtils import SedonaMapUtils class SedonaPyDeck: diff --git a/python/sedona/core/jvm/__init__.py b/python/sedona/spark/maps/__init__.py similarity index 100% rename from python/sedona/core/jvm/__init__.py rename to python/sedona/spark/maps/__init__.py diff --git a/python/sedona/maps/__init__.py b/python/sedona/spark/raster/__init__.py similarity index 100% rename from python/sedona/maps/__init__.py rename to python/sedona/spark/raster/__init__.py diff --git a/python/sedona/raster/awt_raster.py b/python/sedona/spark/raster/awt_raster.py similarity index 100% rename from python/sedona/raster/awt_raster.py rename to python/sedona/spark/raster/awt_raster.py diff --git a/python/sedona/raster/data_buffer.py b/python/sedona/spark/raster/data_buffer.py similarity index 100% rename from python/sedona/raster/data_buffer.py rename to python/sedona/spark/raster/data_buffer.py diff --git a/python/sedona/raster/meta.py b/python/sedona/spark/raster/meta.py similarity index 100% rename from python/sedona/raster/meta.py rename to python/sedona/spark/raster/meta.py diff --git a/python/sedona/raster/raster_serde.py b/python/sedona/spark/raster/raster_serde.py similarity index 100% rename from python/sedona/raster/raster_serde.py rename to python/sedona/spark/raster/raster_serde.py diff --git a/python/sedona/raster/sample_model.py b/python/sedona/spark/raster/sample_model.py similarity index 100% rename from python/sedona/raster/sample_model.py rename to python/sedona/spark/raster/sample_model.py diff --git a/python/sedona/raster/sedona_raster.py b/python/sedona/spark/raster/sedona_raster.py similarity index 100% rename from python/sedona/raster/sedona_raster.py rename to python/sedona/spark/raster/sedona_raster.py diff --git a/python/sedona/raster_utils/SedonaUtils.py b/python/sedona/spark/raster_utils/SedonaUtils.py similarity index 94% rename from python/sedona/raster_utils/SedonaUtils.py rename to python/sedona/spark/raster_utils/SedonaUtils.py index d35fcd62102..c28dbfb8215 100644 --- a/python/sedona/raster_utils/SedonaUtils.py +++ b/python/sedona/spark/raster_utils/SedonaUtils.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -from sedona.maps.SedonaMapUtils import SedonaMapUtils +from sedona.spark.maps.SedonaMapUtils import SedonaMapUtils class SedonaUtils: diff --git a/python/sedona/raster/__init__.py b/python/sedona/spark/raster_utils/__init__.py similarity index 100% rename from python/sedona/raster/__init__.py rename to python/sedona/spark/raster_utils/__init__.py diff --git a/python/sedona/register/__init__.py b/python/sedona/spark/register/__init__.py similarity index 92% rename from python/sedona/register/__init__.py rename to python/sedona/spark/register/__init__.py index 9b6fcffe9de..b4f0f682021 100644 --- a/python/sedona/register/__init__.py +++ b/python/sedona/spark/register/__init__.py @@ -15,6 +15,6 @@ # specific language governing permissions and limitations # under the License. -from sedona.register.geo_registrator import SedonaRegistrator +from sedona.spark.register.geo_registrator import SedonaRegistrator __all__ = ["SedonaRegistrator"] diff --git a/python/sedona/register/geo_registrator.py b/python/sedona/spark/register/geo_registrator.py similarity index 91% rename from python/sedona/register/geo_registrator.py rename to python/sedona/spark/register/geo_registrator.py index 8b80f3b17db..1b4de70a3e4 100644 --- a/python/sedona/register/geo_registrator.py +++ b/python/sedona/spark/register/geo_registrator.py @@ -19,9 +19,9 @@ from py4j.java_gateway import java_import from pyspark.sql import SparkSession -from sedona.core.jvm.config import deprecated -from sedona.register.java_libs import SedonaJvmLib -from sedona.utils.prep import assign_all +from sedona.spark.core.jvm.config import deprecated +from sedona.spark.register.java_libs import SedonaJvmLib +from sedona.spark.utils.prep import assign_all assign_all() jvm_import = str @@ -55,7 +55,7 @@ class PackageImporter: @staticmethod def import_jvm_lib(jvm) -> bool: - from sedona.core.utils import ImportedJvmLib + from sedona.spark.core.utils import ImportedJvmLib """ Imports all the specified methods and functions in jvm diff --git a/python/sedona/register/java_libs.py b/python/sedona/spark/register/java_libs.py similarity index 100% rename from python/sedona/register/java_libs.py rename to python/sedona/spark/register/java_libs.py diff --git a/python/sedona/sql/__init__.py b/python/sedona/spark/sql/__init__.py similarity index 74% rename from python/sedona/sql/__init__.py rename to python/sedona/spark/sql/__init__.py index c8e002fb0ac..3864d78f41e 100644 --- a/python/sedona/sql/__init__.py +++ b/python/sedona/spark/sql/__init__.py @@ -14,20 +14,21 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import inspect -import sys # These allow use to access the __all__ -import sedona.sql.st_aggregates as st_aggregates -import sedona.sql.st_constructors as st_constructors -import sedona.sql.st_functions as st_functions -import sedona.sql.st_predicates as st_predicates +import sedona.spark.sql.st_aggregates as st_aggregates +import sedona.spark.sql.st_constructors as st_constructors +import sedona.spark.sql.st_functions as st_functions +import sedona.spark.sql.st_predicates as st_predicates # These bring the contents of the modules into this module -from sedona.sql.st_aggregates import * -from sedona.sql.st_constructors import * -from sedona.sql.st_functions import * -from sedona.sql.st_predicates import * +from sedona.spark.sql.st_aggregates import * +from sedona.spark.sql.st_constructors import * +from sedona.spark.sql.st_functions import * +from sedona.spark.sql.st_predicates import * +import inspect +import sys + __all__ = ( [ diff --git a/python/sedona/sql/connect.py b/python/sedona/spark/sql/connect.py similarity index 100% rename from python/sedona/sql/connect.py rename to python/sedona/spark/sql/connect.py diff --git a/python/sedona/sql/dataframe_api.py b/python/sedona/spark/sql/dataframe_api.py similarity index 99% rename from python/sedona/sql/dataframe_api.py rename to python/sedona/spark/sql/dataframe_api.py index 4c73a8af07e..2efe479591f 100644 --- a/python/sedona/sql/dataframe_api.py +++ b/python/sedona/spark/sql/dataframe_api.py @@ -37,7 +37,7 @@ class ConnectColumn: pass else: - from sedona.sql.connect import call_sedona_function_connect + from sedona.spark.sql.connect import call_sedona_function_connect ColumnOrName = Union[Column, ConnectColumn, str] diff --git a/python/sedona/sql/exceptions.py b/python/sedona/spark/sql/exceptions.py similarity index 100% rename from python/sedona/sql/exceptions.py rename to python/sedona/spark/sql/exceptions.py diff --git a/python/sedona/sql/functions.py b/python/sedona/spark/sql/functions.py similarity index 97% rename from python/sedona/sql/functions.py rename to python/sedona/spark/sql/functions.py index 83648ff53f4..eb3b6f3c950 100644 --- a/python/sedona/sql/functions.py +++ b/python/sedona/spark/sql/functions.py @@ -21,8 +21,8 @@ import pandas as pd -from sedona.sql.types import GeometryType -from sedona.utils import geometry_serde +from sedona.spark.sql.types import GeometryType +from sedona.spark.utils import geometry_serde from pyspark.sql.udf import UserDefinedFunction from pyspark.sql.types import DataType from shapely.geometry.base import BaseGeometry diff --git a/python/sedona/sql/st_aggregates.py b/python/sedona/spark/sql/st_aggregates.py similarity index 98% rename from python/sedona/sql/st_aggregates.py rename to python/sedona/spark/sql/st_aggregates.py index 2156285bc28..f74cca7f84a 100644 --- a/python/sedona/sql/st_aggregates.py +++ b/python/sedona/spark/sql/st_aggregates.py @@ -20,7 +20,7 @@ from pyspark.sql import Column -from sedona.sql.dataframe_api import ( +from sedona.spark.sql.dataframe_api import ( ColumnOrName, call_sedona_function, validate_argument_types, diff --git a/python/sedona/sql/st_constructors.py b/python/sedona/spark/sql/st_constructors.py similarity index 99% rename from python/sedona/sql/st_constructors.py rename to python/sedona/spark/sql/st_constructors.py index c37f302af11..f47a33905d8 100644 --- a/python/sedona/sql/st_constructors.py +++ b/python/sedona/spark/sql/st_constructors.py @@ -21,7 +21,7 @@ from pyspark.sql import Column -from sedona.sql.dataframe_api import ( +from sedona.spark.sql.dataframe_api import ( ColumnOrName, ColumnOrNameOrNumber, call_sedona_function, diff --git a/python/sedona/sql/st_functions.py b/python/sedona/spark/sql/st_functions.py similarity index 99% rename from python/sedona/sql/st_functions.py rename to python/sedona/spark/sql/st_functions.py index 82bead1db2e..495444917af 100644 --- a/python/sedona/sql/st_functions.py +++ b/python/sedona/spark/sql/st_functions.py @@ -22,7 +22,7 @@ from pyspark.sql import Column from pyspark.sql.functions import lit -from sedona.sql.dataframe_api import ( +from sedona.spark.sql.dataframe_api import ( ColumnOrName, ColumnOrNameOrNumber, call_sedona_function, diff --git a/python/sedona/sql/st_predicates.py b/python/sedona/spark/sql/st_predicates.py similarity index 99% rename from python/sedona/sql/st_predicates.py rename to python/sedona/spark/sql/st_predicates.py index 18b33c1605e..71c464bffdf 100644 --- a/python/sedona/sql/st_predicates.py +++ b/python/sedona/spark/sql/st_predicates.py @@ -21,7 +21,7 @@ from pyspark.sql import Column -from sedona.sql.dataframe_api import ( +from sedona.spark.sql.dataframe_api import ( ColumnOrName, call_sedona_function, validate_argument_types, diff --git a/python/sedona/sql/types.py b/python/sedona/spark/sql/types.py similarity index 89% rename from python/sedona/sql/types.py rename to python/sedona/spark/sql/types.py index 1b8a9ee03bb..5465b283eb9 100644 --- a/python/sedona/sql/types.py +++ b/python/sedona/spark/sql/types.py @@ -24,16 +24,16 @@ rasterio = None if rasterio is not None: - from ..raster import raster_serde - from ..raster.sedona_raster import SedonaRaster + from sedona.spark.raster import raster_serde + from sedona.spark.raster.sedona_raster import SedonaRaster else: # We'll skip RasterType UDT registration and raise error when deserializing # RasterUDT objects if rasterio is not installed raster_serde = None SedonaRaster = None -from ..utils import geometry_serde -from ..core.geom.geography import Geography +from sedona.spark.utils import geometry_serde +from sedona.spark.core.geom.geography import Geography class GeometryType(UserDefinedType): @@ -51,7 +51,7 @@ def deserialize(self, datum): @classmethod def module(cls): - return "sedona.sql.types" + return "sedona.spark.sql.types" def needConversion(self): return True @@ -76,7 +76,7 @@ def deserialize(self, datum): @classmethod def module(cls): - return "sedona.sql.types" + return "sedona.spark.sql.types" def needConversion(self): return True @@ -105,7 +105,7 @@ def deserialize(self, datum): @classmethod def module(cls): - return "sedona.sql.types" + return "sedona.spark.sql.types" def needConversion(self): return True diff --git a/python/sedona/raster_utils/__init__.py b/python/sedona/spark/stac/__init__.py similarity index 94% rename from python/sedona/raster_utils/__init__.py rename to python/sedona/spark/stac/__init__.py index a67d5ea255b..f4bd70986d5 100644 --- a/python/sedona/raster_utils/__init__.py +++ b/python/sedona/spark/stac/__init__.py @@ -14,3 +14,4 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from sedona.spark.stac.client import Client diff --git a/python/sedona/stac/client.py b/python/sedona/spark/stac/client.py similarity index 98% rename from python/sedona/stac/client.py rename to python/sedona/spark/stac/client.py index 50ce0afc6c6..8f38cffc996 100644 --- a/python/sedona/stac/client.py +++ b/python/sedona/spark/stac/client.py @@ -16,7 +16,7 @@ # under the License. from typing import Union, Optional, Iterator -from sedona.stac.collection_client import CollectionClient +from sedona.spark.stac.collection_client import CollectionClient import datetime as python_datetime from pystac import Item as PyStacItem diff --git a/python/sedona/stac/collection_client.py b/python/sedona/spark/stac/collection_client.py similarity index 100% rename from python/sedona/stac/collection_client.py rename to python/sedona/spark/stac/collection_client.py diff --git a/python/sedona/spark/stats/__init__.py b/python/sedona/spark/stats/__init__.py new file mode 100644 index 00000000000..5219e02429c --- /dev/null +++ b/python/sedona/spark/stats/__init__.py @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from sedona.spark.stats.clustering.dbscan import dbscan +from sedona.spark.stats.outlier_detection.local_outlier_factor import ( + local_outlier_factor, +) +from sedona.spark.stats.hotspot_detection.getis_ord import g_local +from sedona.spark.stats.weighting import add_distance_band_column +from sedona.spark.stats.weighting import add_binary_distance_band_column +from sedona.spark.stats.weighting import add_weighted_distance_band_column diff --git a/python/sedona/spark/stats/clustering/__init__.py b/python/sedona/spark/stats/clustering/__init__.py new file mode 100644 index 00000000000..d2399abf8a8 --- /dev/null +++ b/python/sedona/spark/stats/clustering/__init__.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""The clustering module contains spark based implementations of popular geospatial clustering algorithms. + +These implementations are designed to scale to larger datasets and support various geometric feature types. +""" diff --git a/python/sedona/stats/clustering/dbscan.py b/python/sedona/spark/stats/clustering/dbscan.py similarity index 100% rename from python/sedona/stats/clustering/dbscan.py rename to python/sedona/spark/stats/clustering/dbscan.py diff --git a/python/sedona/spark/stats/hotspot_detection/__init__.py b/python/sedona/spark/stats/hotspot_detection/__init__.py new file mode 100644 index 00000000000..ed2c4706d18 --- /dev/null +++ b/python/sedona/spark/stats/hotspot_detection/__init__.py @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Detecting across a region where a variable's value is significantly different from other values nearby.""" diff --git a/python/sedona/stats/hotspot_detection/getis_ord.py b/python/sedona/spark/stats/hotspot_detection/getis_ord.py similarity index 100% rename from python/sedona/stats/hotspot_detection/getis_ord.py rename to python/sedona/spark/stats/hotspot_detection/getis_ord.py diff --git a/python/sedona/spark/stats/outlier_detection/__init__.py b/python/sedona/spark/stats/outlier_detection/__init__.py new file mode 100644 index 00000000000..4dd25a3ff63 --- /dev/null +++ b/python/sedona/spark/stats/outlier_detection/__init__.py @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Algorithms for detecting outliers in spatial datasets.""" diff --git a/python/sedona/stats/outlier_detection/local_outlier_factor.py b/python/sedona/spark/stats/outlier_detection/local_outlier_factor.py similarity index 100% rename from python/sedona/stats/outlier_detection/local_outlier_factor.py rename to python/sedona/spark/stats/outlier_detection/local_outlier_factor.py diff --git a/python/sedona/stats/weighting.py b/python/sedona/spark/stats/weighting.py similarity index 100% rename from python/sedona/stats/weighting.py rename to python/sedona/spark/stats/weighting.py diff --git a/python/sedona/spark/utils/__init__.py b/python/sedona/spark/utils/__init__.py new file mode 100644 index 00000000000..8c3ca093394 --- /dev/null +++ b/python/sedona/spark/utils/__init__.py @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .serde import KryoSerializer, SedonaKryoRegistrator + +__all__ = ["KryoSerializer", "SedonaKryoRegistrator"] diff --git a/python/sedona/utils/abstract_parser.py b/python/sedona/spark/utils/abstract_parser.py similarity index 100% rename from python/sedona/utils/abstract_parser.py rename to python/sedona/spark/utils/abstract_parser.py diff --git a/python/sedona/utils/adapter.py b/python/sedona/spark/utils/adapter.py similarity index 94% rename from python/sedona/utils/adapter.py rename to python/sedona/spark/utils/adapter.py index 7b978e72122..42db222070c 100644 --- a/python/sedona/utils/adapter.py +++ b/python/sedona/spark/utils/adapter.py @@ -20,10 +20,10 @@ from pyspark import RDD from pyspark.sql import DataFrame, SparkSession -from sedona.core.enums.spatial import SpatialType -from sedona.core.spatialOperator.rdd import SedonaPairRDD, SedonaRDD -from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD -from sedona.utils.meta import MultipleMeta +from sedona.spark.core.enums.spatial import SpatialType +from sedona.spark.core.spatialOperator.rdd import SedonaPairRDD, SedonaRDD +from sedona.spark.core.SpatialRDD.spatial_rdd import SpatialRDD +from sedona.spark.utils.meta import MultipleMeta class Adapter(metaclass=MultipleMeta): @@ -40,7 +40,7 @@ def _create_dataframe(jdf, sparkSession: SparkSession) -> DataFrame: @classmethod def toRdd(cls, dataFrame: DataFrame) -> "JvmSpatialRDD": - from sedona.core.SpatialRDD.spatial_rdd import JvmSpatialRDD + from sedona.spark.core.SpatialRDD.spatial_rdd import JvmSpatialRDD sc = dataFrame._sc jvm = sc._jvm diff --git a/python/sedona/utils/binary_parser.py b/python/sedona/spark/utils/binary_parser.py similarity index 100% rename from python/sedona/utils/binary_parser.py rename to python/sedona/spark/utils/binary_parser.py diff --git a/python/sedona/utils/decorators.py b/python/sedona/spark/utils/decorators.py similarity index 97% rename from python/sedona/utils/decorators.py rename to python/sedona/spark/utils/decorators.py index 2794baba182..50350bf6da9 100644 --- a/python/sedona/utils/decorators.py +++ b/python/sedona/spark/utils/decorators.py @@ -44,7 +44,7 @@ def get_first_meet_criteria_element_from_iterable( def require(library_names: List[str]): def wrapper(func): def run_function(*args, **kwargs): - from sedona.core.utils import ImportedJvmLib + from sedona.spark.core.utils import ImportedJvmLib has_all_libs = [lib for lib in library_names] first_not_fulfill_value = get_first_meet_criteria_element_from_iterable( diff --git a/python/sedona/utils/geometry_adapter.py b/python/sedona/spark/utils/geometry_adapter.py similarity index 87% rename from python/sedona/utils/geometry_adapter.py rename to python/sedona/spark/utils/geometry_adapter.py index 9a81307d862..00ca55e139b 100644 --- a/python/sedona/utils/geometry_adapter.py +++ b/python/sedona/spark/utils/geometry_adapter.py @@ -17,9 +17,9 @@ from shapely.geometry.base import BaseGeometry -from sedona.core.geom.envelope import Envelope -from sedona.core.jvm.translate import JvmGeometryAdapter -from sedona.utils.spatial_rdd_parser import GeometryFactory +from sedona.spark.core.geom.envelope import Envelope +from sedona.spark.core.jvm.translate import JvmGeometryAdapter +from sedona.spark.utils.spatial_rdd_parser import GeometryFactory class GeometryAdapter: diff --git a/python/sedona/utils/geometry_serde.py b/python/sedona/spark/utils/geometry_serde.py similarity index 100% rename from python/sedona/utils/geometry_serde.py rename to python/sedona/spark/utils/geometry_serde.py diff --git a/python/sedona/utils/geometry_serde_general.py b/python/sedona/spark/utils/geometry_serde_general.py similarity index 100% rename from python/sedona/utils/geometry_serde_general.py rename to python/sedona/spark/utils/geometry_serde_general.py diff --git a/python/sedona/utils/jvm.py b/python/sedona/spark/utils/jvm.py similarity index 92% rename from python/sedona/utils/jvm.py rename to python/sedona/spark/utils/jvm.py index 1b843a001fc..50b1670199c 100644 --- a/python/sedona/utils/jvm.py +++ b/python/sedona/spark/utils/jvm.py @@ -17,8 +17,8 @@ import attr -from sedona.core.jvm.abstract import JvmObject -from sedona.utils.decorators import require +from sedona.spark.core.jvm.abstract import JvmObject +from sedona.spark.utils.decorators import require @attr.s diff --git a/python/sedona/utils/meta.py b/python/sedona/spark/utils/meta.py similarity index 100% rename from python/sedona/utils/meta.py rename to python/sedona/spark/utils/meta.py diff --git a/python/sedona/utils/prep.py b/python/sedona/spark/utils/prep.py similarity index 91% rename from python/sedona/utils/prep.py rename to python/sedona/spark/utils/prep.py index 8cabfa05c2a..11bec774483 100644 --- a/python/sedona/utils/prep.py +++ b/python/sedona/spark/utils/prep.py @@ -28,7 +28,7 @@ ) from shapely.geometry.base import BaseGeometry -from ..core.geom.geography import Geography +from sedona.spark.core.geom.geography import Geography def assign_all() -> bool: @@ -48,7 +48,7 @@ def assign_all() -> bool: def assign_udt_shapely_objects(geoms: List[type(BaseGeometry)]) -> bool: - from sedona.sql.types import GeometryType + from sedona.spark.sql.types import GeometryType for geom in geoms: geom.__UDT__ = GeometryType() @@ -61,6 +61,6 @@ def assign_user_data_to_shapely_objects(geoms: List[type(BaseGeometry)]) -> bool def assign_udt_geography(): - from sedona.sql.types import GeographyType + from sedona.spark.sql.types import GeographyType Geography.__UDT__ = GeographyType() diff --git a/python/sedona/utils/serde.py b/python/sedona/spark/utils/serde.py similarity index 95% rename from python/sedona/utils/serde.py rename to python/sedona/spark/utils/serde.py index e4d6e786843..37608c7b7b6 100644 --- a/python/sedona/utils/serde.py +++ b/python/sedona/spark/utils/serde.py @@ -17,7 +17,7 @@ from abc import ABC -from sedona.utils.decorators import classproperty +from sedona.spark.utils.decorators import classproperty class Serializer(ABC): diff --git a/python/sedona/utils/spatial_rdd_parser.py b/python/sedona/spark/utils/spatial_rdd_parser.py similarity index 96% rename from python/sedona/utils/spatial_rdd_parser.py rename to python/sedona/spark/utils/spatial_rdd_parser.py index d99eecf8075..0a8be26aca9 100644 --- a/python/sedona/utils/spatial_rdd_parser.py +++ b/python/sedona/spark/utils/spatial_rdd_parser.py @@ -30,8 +30,7 @@ from shapely.wkb import dumps -from sedona.core.geom.circle import Circle -from sedona.utils.binary_parser import BinaryParser +from sedona.spark.utils.binary_parser import BinaryParser class GeoData: @@ -49,6 +48,8 @@ def getUserData(self): return self.userData def __getstate__(self): + from sedona.spark.core.geom.circle import Circle + attributes = copy(self.__slots__) geom = getattr(self, attributes[0]) @@ -63,6 +64,8 @@ def __getstate__(self): ) def __setstate__(self, attributes): + from sedona.spark.core.geom.circle import Circle + bin_parser = BinaryParser(attributes["geom"]) is_circle = bin_parser.read_byte() geom_bytes = attributes["geom"] @@ -234,13 +237,15 @@ class CircleGeometryFactory: @classmethod def geometry_from_bytes(cls, bin_parser: BinaryParser) -> GeoData: + from sedona.spark.core.geom.circle import Circle + geom, user_data = read_geometry_from_bytes(bin_parser) radius = bin_parser.read_double() geo_data = GeoData(geom=Circle(geom, radius), userData=user_data) return geo_data @classmethod - def to_bytes(cls, geom: Circle) -> List[int]: + def to_bytes(cls, geom: "Circle") -> List[int]: return ( struct.pack("b", 1) + struct.pack("d", geom.radius) diff --git a/python/sedona/utils/structured_adapter.py b/python/sedona/spark/utils/structured_adapter.py similarity index 96% rename from python/sedona/utils/structured_adapter.py rename to python/sedona/spark/utils/structured_adapter.py index bc3b9593638..6af7df8a245 100644 --- a/python/sedona/utils/structured_adapter.py +++ b/python/sedona/spark/utils/structured_adapter.py @@ -18,8 +18,8 @@ from pyspark.sql import DataFrame, SparkSession from pyspark.sql.types import StructType -from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD -from sedona.core.spatialOperator.rdd import SedonaPairRDD +from sedona.spark.core.SpatialRDD.spatial_rdd import SpatialRDD +from sedona.spark.core.spatialOperator.rdd import SedonaPairRDD class StructuredAdapter: diff --git a/python/sedona/utils/types.py b/python/sedona/spark/utils/types.py similarity index 100% rename from python/sedona/utils/types.py rename to python/sedona/spark/utils/types.py diff --git a/python/sedona/stac/__init__.py b/python/sedona/stac/__init__.py index a67d5ea255b..aa0ffeea502 100644 --- a/python/sedona/stac/__init__.py +++ b/python/sedona/stac/__init__.py @@ -14,3 +14,17 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + +from sedona.spark.stac.client import Client + +import warnings + +warnings.warn( + "The 'sedona.stac' module is deprecated and will be removed in future versions. Please use 'sedona.spark.stac' instead.", + DeprecationWarning, + stacklevel=2, +) + +__all__ = [ + "Client", +] diff --git a/python/sedona/stats/clustering/__init__.py b/python/sedona/stats/clustering/__init__.py index d2399abf8a8..a67d5ea255b 100644 --- a/python/sedona/stats/clustering/__init__.py +++ b/python/sedona/stats/clustering/__init__.py @@ -14,8 +14,3 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - -"""The clustering module contains spark based implementations of popular geospatial clustering algorithms. - -These implementations are designed to scale to larger datasets and support various geometric feature types. -""" diff --git a/python/sedona/stats/clustering/dbscan/__init__.py b/python/sedona/stats/clustering/dbscan/__init__.py new file mode 100644 index 00000000000..c6955ddbc35 --- /dev/null +++ b/python/sedona/stats/clustering/dbscan/__init__.py @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from sedona.spark.stats.clustering.dbscan import dbscan + +import warnings + +warnings.warn( + "The 'sedona.stats.clustering.dbscan' module is deprecated and will be removed in future versions. Please use 'sedona.spark.stats' instead.", + DeprecationWarning, + stacklevel=2, +) + +__all__ = ["dbscan"] diff --git a/python/sedona/stats/hotspot_detection/__init__.py b/python/sedona/stats/hotspot_detection/__init__.py index ed2c4706d18..a67d5ea255b 100644 --- a/python/sedona/stats/hotspot_detection/__init__.py +++ b/python/sedona/stats/hotspot_detection/__init__.py @@ -14,5 +14,3 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - -"""Detecting across a region where a variable's value is significantly different from other values nearby.""" diff --git a/python/sedona/stats/hotspot_detection/getis_ord/__init__.py b/python/sedona/stats/hotspot_detection/getis_ord/__init__.py new file mode 100644 index 00000000000..888439b800d --- /dev/null +++ b/python/sedona/stats/hotspot_detection/getis_ord/__init__.py @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from sedona.spark.stats.hotspot_detection.getis_ord import g_local + +import warnings + +warnings.warn( + "The 'sedona.stats.hotspot_detection.getis_ord' module is deprecated and will be removed in future versions. Please use 'sedona.spark.stats' instead.", + DeprecationWarning, + stacklevel=2, +) diff --git a/python/sedona/stats/outlier_detection/__init__.py b/python/sedona/stats/outlier_detection/__init__.py index 4dd25a3ff63..a67d5ea255b 100644 --- a/python/sedona/stats/outlier_detection/__init__.py +++ b/python/sedona/stats/outlier_detection/__init__.py @@ -14,5 +14,3 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - -"""Algorithms for detecting outliers in spatial datasets.""" diff --git a/python/sedona/stats/outlier_detection/local_outlier_factor/__init__.py b/python/sedona/stats/outlier_detection/local_outlier_factor/__init__.py new file mode 100644 index 00000000000..613095bfd81 --- /dev/null +++ b/python/sedona/stats/outlier_detection/local_outlier_factor/__init__.py @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from sedona.spark.stats.outlier_detection.local_outlier_factor import ( + local_outlier_factor, +) + +import warnings + +warnings.warn( + "The 'sedona.stats.outlier_detection.local_outlier_factor' module is deprecated and will be removed in future versions. Please use 'sedona.spark.stats' instead.", + DeprecationWarning, + stacklevel=2, +) + +__all__ = ["local_outlier_factor"] diff --git a/python/sedona/stats/weighting/__init__.py b/python/sedona/stats/weighting/__init__.py new file mode 100644 index 00000000000..e1b827a92d9 --- /dev/null +++ b/python/sedona/stats/weighting/__init__.py @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from sedona.spark.stats.weighting import add_binary_distance_band_column + +import warnings + +warnings.warn( + "The 'sedona.stats.weighting' module is deprecated and will be removed in future versions. Please use 'sedona.spark.stats' instead.", + DeprecationWarning, + stacklevel=2, +) diff --git a/python/sedona/utils/__init__.py b/python/sedona/utils/__init__.py index 8c3ca093394..a67d5ea255b 100644 --- a/python/sedona/utils/__init__.py +++ b/python/sedona/utils/__init__.py @@ -14,7 +14,3 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - -from .serde import KryoSerializer, SedonaKryoRegistrator - -__all__ = ["KryoSerializer", "SedonaKryoRegistrator"] diff --git a/python/sedona/utils/geoarrow/__init__.py b/python/sedona/utils/geoarrow/__init__.py new file mode 100644 index 00000000000..aea65415f32 --- /dev/null +++ b/python/sedona/utils/geoarrow/__init__.py @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import warnings + +from sedona.spark.geoarrow.geoarrow import create_spatial_dataframe + +warnings.warn( + "The 'sedona.utils.geoarrow' module is deprecated and will be removed in future versions. Please use 'sedona.spark.geoarrow' instead.", + DeprecationWarning, + stacklevel=2, +) + +__all__ = ["create_spatial_dataframe"] diff --git a/python/sedona/utils/structured_adapter/__init__.py b/python/sedona/utils/structured_adapter/__init__.py new file mode 100644 index 00000000000..b0063f07e35 --- /dev/null +++ b/python/sedona/utils/structured_adapter/__init__.py @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from sedona.spark.utils.structured_adapter import StructuredAdapter + +import warnings + +warnings.warn( + "The 'sedona.utils.structured_adapter' module is deprecated and will be removed in future versions. Please use 'sedona.spark.utils.structured_adapter' instead.", + DeprecationWarning, + stacklevel=2, +) + +__all__ = [ + "StructuredAdapter", +] diff --git a/python/setup.py b/python/setup.py index a6bc7e95601..9897e082905 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,19 +1,19 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. import os @@ -34,7 +34,7 @@ ext_modules = [ Extension( - "sedona.utils.geomserde_speedup", + "sedona.spark.utils.geomserde_speedup", sources=[ "src/geomserde_speedup_module.c", "src/geomserde.c", diff --git a/python/tests/__init__.py b/python/tests/__init__.py index 0f8fa55be85..f05c0eb8947 100644 --- a/python/tests/__init__.py +++ b/python/tests/__init__.py @@ -17,7 +17,12 @@ import os -from tests.tools import tests_resource +from os import path + +tests_path = os.path.abspath(os.path.join(__file__, "../../../spark/common/src/test/")) +tests_resource = os.path.abspath( + path.join(__file__, "../../../spark/common/src/test/resources/") +) mixed_wkb_geometry_input_location = os.path.join(tests_resource, "county_small_wkb.tsv") mixed_wkt_geometry_input_location = os.path.join(tests_resource, "county_small.tsv") diff --git a/python/tests/core/test_avoiding_python_jvm_serde_df.py b/python/tests/core/test_avoiding_python_jvm_serde_df.py index 117429d9960..bb3df988639 100644 --- a/python/tests/core/test_avoiding_python_jvm_serde_df.py +++ b/python/tests/core/test_avoiding_python_jvm_serde_df.py @@ -18,17 +18,18 @@ from pyspark.sql.types import StructField, StructType from shapely.wkt import loads + +from tests import tests_resource from tests.test_base import TestBase -from tests.tools import tests_resource - -from sedona.core.enums import GridType, IndexType -from sedona.core.formatMapper import WktReader -from sedona.core.spatialOperator.join_params import JoinParams -from sedona.core.spatialOperator.join_query_raw import JoinQueryRaw -from sedona.core.spatialOperator.range_query_raw import RangeQueryRaw -from sedona.core.SpatialRDD import CircleRDD -from sedona.sql.types import GeometryType -from sedona.utils.adapter import Adapter + +from sedona.spark.core.enums import GridType, IndexType +from sedona.spark.core.formatMapper import WktReader +from sedona.spark.core.spatialOperator.join_params import JoinParams +from sedona.spark.core.spatialOperator.join_query_raw import JoinQueryRaw +from sedona.spark.core.spatialOperator.range_query_raw import RangeQueryRaw +from sedona.spark.core.SpatialRDD import CircleRDD +from sedona.spark.sql.types import GeometryType +from sedona.spark.utils.adapter import Adapter bank_csv_path = os.path.join(tests_resource, "small/points.csv") areas_csv_path = os.path.join(tests_resource, "small/areas.csv") diff --git a/python/tests/core/test_avoiding_python_jvm_serde_to_rdd.py b/python/tests/core/test_avoiding_python_jvm_serde_to_rdd.py index 5440034fd61..99c08ef2ff3 100644 --- a/python/tests/core/test_avoiding_python_jvm_serde_to_rdd.py +++ b/python/tests/core/test_avoiding_python_jvm_serde_to_rdd.py @@ -18,15 +18,16 @@ import os from shapely.wkt import loads + +from tests import tests_resource from tests.test_base import TestBase -from tests.tools import tests_resource - -from sedona.core.enums import GridType, IndexType -from sedona.core.formatMapper import WktReader -from sedona.core.spatialOperator.join_params import JoinParams -from sedona.core.spatialOperator.join_query_raw import JoinQueryRaw -from sedona.core.spatialOperator.range_query_raw import RangeQueryRaw -from sedona.core.SpatialRDD import CircleRDD + +from sedona.spark.core.enums import GridType, IndexType +from sedona.spark.core.formatMapper import WktReader +from sedona.spark.core.spatialOperator.join_params import JoinParams +from sedona.spark.core.spatialOperator.join_query_raw import JoinQueryRaw +from sedona.spark.core.spatialOperator.range_query_raw import RangeQueryRaw +from sedona.spark.core.SpatialRDD import CircleRDD bank_csv_path = os.path.join(tests_resource, "small/points.csv") areas_csv_path = os.path.join(tests_resource, "small/areas.csv") diff --git a/python/tests/core/test_config.py b/python/tests/core/test_config.py index f188d99e942..926a1732833 100644 --- a/python/tests/core/test_config.py +++ b/python/tests/core/test_config.py @@ -19,7 +19,7 @@ from tests.test_base import TestBase -from sedona.core.jvm.config import SedonaMeta, SparkJars +from sedona.spark.core.jvm.config import SedonaMeta, SparkJars class TestCoreJVMConfig(TestBase): diff --git a/python/tests/core/test_core_geom_primitives.py b/python/tests/core/test_core_geom_primitives.py index 09717cef16f..708b3f64c2c 100644 --- a/python/tests/core/test_core_geom_primitives.py +++ b/python/tests/core/test_core_geom_primitives.py @@ -17,7 +17,7 @@ from tests.test_base import TestBase -from sedona.core.geom.envelope import Envelope +from sedona.spark.core.geom.envelope import Envelope class TestGeomPrimitives(TestBase): diff --git a/python/tests/core/test_core_rdd.py b/python/tests/core/test_core_rdd.py index 8567fd84d25..c83551b79b7 100644 --- a/python/tests/core/test_core_rdd.py +++ b/python/tests/core/test_core_rdd.py @@ -17,11 +17,11 @@ import os +from tests import tests_resource from tests.test_base import TestBase -from tests.tools import tests_resource -from sedona.core.enums import FileDataSplitter -from sedona.core.SpatialRDD import PointRDD, PolygonRDD +from sedona.spark.core.enums import FileDataSplitter +from sedona.spark.core.SpatialRDD import PointRDD, PolygonRDD point_path = os.path.join(tests_resource, "points.csv") counties_path = os.path.join(tests_resource, "county_small.tsv") diff --git a/python/tests/core/test_rdd.py b/python/tests/core/test_rdd.py index abf5e1c9b95..ff649a58f96 100644 --- a/python/tests/core/test_rdd.py +++ b/python/tests/core/test_rdd.py @@ -19,6 +19,8 @@ import os from shapely.geometry import Point + +from tests import tests_resource from tests.properties.polygon_properties import ( polygon_rdd_end_offset, polygon_rdd_index_type, @@ -27,14 +29,13 @@ polygon_rdd_start_offset, ) from tests.test_base import TestBase -from tests.tools import tests_resource - -from sedona.core.enums import FileDataSplitter, GridType, IndexType -from sedona.core.enums.join_build_side import JoinBuildSide -from sedona.core.geom.envelope import Envelope -from sedona.core.spatialOperator import JoinQuery, KNNQuery, RangeQuery -from sedona.core.spatialOperator.join_params import JoinParams -from sedona.core.SpatialRDD import CircleRDD, PointRDD, PolygonRDD + +from sedona.spark.core.enums import FileDataSplitter, GridType, IndexType +from sedona.spark.core.enums.join_build_side import JoinBuildSide +from sedona.spark.core.geom.envelope import Envelope +from sedona.spark.core.spatialOperator import JoinQuery, KNNQuery, RangeQuery +from sedona.spark.core.spatialOperator.join_params import JoinParams +from sedona.spark.core.SpatialRDD import CircleRDD, PointRDD, PolygonRDD point_rdd_input_location = os.path.join(tests_resource, "arealm-small.csv") diff --git a/python/tests/core/test_spatial_rdd_from_disc.py b/python/tests/core/test_spatial_rdd_from_disc.py index f3f3cbabf6b..74cee595cf3 100644 --- a/python/tests/core/test_spatial_rdd_from_disc.py +++ b/python/tests/core/test_spatial_rdd_from_disc.py @@ -18,18 +18,17 @@ import os import shutil -import pytest +from tests import tests_resource from tests.test_base import TestBase -from tests.tools import tests_resource -from sedona.core.enums import GridType, IndexType -from sedona.core.formatMapper.disc_utils import ( +from sedona.spark.core.enums import GridType, IndexType +from sedona.spark.core.formatMapper.disc_utils import ( GeoType, load_spatial_index_rdd_from_disc, load_spatial_rdd_from_disc, ) -from sedona.core.spatialOperator import JoinQuery -from sedona.core.SpatialRDD import LineStringRDD, PointRDD, PolygonRDD +from sedona.spark.core.spatialOperator import JoinQuery +from sedona.spark.core.SpatialRDD import LineStringRDD, PointRDD, PolygonRDD def remove_directory(path: str) -> bool: diff --git a/python/tests/format_mapper/test_geo_json_reader.py b/python/tests/format_mapper/test_geo_json_reader.py index 042bd08fcec..5b550ea51d1 100644 --- a/python/tests/format_mapper/test_geo_json_reader.py +++ b/python/tests/format_mapper/test_geo_json_reader.py @@ -17,12 +17,11 @@ import os -import pyspark +from tests import tests_resource from tests.test_base import TestBase -from tests.tools import tests_resource -from sedona.core.formatMapper.geo_json_reader import GeoJsonReader -from sedona.core.jvm.config import SedonaMeta, is_greater_or_equal_version +from sedona.spark.core.formatMapper.geo_json_reader import GeoJsonReader +from sedona.spark.core.jvm.config import SedonaMeta, is_greater_or_equal_version geo_json_contains_id = os.path.join(tests_resource, "testContainsId.json") geo_json_geom_with_feature_property = os.path.join(tests_resource, "testPolygon.json") diff --git a/python/tests/format_mapper/test_shapefile_reader.py b/python/tests/format_mapper/test_shapefile_reader.py index 8fe47154a9a..0a8afd1f9b6 100644 --- a/python/tests/format_mapper/test_shapefile_reader.py +++ b/python/tests/format_mapper/test_shapefile_reader.py @@ -18,13 +18,13 @@ import os import pytest + +from tests import tests_resource from tests.test_base import TestBase -from tests.tools import tests_resource -from sedona.core.formatMapper.shapefileParser import ShapefileReader -from sedona.core.geom.envelope import Envelope -from sedona.core.jvm.config import SedonaMeta, is_greater_or_equal_version -from sedona.core.spatialOperator import RangeQuery +from sedona.spark.core.formatMapper.shapefileParser import ShapefileReader +from sedona.spark.core.geom.envelope import Envelope +from sedona.spark.core.spatialOperator import RangeQuery undefined_type_shape_location = os.path.join(tests_resource, "shapefiles/undefined") polygon_shape_location = os.path.join(tests_resource, "shapefiles/polygon") diff --git a/python/tests/format_mapper/test_wkb_reader.py b/python/tests/format_mapper/test_wkb_reader.py index d9452babbc9..c3dc5df531d 100644 --- a/python/tests/format_mapper/test_wkb_reader.py +++ b/python/tests/format_mapper/test_wkb_reader.py @@ -17,10 +17,10 @@ import os +from tests import tests_resource from tests.test_base import TestBase -from tests.tools import tests_resource -from sedona.core.formatMapper.wkb_reader import WkbReader +from sedona.spark.core.formatMapper.wkb_reader import WkbReader class TestWkbReader(TestBase): diff --git a/python/tests/format_mapper/test_wkt_reader.py b/python/tests/format_mapper/test_wkt_reader.py index ec64158fe3b..83e7715ff0d 100644 --- a/python/tests/format_mapper/test_wkt_reader.py +++ b/python/tests/format_mapper/test_wkt_reader.py @@ -17,10 +17,10 @@ import os +from tests import tests_resource from tests.test_base import TestBase -from tests.tools import tests_resource -from sedona.core.formatMapper import WktReader +from sedona.spark.core.formatMapper import WktReader class TestWktReader(TestBase): diff --git a/python/tests/maps/test_sedonakepler_visualization.py b/python/tests/maps/test_sedonakepler_visualization.py index 808b72a2f19..24501aa06d5 100644 --- a/python/tests/maps/test_sedonakepler_visualization.py +++ b/python/tests/maps/test_sedonakepler_visualization.py @@ -25,7 +25,7 @@ ) from tests.test_base import TestBase -from sedona.maps.SedonaKepler import SedonaKepler +from sedona.spark import SedonaKepler class TestVisualization(TestBase): diff --git a/python/tests/maps/test_sedonapydeck.py b/python/tests/maps/test_sedonapydeck.py index daa1d397b48..106fc28eb22 100644 --- a/python/tests/maps/test_sedonapydeck.py +++ b/python/tests/maps/test_sedonapydeck.py @@ -22,7 +22,7 @@ from tests import chicago_crimes_input_location, google_buildings_input_location from tests.test_base import TestBase -from sedona.maps.SedonaPyDeck import SedonaPyDeck +from sedona.spark import SedonaPyDeck class TestVisualization(TestBase): diff --git a/python/tests/properties/crs_transform.py b/python/tests/properties/crs_transform.py index af6437a0efe..0ae05c143b1 100644 --- a/python/tests/properties/crs_transform.py +++ b/python/tests/properties/crs_transform.py @@ -18,10 +18,10 @@ import os from shapely.geometry import Point -from tests.tools import tests_resource -from sedona.core.enums import FileDataSplitter, IndexType -from sedona.core.geom.envelope import Envelope +from sedona.spark.core.enums import FileDataSplitter, IndexType +from sedona.spark.core.geom.envelope import Envelope +from tests import tests_resource input_location = os.path.join(tests_resource, "crs-test-point.csv") offset = 0 diff --git a/python/tests/properties/linestring_properties.py b/python/tests/properties/linestring_properties.py index e53fc14b72c..627197180db 100644 --- a/python/tests/properties/linestring_properties.py +++ b/python/tests/properties/linestring_properties.py @@ -17,10 +17,10 @@ import os -from tests.tools import tests_resource -from sedona.core.enums import FileDataSplitter -from sedona.core.geom.envelope import Envelope +from sedona.spark.core.enums import FileDataSplitter +from sedona.spark.core.geom.envelope import Envelope +from tests import tests_resource input_location = os.path.join(tests_resource, "primaryroads-linestring.csv") query_window_set = os.path.join(tests_resource, "zcta510-small.csv") diff --git a/python/tests/properties/point_properties.py b/python/tests/properties/point_properties.py index bea7dcf1054..664b2bfb340 100644 --- a/python/tests/properties/point_properties.py +++ b/python/tests/properties/point_properties.py @@ -17,10 +17,9 @@ import os -from tests.tools import tests_resource - -from sedona.core.enums import FileDataSplitter -from sedona.core.geom.envelope import Envelope +from sedona.spark.core.enums import FileDataSplitter +from sedona.spark.core.geom.envelope import Envelope +from tests import tests_resource input_location = os.path.join(tests_resource, "arealm-small.csv") query_window_set = os.path.join(tests_resource, "zcta510-small.csv") diff --git a/python/tests/properties/polygon_properties.py b/python/tests/properties/polygon_properties.py index 09680f08619..6e1912836ca 100644 --- a/python/tests/properties/polygon_properties.py +++ b/python/tests/properties/polygon_properties.py @@ -17,10 +17,9 @@ import os -from tests.tools import tests_resource - -from sedona.core.enums import FileDataSplitter, IndexType -from sedona.core.geom.envelope import Envelope +from sedona.spark.core.enums import FileDataSplitter, IndexType +from sedona.spark.core.geom.envelope import Envelope +from tests import tests_resource input_location = os.path.join(tests_resource, "primaryroads-polygon.csv") query_window_set = os.path.join(tests_resource, "zcta510-small.csv") diff --git a/python/tests/raster/test_meta.py b/python/tests/raster/test_meta.py index 9c4d01dd59c..903b101ff88 100644 --- a/python/tests/raster/test_meta.py +++ b/python/tests/raster/test_meta.py @@ -15,10 +15,9 @@ # specific language governing permissions and limitations # under the License. -import pytest from pytest import approx -from sedona.raster.meta import AffineTransform, PixelAnchor +from sedona.spark.raster.meta import AffineTransform, PixelAnchor class TestAffineTransform: diff --git a/python/tests/raster/test_pandas_udf.py b/python/tests/raster/test_pandas_udf.py index b33c8370d97..31bf210ebef 100644 --- a/python/tests/raster/test_pandas_udf.py +++ b/python/tests/raster/test_pandas_udf.py @@ -22,7 +22,6 @@ import rasterio from pyspark.sql.functions import expr, pandas_udf from pyspark.sql.types import IntegerType -from tests import world_map_raster_input_location from tests.test_base import TestBase @@ -42,7 +41,7 @@ def test_raster_as_param(self): # A Python Pandas UDF that takes a raster as input @pandas_udf(IntegerType()) def pandas_udf_raster_as_param(s: pd.Series) -> pd.Series: - from sedona.raster import raster_serde + from sedona.spark.raster import raster_serde def func(x): with raster_serde.deserialize(x) as raster: @@ -54,7 +53,7 @@ def func(x): # A Python Pandas UDF that takes a raster as input @pandas_udf(IntegerType()) def pandas_udf_raster_as_param_2(s: pd.Series) -> pd.Series: - from sedona.raster import raster_serde + from sedona.spark.raster import raster_serde def func(x): with raster_serde.deserialize(x) as raster: diff --git a/python/tests/raster/test_serde.py b/python/tests/raster/test_serde.py index 4dfe8b973af..05c0c0e1acc 100644 --- a/python/tests/raster/test_serde.py +++ b/python/tests/raster/test_serde.py @@ -22,7 +22,7 @@ from tests import world_map_raster_input_location from tests.test_base import TestBase -from sedona.sql.types import RasterType +from sedona.spark.sql.types import RasterType class TestRasterSerde(TestBase): diff --git a/python/tests/raster_viz_utils/test_sedonautils.py b/python/tests/raster_viz_utils/test_sedonautils.py index 28f3f91b196..e3202d010e1 100644 --- a/python/tests/raster_viz_utils/test_sedonautils.py +++ b/python/tests/raster_viz_utils/test_sedonautils.py @@ -15,11 +15,10 @@ # specific language governing permissions and limitations # under the License. -from IPython.display import HTML, display from tests import world_map_raster_input_location from tests.test_base import TestBase -from sedona.raster_utils.SedonaUtils import SedonaUtils +from sedona.spark import SedonaUtils class TestSedonaUtils(TestBase): diff --git a/python/tests/serialization/test_direct_serialization.py b/python/tests/serialization/test_direct_serialization.py index c960468c87a..fa87268b667 100644 --- a/python/tests/serialization/test_direct_serialization.py +++ b/python/tests/serialization/test_direct_serialization.py @@ -19,7 +19,7 @@ from shapely.wkt import loads from tests.test_base import TestBase -from sedona.utils.geometry_adapter import GeometryAdapter +from sedona.spark.utils.geometry_adapter import GeometryAdapter class TestDirectSerialization(TestBase): diff --git a/python/tests/serialization/test_rdd_serialization.py b/python/tests/serialization/test_rdd_serialization.py index e82ae1ce8ea..636f1fa697f 100644 --- a/python/tests/serialization/test_rdd_serialization.py +++ b/python/tests/serialization/test_rdd_serialization.py @@ -17,11 +17,11 @@ import os +from tests import tests_resource from tests.test_base import TestBase -from tests.tools import tests_resource -from sedona.core.enums import FileDataSplitter, IndexType -from sedona.core.SpatialRDD import CircleRDD, LineStringRDD, PointRDD, PolygonRDD +from sedona.spark.core.enums import FileDataSplitter, IndexType +from sedona.spark.core.SpatialRDD import LineStringRDD, PointRDD, PolygonRDD point_rdd_input_location = os.path.join(tests_resource, "arealm-small.csv") polygon_rdd_input_location = os.path.join(tests_resource, "primaryroads-polygon.csv") diff --git a/python/tests/serialization/test_serializers.py b/python/tests/serialization/test_serializers.py index 362462c6ee7..1e05dfaee60 100644 --- a/python/tests/serialization/test_serializers.py +++ b/python/tests/serialization/test_serializers.py @@ -32,7 +32,7 @@ from tests import tests_resource from tests.test_base import TestBase -from sedona.sql.types import GeometryType +from sedona.spark.sql.types import GeometryType class TestsSerializers(TestBase): diff --git a/python/tests/serialization/test_with_sc_parellize.py b/python/tests/serialization/test_with_sc_parellize.py index 60852704c20..138e9058592 100644 --- a/python/tests/serialization/test_with_sc_parellize.py +++ b/python/tests/serialization/test_with_sc_parellize.py @@ -17,10 +17,11 @@ from shapely.geometry import LineString, Point, Polygon from shapely.wkt import loads + +from sedona.spark.utils.spatial_rdd_parser import GeoData from tests.test_base import TestBase -from sedona.core.SpatialRDD import LineStringRDD, PointRDD, PolygonRDD -from sedona.utils.spatial_rdd_parser import GeoData +from sedona.spark.core.SpatialRDD import LineStringRDD, PointRDD, PolygonRDD class TestWithScParallelize(TestBase): diff --git a/python/tests/spatial_operator/test_join_base.py b/python/tests/spatial_operator/test_join_base.py index 2aac30fb522..b6f19e2da4c 100644 --- a/python/tests/spatial_operator/test_join_base.py +++ b/python/tests/spatial_operator/test_join_base.py @@ -15,12 +15,16 @@ # specific language governing permissions and limitations # under the License. -import pytest from tests.test_base import TestBase -from sedona.core.enums import GridType -from sedona.core.SpatialRDD import LineStringRDD, PointRDD, PolygonRDD, RectangleRDD -from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD +from sedona.spark.core.enums import GridType +from sedona.spark.core.SpatialRDD import ( + LineStringRDD, + PointRDD, + PolygonRDD, + RectangleRDD, +) +from sedona.spark.core.SpatialRDD.spatial_rdd import SpatialRDD class TestJoinBase(TestBase): diff --git a/python/tests/spatial_operator/test_join_query_correctness.py b/python/tests/spatial_operator/test_join_query_correctness.py index e4995bef443..1659bc93931 100644 --- a/python/tests/spatial_operator/test_join_query_correctness.py +++ b/python/tests/spatial_operator/test_join_query_correctness.py @@ -17,13 +17,14 @@ from shapely.geometry import LineString, Point, Polygon from shapely.geometry.base import BaseGeometry + +from sedona.spark.utils.spatial_rdd_parser import GeoData from tests.test_base import TestBase -from sedona.core.enums import GridType, IndexType -from sedona.core.spatialOperator import JoinQuery -from sedona.core.SpatialRDD import CircleRDD, LineStringRDD, PointRDD, PolygonRDD -from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD -from sedona.utils.spatial_rdd_parser import GeoData +from sedona.spark.core.enums import GridType, IndexType +from sedona.spark.core.spatialOperator import JoinQuery +from sedona.spark.core.SpatialRDD import CircleRDD, LineStringRDD, PointRDD, PolygonRDD +from sedona.spark.core.SpatialRDD.spatial_rdd import SpatialRDD class TestJoinQueryCorrectness(TestBase): diff --git a/python/tests/spatial_operator/test_linestring_join.py b/python/tests/spatial_operator/test_linestring_join.py index af140d41181..b40d5247e28 100644 --- a/python/tests/spatial_operator/test_linestring_join.py +++ b/python/tests/spatial_operator/test_linestring_join.py @@ -17,14 +17,13 @@ import os -import pytest +from tests import tests_resource from tests.spatial_operator.test_join_base import TestJoinBase -from tests.tools import tests_resource -from sedona.core.enums import FileDataSplitter, GridType, IndexType -from sedona.core.enums.join_build_side import JoinBuildSide -from sedona.core.spatialOperator import JoinQuery -from sedona.core.spatialOperator.join_params import JoinParams +from sedona.spark.core.enums import FileDataSplitter, GridType, IndexType +from sedona.spark.core.enums.join_build_side import JoinBuildSide +from sedona.spark.core.spatialOperator import JoinQuery +from sedona.spark.core.spatialOperator.join_params import JoinParams input_location = os.path.join(tests_resource, "primaryroads-linestring.csv") query_window_set = os.path.join(tests_resource, "zcta510-small.csv") diff --git a/python/tests/spatial_operator/test_linestring_knn.py b/python/tests/spatial_operator/test_linestring_knn.py index 8de3ab93c71..f5a9af29da7 100644 --- a/python/tests/spatial_operator/test_linestring_knn.py +++ b/python/tests/spatial_operator/test_linestring_knn.py @@ -18,12 +18,13 @@ import os from shapely.geometry import Point + +from tests import tests_resource from tests.test_base import TestBase -from tests.tools import tests_resource -from sedona.core.enums import FileDataSplitter, IndexType -from sedona.core.spatialOperator import KNNQuery -from sedona.core.SpatialRDD import LineStringRDD +from sedona.spark.core.enums import FileDataSplitter, IndexType +from sedona.spark.core.spatialOperator import KNNQuery +from sedona.spark.core.SpatialRDD import LineStringRDD input_location = os.path.join(tests_resource, "primaryroads-linestring.csv") offset = 0 diff --git a/python/tests/spatial_operator/test_linestring_range.py b/python/tests/spatial_operator/test_linestring_range.py index e6b5df971ab..0f50785f6bf 100644 --- a/python/tests/spatial_operator/test_linestring_range.py +++ b/python/tests/spatial_operator/test_linestring_range.py @@ -17,13 +17,13 @@ import os +from tests import tests_resource from tests.test_base import TestBase -from tests.tools import tests_resource -from sedona.core.enums import FileDataSplitter, IndexType -from sedona.core.geom.envelope import Envelope -from sedona.core.spatialOperator import RangeQuery -from sedona.core.SpatialRDD import LineStringRDD +from sedona.spark.core.enums import FileDataSplitter, IndexType +from sedona.spark.core.geom.envelope import Envelope +from sedona.spark.core.spatialOperator import RangeQuery +from sedona.spark.core.SpatialRDD import LineStringRDD input_location = os.path.join(tests_resource, "primaryroads-linestring.csv") offset = 0 diff --git a/python/tests/spatial_operator/test_point_join.py b/python/tests/spatial_operator/test_point_join.py index dcfdaeb790d..9943f532654 100644 --- a/python/tests/spatial_operator/test_point_join.py +++ b/python/tests/spatial_operator/test_point_join.py @@ -17,14 +17,13 @@ import os -import pytest +from tests import tests_resource from tests.spatial_operator.test_join_base import TestJoinBase -from tests.tools import tests_resource -from sedona.core.enums import FileDataSplitter, GridType, IndexType -from sedona.core.enums.join_build_side import JoinBuildSide -from sedona.core.spatialOperator import JoinQuery -from sedona.core.spatialOperator.join_params import JoinParams +from sedona.spark.core.enums import FileDataSplitter, GridType, IndexType +from sedona.spark.core.enums.join_build_side import JoinBuildSide +from sedona.spark.core.spatialOperator import JoinQuery +from sedona.spark.core.spatialOperator.join_params import JoinParams input_location = os.path.join(tests_resource, "arealm-small.csv") input_location_query_window = os.path.join(tests_resource, "zcta510-small.csv") diff --git a/python/tests/spatial_operator/test_point_knn.py b/python/tests/spatial_operator/test_point_knn.py index 5f5cc951f96..eff101b53ae 100644 --- a/python/tests/spatial_operator/test_point_knn.py +++ b/python/tests/spatial_operator/test_point_knn.py @@ -18,12 +18,14 @@ import os from shapely.geometry import Point + +from tests import tests_resource from tests.test_base import TestBase -from tests.tools import distance_sorting_functions, tests_resource -from sedona.core.enums import FileDataSplitter, IndexType -from sedona.core.spatialOperator import KNNQuery -from sedona.core.SpatialRDD import PointRDD +from sedona.spark.core.enums import FileDataSplitter, IndexType +from sedona.spark.core.spatialOperator import KNNQuery +from sedona.spark.core.SpatialRDD import PointRDD +from tests.tools import distance_sorting_functions input_location = os.path.join(tests_resource, "arealm-small.csv") queryWindowSet = os.path.join(tests_resource, "zcta510-small.csv") diff --git a/python/tests/spatial_operator/test_point_range.py b/python/tests/spatial_operator/test_point_range.py index 0c85af23598..d9fd788e8d0 100644 --- a/python/tests/spatial_operator/test_point_range.py +++ b/python/tests/spatial_operator/test_point_range.py @@ -17,13 +17,13 @@ import os +from tests import tests_resource from tests.test_base import TestBase -from tests.tools import tests_resource -from sedona.core.enums import FileDataSplitter, IndexType -from sedona.core.geom.envelope import Envelope -from sedona.core.spatialOperator import RangeQuery -from sedona.core.SpatialRDD import PointRDD +from sedona.spark.core.enums import FileDataSplitter, IndexType +from sedona.spark.core.geom.envelope import Envelope +from sedona.spark.core.spatialOperator import RangeQuery +from sedona.spark.core.SpatialRDD import PointRDD input_location = os.path.join(tests_resource, "arealm-small.csv") queryWindowSet = os.path.join(tests_resource, "zcta510-small.csv") diff --git a/python/tests/spatial_operator/test_polygon_join.py b/python/tests/spatial_operator/test_polygon_join.py index 70e55e96592..2f4a6688aa9 100644 --- a/python/tests/spatial_operator/test_polygon_join.py +++ b/python/tests/spatial_operator/test_polygon_join.py @@ -17,14 +17,13 @@ import os -import pytest +from tests import tests_resource from tests.spatial_operator.test_join_base import TestJoinBase -from tests.tools import tests_resource -from sedona.core.enums import FileDataSplitter, GridType, IndexType -from sedona.core.enums.join_build_side import JoinBuildSide -from sedona.core.spatialOperator import JoinQuery -from sedona.core.spatialOperator.join_params import JoinParams +from sedona.spark.core.enums import FileDataSplitter, GridType, IndexType +from sedona.spark.core.enums.join_build_side import JoinBuildSide +from sedona.spark.core.spatialOperator import JoinQuery +from sedona.spark.core.spatialOperator.join_params import JoinParams input_location = os.path.join(tests_resource, "primaryroads-polygon.csv") query_window_set = os.path.join(tests_resource, "zcta510-small.csv") diff --git a/python/tests/spatial_operator/test_polygon_knn.py b/python/tests/spatial_operator/test_polygon_knn.py index fd604036c34..4da876a5560 100644 --- a/python/tests/spatial_operator/test_polygon_knn.py +++ b/python/tests/spatial_operator/test_polygon_knn.py @@ -18,12 +18,14 @@ import os from shapely.geometry import Point + +from tests import tests_resource from tests.test_base import TestBase -from tests.tools import distance_sorting_functions, tests_resource +from tests.tools import distance_sorting_functions -from sedona.core.enums import FileDataSplitter, IndexType -from sedona.core.spatialOperator import KNNQuery -from sedona.core.SpatialRDD import PolygonRDD +from sedona.spark.core.enums import FileDataSplitter, IndexType +from sedona.spark.core.spatialOperator import KNNQuery +from sedona.spark.core.SpatialRDD import PolygonRDD input_location = os.path.join(tests_resource, "primaryroads-polygon.csv") splitter = FileDataSplitter.CSV diff --git a/python/tests/spatial_operator/test_polygon_range.py b/python/tests/spatial_operator/test_polygon_range.py index cef018ab45c..f2e359f4339 100644 --- a/python/tests/spatial_operator/test_polygon_range.py +++ b/python/tests/spatial_operator/test_polygon_range.py @@ -17,13 +17,13 @@ import os +from tests import tests_resource from tests.test_base import TestBase -from tests.tools import tests_resource -from sedona.core.enums import FileDataSplitter, IndexType -from sedona.core.geom.envelope import Envelope -from sedona.core.spatialOperator import RangeQuery -from sedona.core.SpatialRDD import PolygonRDD +from sedona.spark.core.enums import FileDataSplitter, IndexType +from sedona.spark.core.geom.envelope import Envelope +from sedona.spark.core.spatialOperator import RangeQuery +from sedona.spark.core.SpatialRDD import PolygonRDD input_location = os.path.join(tests_resource, "primaryroads-polygon.csv") splitter = FileDataSplitter.CSV diff --git a/python/tests/spatial_operator/test_rectangle_join.py b/python/tests/spatial_operator/test_rectangle_join.py index fc69f8d1ce1..eea91796da2 100644 --- a/python/tests/spatial_operator/test_rectangle_join.py +++ b/python/tests/spatial_operator/test_rectangle_join.py @@ -17,14 +17,14 @@ import os +from tests import tests_resource from tests.spatial_operator.test_join_base import TestJoinBase -from tests.tools import tests_resource -from sedona.core.enums import FileDataSplitter, GridType, IndexType -from sedona.core.enums.join_build_side import JoinBuildSide -from sedona.core.geom.envelope import Envelope -from sedona.core.spatialOperator import JoinQuery -from sedona.core.spatialOperator.join_params import JoinParams +from sedona.spark.core.enums import FileDataSplitter, GridType, IndexType +from sedona.spark.core.enums.join_build_side import JoinBuildSide +from sedona.spark.core.geom.envelope import Envelope +from sedona.spark.core.spatialOperator import JoinQuery +from sedona.spark.core.spatialOperator.join_params import JoinParams input_location = os.path.join(tests_resource, "zcta510-small.csv") query_window_set = os.path.join(tests_resource, "zcta510-small.csv") diff --git a/python/tests/spatial_operator/test_rectangle_knn.py b/python/tests/spatial_operator/test_rectangle_knn.py index 793f30f20d8..30a6159d783 100644 --- a/python/tests/spatial_operator/test_rectangle_knn.py +++ b/python/tests/spatial_operator/test_rectangle_knn.py @@ -18,13 +18,15 @@ import os from shapely.geometry import LineString, Point, Polygon + +from tests import tests_resource from tests.test_base import TestBase -from tests.tools import distance_sorting_functions, tests_resource +from tests.tools import distance_sorting_functions -from sedona.core.enums import FileDataSplitter, IndexType -from sedona.core.geom.envelope import Envelope -from sedona.core.spatialOperator import KNNQuery -from sedona.core.SpatialRDD import RectangleRDD +from sedona.spark.core.enums import FileDataSplitter, IndexType +from sedona.spark.core.geom.envelope import Envelope +from sedona.spark.core.spatialOperator import KNNQuery +from sedona.spark.core.SpatialRDD import RectangleRDD inputLocation = os.path.join(tests_resource, "zcta510-small.csv") queryWindowSet = os.path.join(tests_resource, "zcta510-small.csv") diff --git a/python/tests/spatial_operator/test_rectangle_range.py b/python/tests/spatial_operator/test_rectangle_range.py index 8dbde754341..39a75bbd451 100644 --- a/python/tests/spatial_operator/test_rectangle_range.py +++ b/python/tests/spatial_operator/test_rectangle_range.py @@ -17,13 +17,13 @@ import os +from tests import tests_resource from tests.test_base import TestBase -from tests.tools import tests_resource -from sedona.core.enums import FileDataSplitter, IndexType -from sedona.core.geom.envelope import Envelope -from sedona.core.spatialOperator import RangeQuery -from sedona.core.SpatialRDD import RectangleRDD +from sedona.spark.core.enums import FileDataSplitter, IndexType +from sedona.spark.core.geom.envelope import Envelope +from sedona.spark.core.spatialOperator import RangeQuery +from sedona.spark.core.SpatialRDD import RectangleRDD inputLocation = os.path.join(tests_resource, "zcta510-small.csv") queryWindowSet = os.path.join(tests_resource, "zcta510-small.csv") diff --git a/python/tests/spatial_rdd/test_circle_rdd.py b/python/tests/spatial_rdd/test_circle_rdd.py index d0151ec880f..56cd54f1aa2 100644 --- a/python/tests/spatial_rdd/test_circle_rdd.py +++ b/python/tests/spatial_rdd/test_circle_rdd.py @@ -23,7 +23,7 @@ ) from tests.test_base import TestBase -from sedona.core.SpatialRDD import CircleRDD, PointRDD +from sedona.spark.core.SpatialRDD import CircleRDD, PointRDD class TestCircleRDD(TestBase): diff --git a/python/tests/spatial_rdd/test_linestring_rdd.py b/python/tests/spatial_rdd/test_linestring_rdd.py index 04ff76a09ab..4c1746c3d76 100644 --- a/python/tests/spatial_rdd/test_linestring_rdd.py +++ b/python/tests/spatial_rdd/test_linestring_rdd.py @@ -23,14 +23,12 @@ input_location, num_partitions, splitter, - transformed_envelope, - transformed_envelope_2, ) from tests.test_base import TestBase -from sedona.core.enums import GridType, IndexType -from sedona.core.geom.envelope import Envelope -from sedona.core.SpatialRDD import LineStringRDD +from sedona.spark.core.enums import IndexType +from sedona.spark.core.geom.envelope import Envelope +from sedona.spark.core.SpatialRDD import LineStringRDD class TestLineStringRDD(TestBase): diff --git a/python/tests/spatial_rdd/test_point_rdd.py b/python/tests/spatial_rdd/test_point_rdd.py index e21bbf4c52d..0335c9e5abd 100644 --- a/python/tests/spatial_rdd/test_point_rdd.py +++ b/python/tests/spatial_rdd/test_point_rdd.py @@ -17,7 +17,6 @@ from tests.properties.point_properties import ( crs_envelope, - crs_envelope_transformed, crs_point_test, input_boundary, input_count, @@ -25,14 +24,13 @@ num_partitions, offset, splitter, - transformed_envelope, ) from tests.test_base import TestBase -from sedona.core.enums import GridType, IndexType -from sedona.core.geom.envelope import Envelope -from sedona.core.SpatialRDD import PointRDD -from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD +from sedona.spark.core.enums import GridType, IndexType +from sedona.spark.core.geom.envelope import Envelope +from sedona.spark.core.SpatialRDD import PointRDD +from sedona.spark.core.SpatialRDD.spatial_rdd import SpatialRDD class TestPointRDD(TestBase): diff --git a/python/tests/spatial_rdd/test_polygon_rdd.py b/python/tests/spatial_rdd/test_polygon_rdd.py index a038f84ab4b..b030bfec565 100644 --- a/python/tests/spatial_rdd/test_polygon_rdd.py +++ b/python/tests/spatial_rdd/test_polygon_rdd.py @@ -28,15 +28,14 @@ polygon_rdd_input_location, polygon_rdd_splitter, polygon_rdd_start_offset, - query_envelope, splitter, ) from tests.test_base import TestBase -from sedona.core.enums import FileDataSplitter, GridType, IndexType -from sedona.core.geom.envelope import Envelope -from sedona.core.SpatialRDD import PolygonRDD -from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD +from sedona.spark.core.enums import FileDataSplitter, IndexType +from sedona.spark.core.geom.envelope import Envelope +from sedona.spark.core.SpatialRDD import PolygonRDD +from sedona.spark.core.SpatialRDD.spatial_rdd import SpatialRDD class TestPolygonRDD(TestBase): diff --git a/python/tests/spatial_rdd/test_rectangle_rdd.py b/python/tests/spatial_rdd/test_rectangle_rdd.py index 4c67e751ae6..c9641011650 100644 --- a/python/tests/spatial_rdd/test_rectangle_rdd.py +++ b/python/tests/spatial_rdd/test_rectangle_rdd.py @@ -17,13 +17,13 @@ import os -import pytest +from tests import tests_resource from tests.test_base import TestBase -from tests.tools import tests_resource -from sedona.core.enums import FileDataSplitter, GridType, IndexType -from sedona.core.geom.envelope import Envelope -from sedona.core.SpatialRDD import RectangleRDD +from sedona.spark.core.enums import FileDataSplitter, IndexType +from sedona.spark.core.geom.envelope import Envelope +from sedona.spark.core.SpatialRDD import RectangleRDD + inputLocation = os.path.join(tests_resource, "zcta510-small.csv") queryWindowSet = os.path.join(tests_resource, "zcta510-small.csv") diff --git a/python/tests/spatial_rdd/test_spatial_rdd.py b/python/tests/spatial_rdd/test_spatial_rdd.py index 4230b87c53d..f68c429387b 100644 --- a/python/tests/spatial_rdd/test_spatial_rdd.py +++ b/python/tests/spatial_rdd/test_spatial_rdd.py @@ -17,18 +17,18 @@ import os -import pyspark import pytest from pyspark import RDD from shapely.geometry import Point + +from tests import tests_resource from tests.test_base import TestBase -from tests.tools import tests_resource -from sedona.core.enums import FileDataSplitter, GridType, IndexType -from sedona.core.formatMapper.geo_json_reader import GeoJsonReader -from sedona.utils.adapter import Adapter -from sedona.core.geom.envelope import Envelope -from sedona.core.SpatialRDD import PointRDD +from sedona.spark.core.enums import FileDataSplitter, GridType, IndexType +from sedona.spark.core.formatMapper.geo_json_reader import GeoJsonReader +from sedona.spark import Adapter +from sedona.spark.core.geom.envelope import Envelope +from sedona.spark.core.SpatialRDD import PointRDD input_file_location = os.path.join(tests_resource, "arealm-small.csv") crs_test_point = os.path.join(tests_resource, "crs-test-point.csv") diff --git a/python/tests/spatial_rdd/test_spatial_rdd_writer.py b/python/tests/spatial_rdd/test_spatial_rdd_writer.py index 2e5a571c369..b01b1265458 100644 --- a/python/tests/spatial_rdd/test_spatial_rdd_writer.py +++ b/python/tests/spatial_rdd/test_spatial_rdd_writer.py @@ -19,12 +19,13 @@ import shutil import pytest + +from tests import tests_resource from tests.test_base import TestBase -from tests.tools import tests_resource -from sedona.core.enums import FileDataSplitter -from sedona.core.geom.envelope import Envelope -from sedona.core.SpatialRDD import PointRDD +from sedona.spark.core.enums import FileDataSplitter +from sedona.spark.core.geom.envelope import Envelope +from sedona.spark.core.SpatialRDD import PointRDD wkb_folder = "wkb" wkt_folder = "wkt" diff --git a/python/tests/sql/resource/sample_data.py b/python/tests/sql/resource/sample_data.py index d907f66c48a..7c3b27fa6f5 100644 --- a/python/tests/sql/resource/sample_data.py +++ b/python/tests/sql/resource/sample_data.py @@ -23,7 +23,7 @@ from pyspark.sql.types import StructField, StructType from shapely import wkt -from sedona.sql.types import GeometryType +from sedona.spark.sql.types import GeometryType data_path = path.abspath(path.dirname(__file__)) diff --git a/python/tests/sql/test_adapter.py b/python/tests/sql/test_adapter.py index 1c8ff34121c..396d7948a37 100644 --- a/python/tests/sql/test_adapter.py +++ b/python/tests/sql/test_adapter.py @@ -17,7 +17,6 @@ import logging -import pyspark import pytest from pyspark.sql import DataFrame from pyspark.sql.functions import col, expr @@ -32,13 +31,15 @@ from tests.test_base import TestBase from sedona import version -from sedona.core.enums import FileDataSplitter, GridType, IndexType -from sedona.core.formatMapper.shapefileParser.shape_file_reader import ShapefileReader -from sedona.core.geom.envelope import Envelope -from sedona.core.jvm.config import is_greater_or_equal_version -from sedona.core.spatialOperator import JoinQuery -from sedona.core.SpatialRDD import CircleRDD, PolygonRDD -from sedona.utils.adapter import Adapter +from sedona.spark.core.enums import FileDataSplitter, GridType, IndexType +from sedona.spark.core.formatMapper.shapefileParser.shape_file_reader import ( + ShapefileReader, +) +from sedona.spark.core.geom.envelope import Envelope +from sedona.spark.core.jvm.config import is_greater_or_equal_version +from sedona.spark.core.spatialOperator import JoinQuery +from sedona.spark.core.SpatialRDD import CircleRDD, PolygonRDD +from sedona.spark import Adapter class TestAdapter(TestBase): diff --git a/python/tests/sql/test_dataframe_api.py b/python/tests/sql/test_dataframe_api.py index 22ee40c799d..6676063f7de 100644 --- a/python/tests/sql/test_dataframe_api.py +++ b/python/tests/sql/test_dataframe_api.py @@ -16,7 +16,6 @@ # under the License. from math import radians import os -import threading import concurrent.futures from typing import Callable, Tuple @@ -26,15 +25,12 @@ from shapely.geometry.base import BaseGeometry from tests.test_base import TestBase -from sedona.core.geom.geography import Geography -from sedona.sql import st_aggregates as sta -from sedona.sql import st_constructors as stc -from sedona.sql import st_functions as stf -from sedona.sql import st_predicates as stp -from sedona.sql.st_aggregates import * -from sedona.sql.st_constructors import * -from sedona.sql.st_functions import * -from sedona.sql.st_predicates import * +from sedona.spark.core.geom.geography import Geography +from sedona.spark.sql import st_aggregates as sta +from sedona.spark.sql import st_constructors as stc +from sedona.spark.sql import st_functions as stf +from sedona.spark.sql import st_predicates as stp +from sedona.spark.sql.st_functions import * test_configurations = [ # constructors diff --git a/python/tests/sql/test_function.py b/python/tests/sql/test_function.py index 93ab59f24fd..0d6ac29330b 100644 --- a/python/tests/sql/test_function.py +++ b/python/tests/sql/test_function.py @@ -34,7 +34,7 @@ ) from tests.test_base import TestBase -from sedona.sql.types import GeometryType +from sedona.spark.sql.types import GeometryType class TestPredicateJoin(TestBase): diff --git a/python/tests/sql/test_geography.py b/python/tests/sql/test_geography.py index 3105b6a8b40..5721248342b 100644 --- a/python/tests/sql/test_geography.py +++ b/python/tests/sql/test_geography.py @@ -15,12 +15,11 @@ # specific language governing permissions and limitations # under the License. -import pytest from pyspark.sql.functions import expr from pyspark.sql.types import StructType from shapely.wkt import loads as wkt_loads -from sedona.core.geom.geography import Geography -from sedona.sql.types import GeographyType +from sedona.spark.core.geom.geography import Geography +from sedona.spark.sql.types import GeographyType from tests.test_base import TestBase diff --git a/python/tests/sql/test_shapefile.py b/python/tests/sql/test_shapefile.py index a6d5a578c6b..46629360f4f 100644 --- a/python/tests/sql/test_shapefile.py +++ b/python/tests/sql/test_shapefile.py @@ -18,9 +18,8 @@ import datetime import os.path -import pytest +from tests import tests_resource from tests.test_base import TestBase -from tests.tools import tests_resource class TestShapefile(TestBase): diff --git a/python/tests/sql/test_spatial_rdd_to_spatial_dataframe.py b/python/tests/sql/test_spatial_rdd_to_spatial_dataframe.py index 66bbaa71c81..cb1acb46e60 100644 --- a/python/tests/sql/test_spatial_rdd_to_spatial_dataframe.py +++ b/python/tests/sql/test_spatial_rdd_to_spatial_dataframe.py @@ -19,12 +19,13 @@ from pyspark.sql.types import IntegerType, StringType, StructField, StructType from shapely.geometry import Point + +from tests import tests_resource from tests.test_base import TestBase -from tests.tools import tests_resource -from sedona.core.enums import FileDataSplitter -from sedona.core.SpatialRDD import PointRDD -from sedona.sql.types import GeometryType +from sedona.spark.core.enums import FileDataSplitter +from sedona.spark.core.SpatialRDD import PointRDD +from sedona.spark.sql.types import GeometryType point_input_path = os.path.join(tests_resource, "arealm-small.csv") diff --git a/python/tests/sql/test_st_function_imports.py b/python/tests/sql/test_st_function_imports.py index f0c38f6b078..c09d1b8ae50 100644 --- a/python/tests/sql/test_st_function_imports.py +++ b/python/tests/sql/test_st_function_imports.py @@ -22,7 +22,12 @@ class TestStFunctionImport(TestBase): def test_import(self): - from sedona.sql import ST_Contains, ST_Distance, ST_Envelope_Aggr, ST_Point + from sedona.spark.sql import ( + ST_Contains, + ST_Distance, + ST_Envelope_Aggr, + ST_Point, + ) ST_Distance ST_Point diff --git a/python/tests/sql/test_structured_adapter.py b/python/tests/sql/test_structured_adapter.py index f14a48f33ef..757bc465500 100644 --- a/python/tests/sql/test_structured_adapter.py +++ b/python/tests/sql/test_structured_adapter.py @@ -19,10 +19,10 @@ from pyspark.sql import DataFrame -from sedona.core.SpatialRDD import CircleRDD -from sedona.core.enums import GridType -from sedona.core.spatialOperator import JoinQueryRaw -from sedona.utils.structured_adapter import StructuredAdapter +from sedona.spark.core.SpatialRDD import CircleRDD +from sedona.spark.core.enums import GridType +from sedona.spark.core.spatialOperator import JoinQueryRaw +from sedona.spark.utils.structured_adapter import StructuredAdapter from tests.test_base import TestBase diff --git a/python/tests/stac/test_client.py b/python/tests/stac/test_client.py index b8b2beed8ac..636cfcef8ee 100644 --- a/python/tests/stac/test_client.py +++ b/python/tests/stac/test_client.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -from sedona.stac.client import Client +from sedona.spark.stac.client import Client from pyspark.sql import DataFrame from tests.test_base import TestBase @@ -124,8 +124,6 @@ def test_search_with_single_datetime(self) -> None: assert len(list(items)) == 0 def test_search_with_YYYY(self) -> None: - from datetime import datetime - client = Client.open(STAC_URLS["PLANETARY-COMPUTER"]) items = client.search( collection_id="aster-l1t", diff --git a/python/tests/stac/test_collection_client.py b/python/tests/stac/test_collection_client.py index aa9a15f4eb6..bc08cabcd50 100644 --- a/python/tests/stac/test_collection_client.py +++ b/python/tests/stac/test_collection_client.py @@ -15,8 +15,8 @@ # specific language governing permissions and limitations # under the License. -from sedona.stac.client import Client -from sedona.stac.collection_client import CollectionClient +from sedona.spark.stac.client import Client +from sedona.spark.stac.collection_client import CollectionClient from tests.test_base import TestBase diff --git a/python/tests/stats/test_dbscan.py b/python/tests/stats/test_dbscan.py index 29080d93fb9..1b74044755b 100644 --- a/python/tests/stats/test_dbscan.py +++ b/python/tests/stats/test_dbscan.py @@ -20,9 +20,9 @@ from sklearn.cluster import DBSCAN as sklearnDBSCAN from tests.test_base import TestBase -from sedona.sql.st_constructors import ST_MakePoint -from sedona.sql.st_functions import ST_Buffer -from sedona.stats.clustering.dbscan import dbscan +from sedona.spark.sql.st_constructors import ST_MakePoint +from sedona.spark.sql.st_functions import ST_Buffer +from sedona.spark.stats import dbscan class TestDBScan(TestBase): diff --git a/python/tests/stats/test_getis_ord.py b/python/tests/stats/test_getis_ord.py index d40ef4056ad..27cbc25de88 100644 --- a/python/tests/stats/test_getis_ord.py +++ b/python/tests/stats/test_getis_ord.py @@ -18,11 +18,12 @@ from esda.getisord import G_Local from libpysal.weights import DistanceBand from pyspark.sql import functions as f + +from sedona.spark.stats import g_local +from sedona.spark.stats import add_distance_band_column from tests.test_base import TestBase -from sedona.sql.st_constructors import ST_MakePoint -from sedona.stats.hotspot_detection.getis_ord import g_local -from sedona.stats.weighting import add_distance_band_column +from sedona.spark.sql.st_constructors import ST_MakePoint class TestGetisOrd(TestBase): diff --git a/python/tests/stats/test_local_outlier_factor.py b/python/tests/stats/test_local_outlier_factor.py index 52ec860a05d..ed03af90c48 100644 --- a/python/tests/stats/test_local_outlier_factor.py +++ b/python/tests/stats/test_local_outlier_factor.py @@ -23,9 +23,9 @@ from sklearn.neighbors import LocalOutlierFactor from tests.test_base import TestBase -from sedona.sql.st_constructors import ST_MakePoint -from sedona.sql.st_functions import ST_X, ST_Y -from sedona.stats.outlier_detection.local_outlier_factor import local_outlier_factor +from sedona.spark.sql.st_constructors import ST_MakePoint +from sedona.spark.sql.st_functions import ST_X, ST_Y +from sedona.spark.stats import local_outlier_factor class TestLOF(TestBase): diff --git a/python/tests/stats/test_weighting.py b/python/tests/stats/test_weighting.py index 5dadcaef46b..82b30a208d5 100644 --- a/python/tests/stats/test_weighting.py +++ b/python/tests/stats/test_weighting.py @@ -18,8 +18,8 @@ import pyspark.sql.functions as f from tests.test_base import TestBase -from sedona.sql.st_constructors import ST_MakePoint -from sedona.stats.weighting import ( +from sedona.spark.sql.st_constructors import ST_MakePoint +from sedona.spark.stats.weighting import ( add_binary_distance_band_column, add_distance_band_column, ) diff --git a/python/tests/streaming/spark/test_constructor_functions.py b/python/tests/streaming/spark/test_constructor_functions.py index 354a5ff2a1b..2cfe2cf1c4c 100644 --- a/python/tests/streaming/spark/test_constructor_functions.py +++ b/python/tests/streaming/spark/test_constructor_functions.py @@ -27,7 +27,7 @@ from tests.streaming.spark.cases_builder import SuiteContainer from tests.test_base import TestBase -from sedona.sql.types import GeometryType +from sedona.spark.sql.types import GeometryType SCHEMA = StructType([StructField("geom", GeometryType())]) diff --git a/python/tests/test_assign_raw_spatial_rdd.py b/python/tests/test_assign_raw_spatial_rdd.py index 303ded9ace3..1a8b1104987 100644 --- a/python/tests/test_assign_raw_spatial_rdd.py +++ b/python/tests/test_assign_raw_spatial_rdd.py @@ -23,7 +23,7 @@ ) from tests.test_base import TestBase -from sedona.core.SpatialRDD import CircleRDD, PointRDD +from sedona.spark.core.SpatialRDD import CircleRDD, PointRDD class TestSpatialRddAssignment(TestBase): diff --git a/python/tests/test_base.py b/python/tests/test_base.py index 2582b9d4bf3..d2794ac8aae 100644 --- a/python/tests/test_base.py +++ b/python/tests/test_base.py @@ -23,7 +23,7 @@ from pyspark.sql import DataFrame from sedona.spark import * -from sedona.utils.decorators import classproperty +from sedona.spark.utils.decorators import classproperty SPARK_REMOTE = os.getenv("SPARK_REMOTE") EXTRA_JARS = os.getenv("SEDONA_PYTHON_EXTRA_JARS") diff --git a/python/tests/test_circle.py b/python/tests/test_circle.py index 688a3996308..508de9728ac 100644 --- a/python/tests/test_circle.py +++ b/python/tests/test_circle.py @@ -20,8 +20,8 @@ from shapely import wkt from shapely.geometry import Point -from sedona.core.geom.circle import Circle -from sedona.core.geom.envelope import Envelope +from sedona.spark.core.geom.circle import Circle +from sedona.spark.core.geom.envelope import Envelope class TestCircle: diff --git a/python/tests/test_multiple_meta.py b/python/tests/test_multiple_meta.py index c37cc72e82f..accaa5070b5 100644 --- a/python/tests/test_multiple_meta.py +++ b/python/tests/test_multiple_meta.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -from sedona.utils.meta import MultipleMeta +from sedona.spark.utils.meta import MultipleMeta class TestMultipleMeta: diff --git a/python/tests/test_scala_example.py b/python/tests/test_scala_example.py index d2e1a33502b..76c4831ed36 100644 --- a/python/tests/test_scala_example.py +++ b/python/tests/test_scala_example.py @@ -19,13 +19,14 @@ from pyspark import StorageLevel from shapely.geometry import Point + +from tests import tests_resource from tests.test_base import TestBase -from tests.tools import tests_resource -from sedona.core.enums import FileDataSplitter, GridType, IndexType -from sedona.core.geom.envelope import Envelope -from sedona.core.spatialOperator import JoinQuery, KNNQuery, RangeQuery -from sedona.core.SpatialRDD import CircleRDD, PointRDD, PolygonRDD +from sedona.spark.core.enums import FileDataSplitter, GridType, IndexType +from sedona.spark.core.geom.envelope import Envelope +from sedona.spark.core.spatialOperator import JoinQuery, KNNQuery, RangeQuery +from sedona.spark.core.SpatialRDD import CircleRDD, PointRDD, PolygonRDD point_rdd_input_location = os.path.join(tests_resource, "arealm-small.csv") @@ -77,207 +78,208 @@ def test_spatial_range_query(self): object_rdd, range_query_window, False, False ).count() - def test_spatial_range_query_using_index(self): - object_rdd = PointRDD( - self.sc, - point_rdd_input_location, - point_rdd_offset, - point_rdd_splitter, - True, - ) - object_rdd.buildIndex(point_rdd_index_type, False) - object_rdd.indexedRawRDD.persist(StorageLevel.MEMORY_ONLY) - assert object_rdd.indexedRawRDD.is_cached - - for _ in range(each_query_loop_times): - result_size = RangeQuery.SpatialRangeQuery( - object_rdd, range_query_window, False, True - ).count - - def test_spatial_knn_query(self): - object_rdd = PointRDD( - self.sc, - point_rdd_input_location, - point_rdd_offset, - point_rdd_splitter, - True, - ) - object_rdd.rawJvmSpatialRDD.persist(StorageLevel.MEMORY_ONLY) - - for _ in range(each_query_loop_times): - result = KNNQuery.SpatialKnnQuery(object_rdd, knn_query_point, 1000, False) - - def test_spatial_knn_query_using_index(self): - object_rdd = PointRDD( - self.sc, - point_rdd_input_location, - point_rdd_offset, - point_rdd_splitter, - True, - ) - object_rdd.buildIndex(point_rdd_index_type, False) - object_rdd.indexedRawRDD.persist(StorageLevel.MEMORY_ONLY) - - for _ in range(each_query_loop_times): - result = KNNQuery.SpatialKnnQuery(object_rdd, knn_query_point, 1000, True) - - def test_spatial_join_query(self): - query_window_rdd = PolygonRDD( - self.sc, - polygon_rdd_input_location, - polygon_rdd_start_offset, - polygon_rdd_end_offset, - polygon_rdd_splitter, - True, - ) - object_rdd = PointRDD( - self.sc, - point_rdd_input_location, - point_rdd_offset, - point_rdd_splitter, - True, - ) - - object_rdd.spatialPartitioning(join_query_partitioning_type) - query_window_rdd.spatialPartitioning(object_rdd.getPartitioner()) - - object_rdd.jvmSpatialPartitionedRDD.persist(StorageLevel.MEMORY_ONLY) - query_window_rdd.jvmSpatialPartitionedRDD.persist(StorageLevel.MEMORY_ONLY) - - for _ in range(each_query_loop_times): - result_size = JoinQuery.SpatialJoinQuery( - object_rdd, query_window_rdd, False, True - ).count() - - def test_spatial_join_using_index(self): - query_window_rdd = PolygonRDD( - self.sc, - polygon_rdd_input_location, - polygon_rdd_start_offset, - polygon_rdd_end_offset, - polygon_rdd_splitter, - True, - ) - object_rdd = PointRDD( - self.sc, - point_rdd_input_location, - point_rdd_offset, - point_rdd_splitter, - True, - ) - - object_rdd.spatialPartitioning(join_query_partitioning_type) - query_window_rdd.spatialPartitioning(object_rdd.getPartitioner()) - - object_rdd.buildIndex(point_rdd_index_type, True) - - object_rdd.indexedRDD.persist(StorageLevel.MEMORY_ONLY) - query_window_rdd.jvmSpatialPartitionedRDD.persist(StorageLevel.MEMORY_ONLY) - - for _ in range(each_query_loop_times): - result_size = JoinQuery.SpatialJoinQuery( - object_rdd, query_window_rdd, True, False - ).count() - - def test_distance_join_query(self): - object_rdd = PointRDD( - self.sc, - point_rdd_input_location, - point_rdd_offset, - point_rdd_splitter, - True, - ) - query_window_rdd = CircleRDD(object_rdd, 0.1) - - object_rdd.spatialPartitioning(GridType.QUADTREE) - query_window_rdd.spatialPartitioning(object_rdd.getPartitioner()) - - object_rdd.spatialPartitionedRDD.persist(StorageLevel.MEMORY_ONLY) - assert object_rdd.spatialPartitionedRDD.is_cached - - query_window_rdd.spatialPartitionedRDD.persist(StorageLevel.MEMORY_ONLY) - - for _ in range(each_query_loop_times): - result_size = JoinQuery.DistanceJoinQuery( - object_rdd, query_window_rdd, False, True - ).count() - - def test_distance_join_using_index(self): - object_rdd = PointRDD( - self.sc, - point_rdd_input_location, - point_rdd_offset, - point_rdd_splitter, - True, - ) - - query_window_rdd = CircleRDD(object_rdd, 0.1) - - object_rdd.spatialPartitioning(GridType.QUADTREE) - query_window_rdd.spatialPartitioning(object_rdd.getPartitioner()) - - object_rdd.buildIndex(IndexType.RTREE, True) - - object_rdd.indexedRDD.persist(StorageLevel.MEMORY_ONLY) - query_window_rdd.spatialPartitionedRDD.persist(StorageLevel.MEMORY_ONLY) - assert object_rdd.indexedRDD.is_cached - assert query_window_rdd.spatialPartitionedRDD.is_cached - - for _ in range(each_query_loop_times): - result_size = JoinQuery.DistanceJoinQuery( - object_rdd, query_window_rdd, True, True - ).count() - - def test_indexed_rdd_assignment(self): - object_rdd = PointRDD( - self.sc, - point_rdd_input_location, - point_rdd_offset, - point_rdd_splitter, - True, - ) - query_window_rdd = CircleRDD(object_rdd, 0.1) - object_rdd.analyze() - object_rdd.spatialPartitioning(GridType.QUADTREE) - object_rdd.buildIndex(IndexType.QUADTREE, True) - - query_window_rdd.spatialPartitioning(object_rdd.getPartitioner()) - - object_rdd.buildIndex(IndexType.RTREE, False) - - object_rdd.indexedRDD.persist(StorageLevel.MEMORY_ONLY) - query_window_rdd.jvmSpatialPartitionedRDD.persist(StorageLevel.MEMORY_ONLY) - query_window_rdd.jvmSpatialPartitionedRDD.count() - object_rdd.indexedRDD.count() - - import time - - start = time.time() - for _ in range(each_query_loop_times): - result_size = JoinQuery.DistanceJoinQuery( - object_rdd, query_window_rdd, True, True - ).count() - diff = time.time() - start - - object_rdd = PointRDD( - self.sc, - point_rdd_input_location, - point_rdd_offset, - point_rdd_splitter, - True, - ) - query_window_rdd = CircleRDD(object_rdd, 0.1) - - object_rdd.analyze() - object_rdd.spatialPartitioning(GridType.QUADTREE) - object_rdd.buildIndex(IndexType.QUADTREE, True) - - query_window_rdd.spatialPartitioning(object_rdd.getPartitioner()) - - object_rdd.buildIndex(IndexType.RTREE, False) - - start1 = time.time() - for _ in range(each_query_loop_times): - result_size = JoinQuery.DistanceJoinQuery( - object_rdd, query_window_rdd, True, True - ).count() + # + # def test_spatial_range_query_using_index(self): + # object_rdd = PointRDD( + # self.sc, + # point_rdd_input_location, + # point_rdd_offset, + # point_rdd_splitter, + # True, + # ) + # object_rdd.buildIndex(point_rdd_index_type, False) + # object_rdd.indexedRawRDD.persist(StorageLevel.MEMORY_ONLY) + # assert object_rdd.indexedRawRDD.is_cached + # + # for _ in range(each_query_loop_times): + # result_size = RangeQuery.SpatialRangeQuery( + # object_rdd, range_query_window, False, True + # ).count + # + # def test_spatial_knn_query(self): + # object_rdd = PointRDD( + # self.sc, + # point_rdd_input_location, + # point_rdd_offset, + # point_rdd_splitter, + # True, + # ) + # object_rdd.rawJvmSpatialRDD.persist(StorageLevel.MEMORY_ONLY) + # + # for _ in range(each_query_loop_times): + # result = KNNQuery.SpatialKnnQuery(object_rdd, knn_query_point, 1000, False) + # + # def test_spatial_knn_query_using_index(self): + # object_rdd = PointRDD( + # self.sc, + # point_rdd_input_location, + # point_rdd_offset, + # point_rdd_splitter, + # True, + # ) + # object_rdd.buildIndex(point_rdd_index_type, False) + # object_rdd.indexedRawRDD.persist(StorageLevel.MEMORY_ONLY) + # + # for _ in range(each_query_loop_times): + # result = KNNQuery.SpatialKnnQuery(object_rdd, knn_query_point, 1000, True) + # + # def test_spatial_join_query(self): + # query_window_rdd = PolygonRDD( + # self.sc, + # polygon_rdd_input_location, + # polygon_rdd_start_offset, + # polygon_rdd_end_offset, + # polygon_rdd_splitter, + # True, + # ) + # object_rdd = PointRDD( + # self.sc, + # point_rdd_input_location, + # point_rdd_offset, + # point_rdd_splitter, + # True, + # ) + # + # object_rdd.spatialPartitioning(join_query_partitioning_type) + # query_window_rdd.spatialPartitioning(object_rdd.getPartitioner()) + # + # object_rdd.jvmSpatialPartitionedRDD.persist(StorageLevel.MEMORY_ONLY) + # query_window_rdd.jvmSpatialPartitionedRDD.persist(StorageLevel.MEMORY_ONLY) + # + # for _ in range(each_query_loop_times): + # result_size = JoinQuery.SpatialJoinQuery( + # object_rdd, query_window_rdd, False, True + # ).count() + # + # def test_spatial_join_using_index(self): + # query_window_rdd = PolygonRDD( + # self.sc, + # polygon_rdd_input_location, + # polygon_rdd_start_offset, + # polygon_rdd_end_offset, + # polygon_rdd_splitter, + # True, + # ) + # object_rdd = PointRDD( + # self.sc, + # point_rdd_input_location, + # point_rdd_offset, + # point_rdd_splitter, + # True, + # ) + # + # object_rdd.spatialPartitioning(join_query_partitioning_type) + # query_window_rdd.spatialPartitioning(object_rdd.getPartitioner()) + # + # object_rdd.buildIndex(point_rdd_index_type, True) + # + # object_rdd.indexedRDD.persist(StorageLevel.MEMORY_ONLY) + # query_window_rdd.jvmSpatialPartitionedRDD.persist(StorageLevel.MEMORY_ONLY) + # + # for _ in range(each_query_loop_times): + # result_size = JoinQuery.SpatialJoinQuery( + # object_rdd, query_window_rdd, True, False + # ).count() + # + # def test_distance_join_query(self): + # object_rdd = PointRDD( + # self.sc, + # point_rdd_input_location, + # point_rdd_offset, + # point_rdd_splitter, + # True, + # ) + # query_window_rdd = CircleRDD(object_rdd, 0.1) + # + # object_rdd.spatialPartitioning(GridType.QUADTREE) + # query_window_rdd.spatialPartitioning(object_rdd.getPartitioner()) + # + # object_rdd.spatialPartitionedRDD.persist(StorageLevel.MEMORY_ONLY) + # assert object_rdd.spatialPartitionedRDD.is_cached + # + # query_window_rdd.spatialPartitionedRDD.persist(StorageLevel.MEMORY_ONLY) + # + # for _ in range(each_query_loop_times): + # result_size = JoinQuery.DistanceJoinQuery( + # object_rdd, query_window_rdd, False, True + # ).count() + # + # def test_distance_join_using_index(self): + # object_rdd = PointRDD( + # self.sc, + # point_rdd_input_location, + # point_rdd_offset, + # point_rdd_splitter, + # True, + # ) + # + # query_window_rdd = CircleRDD(object_rdd, 0.1) + # + # object_rdd.spatialPartitioning(GridType.QUADTREE) + # query_window_rdd.spatialPartitioning(object_rdd.getPartitioner()) + # + # object_rdd.buildIndex(IndexType.RTREE, True) + # + # object_rdd.indexedRDD.persist(StorageLevel.MEMORY_ONLY) + # query_window_rdd.spatialPartitionedRDD.persist(StorageLevel.MEMORY_ONLY) + # assert object_rdd.indexedRDD.is_cached + # assert query_window_rdd.spatialPartitionedRDD.is_cached + # + # for _ in range(each_query_loop_times): + # result_size = JoinQuery.DistanceJoinQuery( + # object_rdd, query_window_rdd, True, True + # ).count() + # + # def test_indexed_rdd_assignment(self): + # object_rdd = PointRDD( + # self.sc, + # point_rdd_input_location, + # point_rdd_offset, + # point_rdd_splitter, + # True, + # ) + # query_window_rdd = CircleRDD(object_rdd, 0.1) + # object_rdd.analyze() + # object_rdd.spatialPartitioning(GridType.QUADTREE) + # object_rdd.buildIndex(IndexType.QUADTREE, True) + # + # query_window_rdd.spatialPartitioning(object_rdd.getPartitioner()) + # + # object_rdd.buildIndex(IndexType.RTREE, False) + # + # object_rdd.indexedRDD.persist(StorageLevel.MEMORY_ONLY) + # query_window_rdd.jvmSpatialPartitionedRDD.persist(StorageLevel.MEMORY_ONLY) + # query_window_rdd.jvmSpatialPartitionedRDD.count() + # object_rdd.indexedRDD.count() + # + # import time + # + # start = time.time() + # for _ in range(each_query_loop_times): + # result_size = JoinQuery.DistanceJoinQuery( + # object_rdd, query_window_rdd, True, True + # ).count() + # diff = time.time() - start + # + # object_rdd = PointRDD( + # self.sc, + # point_rdd_input_location, + # point_rdd_offset, + # point_rdd_splitter, + # True, + # ) + # query_window_rdd = CircleRDD(object_rdd, 0.1) + # + # object_rdd.analyze() + # object_rdd.spatialPartitioning(GridType.QUADTREE) + # object_rdd.buildIndex(IndexType.QUADTREE, True) + # + # query_window_rdd.spatialPartitioning(object_rdd.getPartitioner()) + # + # object_rdd.buildIndex(IndexType.RTREE, False) + # + # start1 = time.time() + # for _ in range(each_query_loop_times): + # result_size = JoinQuery.DistanceJoinQuery( + # object_rdd, query_window_rdd, True, True + # ).count() diff --git a/python/tests/tools.py b/python/tests/tools.py index 0e9be54aecb..bd454d3824a 100644 --- a/python/tests/tools.py +++ b/python/tests/tools.py @@ -14,17 +14,9 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - -from os import path - from shapely.geometry import Point -from sedona.utils.spatial_rdd_parser import GeoData - -tests_path = path.abspath(path.join(__file__, "../../../spark/common/src/test/")) -tests_resource = path.abspath( - path.join(__file__, "../../../spark/common/src/test/resources/") -) +from sedona.spark.utils.spatial_rdd_parser import GeoData def distance_sorting_functions(geo_data: GeoData, query_point: Point): diff --git a/python/tests/utils/test_arrow_conversion_geopandas_to_sedona.py b/python/tests/utils/test_arrow_conversion_geopandas_to_sedona.py index 91314c74061..8afa92fffec 100644 --- a/python/tests/utils/test_arrow_conversion_geopandas_to_sedona.py +++ b/python/tests/utils/test_arrow_conversion_geopandas_to_sedona.py @@ -17,8 +17,8 @@ import pytest -from sedona.sql.types import GeometryType -from sedona.utils.geoarrow import create_spatial_dataframe +from sedona.spark.sql.types import GeometryType +from sedona.spark.geoarrow.geoarrow import create_spatial_dataframe from tests.test_base import TestBase import geopandas as gpd import pyspark diff --git a/python/tests/utils/test_crs_transformation.py b/python/tests/utils/test_crs_transformation.py index b751857fbff..b4a57cf602a 100644 --- a/python/tests/utils/test_crs_transformation.py +++ b/python/tests/utils/test_crs_transformation.py @@ -15,15 +15,17 @@ # specific language governing permissions and limitations # under the License. -from tests.properties.crs_transform import * -from tests.properties.polygon_properties import grid_type +from tests.properties.crs_transform import ( + input_location, + offset, + splitter, + loop_times, + query_envelope, +) from tests.test_base import TestBase -from tests.tools import distance_sorting_functions -from sedona.core.enums import GridType -from sedona.core.geom.circle import Circle -from sedona.core.spatialOperator import JoinQuery, KNNQuery, RangeQuery -from sedona.core.SpatialRDD import CircleRDD, PointRDD, PolygonRDD +from sedona.spark.core.spatialOperator import RangeQuery +from sedona.spark.core.SpatialRDD import PointRDD class TestCrsTransformation(TestBase): diff --git a/python/tests/utils/test_geo_spark_meta.py b/python/tests/utils/test_geo_spark_meta.py index 14f6464a8d3..4ee1b34ddd9 100644 --- a/python/tests/utils/test_geo_spark_meta.py +++ b/python/tests/utils/test_geo_spark_meta.py @@ -17,7 +17,7 @@ from tests.test_base import TestBase -from sedona.core.jvm.config import SedonaMeta, is_greater_or_equal_version +from sedona.spark.core.jvm.config import SedonaMeta, is_greater_or_equal_version class TestGeoSparkMeta(TestBase): diff --git a/python/tests/utils/test_geoarrow.py b/python/tests/utils/test_geoarrow.py index 28499c401ca..6ed3b084865 100644 --- a/python/tests/utils/test_geoarrow.py +++ b/python/tests/utils/test_geoarrow.py @@ -22,7 +22,7 @@ import pytest from tests.test_base import TestBase from pyspark.sql.types import StringType, StructType -from sedona.utils.geoarrow import ( +from sedona.spark.geoarrow.geoarrow import ( dataframe_to_arrow, unique_srid_from_ewkb, wrap_geoarrow_field, diff --git a/python/tests/utils/test_geometry_serde.py b/python/tests/utils/test_geometry_serde.py index 661aa169dd1..7be419ea647 100644 --- a/python/tests/utils/test_geometry_serde.py +++ b/python/tests/utils/test_geometry_serde.py @@ -29,7 +29,7 @@ from shapely.wkt import loads as wkt_loads from tests.test_base import TestBase -from sedona.sql.types import GeometryType +from sedona.spark.sql.types import GeometryType class TestGeometrySerde(TestBase): diff --git a/python/tests/utils/test_geomserde_speedup.py b/python/tests/utils/test_geomserde_speedup.py index ea2edc4a1e9..f4134ff76cc 100644 --- a/python/tests/utils/test_geomserde_speedup.py +++ b/python/tests/utils/test_geomserde_speedup.py @@ -27,7 +27,7 @@ from shapely.geometry.base import BaseGeometry from shapely.wkt import loads as wkt_loads -from sedona.utils import geometry_serde +from sedona.spark.utils import geometry_serde class TestGeomSerdeSpeedup: diff --git a/python/tests/utils/test_pandas_arrow_udf.py b/python/tests/utils/test_pandas_arrow_udf.py index c0d723d214a..6db0b25268c 100644 --- a/python/tests/utils/test_pandas_arrow_udf.py +++ b/python/tests/utils/test_pandas_arrow_udf.py @@ -16,8 +16,8 @@ # under the License. -from sedona.sql.types import GeometryType -from sedona.sql.functions import sedona_vectorized_udf, SedonaUDFType +from sedona.spark.sql.types import GeometryType +from sedona.spark.sql.functions import sedona_vectorized_udf, SedonaUDFType from tests import chicago_crimes_input_location from tests.test_base import TestBase import pyspark.sql.functions as f diff --git a/spark-version-converter.py b/spark-version-converter.py index 8da0d7c2e8a..f08f6f95573 100644 --- a/spark-version-converter.py +++ b/spark-version-converter.py @@ -1,19 +1,19 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # Python 3 import fileinput diff --git a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/GeographyUDT.scala b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/GeographyUDT.scala index 18ad4dfebde..f9eea7f98cb 100644 --- a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/GeographyUDT.scala +++ b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/GeographyUDT.scala @@ -28,7 +28,7 @@ import org.apache.sedona.common.geometryObjects.Geography; class GeographyUDT extends UserDefinedType[Geography] { override def sqlType: DataType = BinaryType - override def pyUDT: String = "sedona.sql.types.GeographyType" + override def pyUDT: String = "sedona.spark.sql.types.GeographyType" override def userClass: Class[Geography] = classOf[Geography] diff --git a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/GeometryUDT.scala b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/GeometryUDT.scala index 65f347307c3..20ed85d52ec 100644 --- a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/GeometryUDT.scala +++ b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/GeometryUDT.scala @@ -28,7 +28,7 @@ import org.locationtech.jts.geom.Geometry class GeometryUDT extends UserDefinedType[Geometry] { override def sqlType: DataType = BinaryType - override def pyUDT: String = "sedona.sql.types.GeometryType" + override def pyUDT: String = "sedona.spark.sql.types.GeometryType" override def userClass: Class[Geometry] = classOf[Geometry] diff --git a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/RasterUDT.scala b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/RasterUDT.scala index 8ebc2762a65..ee05cd15216 100644 --- a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/RasterUDT.scala +++ b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/RasterUDT.scala @@ -25,7 +25,7 @@ import org.geotools.coverage.grid.GridCoverage2D class RasterUDT extends UserDefinedType[GridCoverage2D] { override def sqlType: DataType = BinaryType - override def pyUDT: String = "sedona.sql.types.RasterType" + override def pyUDT: String = "sedona.spark.sql.types.RasterType" // A reasonable size for small in-db rasters. This is used by the optimizer to decide whether to // broadcast the dataframe or not.