Skip to content

Commit 9e2769a

Browse files
jiayuasujiayuasu
committed
Refactor
1 parent b59f17e commit 9e2769a

5 files changed

Lines changed: 48 additions & 55 deletions

File tree

‎examples/java-spark-sql/pom.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@
8585
--add-opens=java.base/sun.nio.ch=ALL-UNNAMED
8686
--add-opens=java.base/java.nio=ALL-UNNAMED
8787
--add-opens=java.base/java.lang=ALL-UNNAMED
88+
--add-opens=java.base/java.lang.invoke=ALL-UNNAMED
8889
--add-opens=java.base/java.util=ALL-UNNAMED
8990
</argLine>
9091
</configuration>

‎examples/java-spark-sql/src/main/java/spark/GeoParquetAccessor.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919

2020
package spark;
2121

22+
import org.apache.sedona.core.spatialOperator.RangeQuery;
23+
import org.apache.sedona.core.spatialOperator.SpatialPredicate;
2224
import org.apache.sedona.core.spatialRDD.SpatialRDD;
2325
import org.apache.sedona.sql.utils.Adapter;
2426
import org.apache.spark.sql.Dataset;
@@ -28,22 +30,20 @@
2830
import org.locationtech.jts.geom.Geometry;
2931
import org.locationtech.jts.geom.GeometryFactory;
3032
import org.locationtech.jts.geom.Polygon;
31-
import org.apache.sedona.core.spatialOperator.SpatialPredicate;
32-
import org.apache.sedona.core.spatialOperator.RangeQuery;
3333

3434
import java.util.List;
3535

3636

3737
public class GeoParquetAccessor {
38-
38+
3939
private final SparkSession session;
4040
private String parquetPath;
4141

4242
public GeoParquetAccessor() {
4343
this.session = new SedonaSparkSession().getSession();
4444
this.parquetPath = "";
4545
}
46-
46+
4747
//Overload with constructor that has Spark session provided
4848
//Use to avoid error - can't have two SparkContext objects on one JVM
4949
public GeoParquetAccessor(SparkSession session, String parquetPath) {
@@ -53,7 +53,7 @@ public GeoParquetAccessor(SparkSession session, String parquetPath) {
5353

5454
public List<Geometry> selectFeaturesByPolygon(double xmin, double ymax,
5555
double xmax, double ymin,
56-
String geometryColumn) {
56+
String geometryColumn) {
5757

5858
//Read the GeoParquet file into a DataFrame
5959
Dataset<Row> insarDF = session.read().format("geoparquet").load(parquetPath);
@@ -86,5 +86,5 @@ public List<Geometry> selectFeaturesByPolygon(double xmin, double ymax,
8686
// Collect the results back to the driver
8787
return resultRDD.getRawSpatialRDD().collect();
8888
}
89-
89+
9090
}

‎examples/java-spark-sql/src/main/java/spark/SedonaGeoParquetMain.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121

2222
import org.locationtech.jts.geom.Coordinate;
2323
import org.locationtech.jts.geom.Geometry;
24-
import java.io.File;
24+
2525
import java.io.IOException;
2626
import java.io.InputStream;
2727
import java.util.List;
@@ -30,18 +30,18 @@
3030
public class SedonaGeoParquetMain {
3131

3232
protected static Properties properties;
33-
protected static String parquetPath;
33+
protected static String parquetPath;
3434
protected static SedonaSparkSession session;
3535

3636
public static void main(String args[]) {
3737

3838
session = new SedonaSparkSession();
3939
//Get parquetPath and any other application.properties
40-
try {
40+
try {
4141
ClassLoader loader = Thread.currentThread().getContextClassLoader();
4242
Properties properties = new Properties();
4343
InputStream is = loader.getResourceAsStream("application.properties");
44-
properties.load(is);
44+
properties.load(is);
4545
parquetPath = properties.getProperty("parquet.path");
4646
} catch (IOException e) {
4747
e.printStackTrace();

‎examples/java-spark-sql/src/main/java/spark/SedonaSparkSession.java‎

Lines changed: 16 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -19,38 +19,33 @@
1919

2020
package spark;
2121

22+
import org.apache.sedona.spark.SedonaContext;
2223
import org.apache.spark.sql.SparkSession;
23-
import org.apache.sedona.sql.utils.SedonaSQLRegistrator;
24-
import org.apache.spark.SparkConf;
25-
import org.apache.spark.sql.RuntimeConfig;
2624

2725

2826
public class SedonaSparkSession {
29-
30-
private SparkConf conf;
31-
private RuntimeConfig config;
27+
3228
public SparkSession session;
3329

3430
public SedonaSparkSession() {
3531

36-
//Set configuration for localhost spark cluster. Intended ot be run from IDE or similar.
37-
this.conf = new SparkConf().setAppName(this.getClass().getSimpleName())
38-
.setMaster("local[*]")
39-
.set("spark.ui.enabled", "false")
40-
.set("spark.driver.extraJavaOptions",
41-
"--add-opens=java.base/sun.nio.ch=ALL-UNNAMED");
42-
this.session = SparkSession.builder()
43-
.config(this.conf)
44-
.getOrCreate();
45-
//Configure Saprk session to execute with Sedona
46-
SedonaSQLRegistrator.registerAll(this.session);
47-
this.config = this.session.conf();
32+
//Set configuration for localhost spark cluster. Intended to be run from IDE or similar.
33+
//Use SedonaContext builder to create SparkSession with Sedona extensions
34+
SparkSession config = SedonaContext.builder()
35+
.appName(this.getClass().getSimpleName())
36+
.master("local[*]")
37+
.config("spark.ui.enabled", "false")
38+
.config("spark.driver.extraJavaOptions",
39+
"--add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED")
40+
.getOrCreate();
41+
42+
//Create Sedona-enabled SparkSession
43+
this.session = SedonaContext.create(config);
4844
}
4945

5046
public SparkSession getSession() {
51-
// Access SparkSession object
47+
// Access SparkSession object
5248
return this.session;
5349
}
54-
55-
}
5650

51+
}

‎examples/java-spark-sql/src/test/java/spark/SedonaParquetTest.java‎

Lines changed: 21 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -19,64 +19,63 @@
1919

2020
package spark;
2121

22+
import org.apache.spark.sql.Dataset;
23+
import org.apache.spark.sql.Row;
24+
import org.junit.jupiter.api.AfterAll;
25+
import org.junit.jupiter.api.AfterEach;
26+
import org.junit.jupiter.api.BeforeAll;
27+
import org.junit.jupiter.api.BeforeEach;
28+
import org.junit.jupiter.api.Test;
29+
2230
import java.io.File;
2331
import java.io.IOException;
2432
import java.io.InputStream;
25-
import java.util.List;
2633
import java.util.Properties;
27-
import org.junit.jupiter.api.AfterEach;
28-
import org.junit.jupiter.api.AfterAll;
29-
import org.junit.jupiter.api.BeforeEach;
30-
import org.junit.jupiter.api.BeforeAll;
31-
import org.junit.jupiter.api.Test;
32-
import static org.junit.jupiter.api.Assertions.*;
3334

34-
import org.apache.spark.sql.Dataset;
35-
import org.apache.spark.sql.Row;
36-
import org.locationtech.jts.geom.Coordinate;
37-
import org.locationtech.jts.geom.Geometry;
35+
import static org.junit.jupiter.api.Assertions.assertNotNull;
36+
import static org.junit.jupiter.api.Assertions.assertTrue;
3837

3938

4039
public class SedonaParquetTest {
41-
42-
40+
41+
4342
protected static Properties properties;
44-
protected static String parquetPath;
43+
protected static String parquetPath;
4544
protected static SedonaSparkSession session;
46-
45+
4746
public SedonaParquetTest() {
4847
}
49-
48+
5049
@BeforeAll
5150
public static void setUpClass() throws IOException {
5251

5352
session = new SedonaSparkSession();
5453
//Get parquetPath and any other application.properties
55-
try {
54+
try {
5655
ClassLoader loader = Thread.currentThread().getContextClassLoader();
5756
Properties properties = new Properties();
5857
InputStream is = loader.getResourceAsStream("application.properties");
59-
properties.load(is);
58+
properties.load(is);
6059
parquetPath = properties.getProperty("parquet.path");
6160
} catch (IOException e) {
6261
e.printStackTrace();
6362
parquetPath = "";
6463
}
65-
64+
6665
}
67-
66+
6867
@AfterAll
6968
public static void tearDownClass() {
7069
}
71-
70+
7271
@BeforeEach
7372
public void setUp() {
7473
}
7574

7675
@AfterEach
7776
public void tearDown() {
7877
}
79-
78+
8079
@Test
8180
public void connects() {
8281
assertNotNull(session, "SparkSedonaSession not initialized correctly.");
@@ -101,5 +100,3 @@ public void canLoadRDD() {
101100
}
102101

103102
}
104-
105-

0 commit comments

Comments
 (0)