Skip to content
This repository was archived by the owner on Oct 8, 2020. It is now read-only.

Commit 2ee242c

Browse files
GezimSejdiuGezimSejdiu
authored
Merge pull request #28 from SANSA-Stack/feature/bump-versions
Bump versions
2 parents 48a3a1e + 4a98b1c commit 2ee242c

6 files changed

Lines changed: 69 additions & 76 deletions

File tree

‎pom.xml‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
<maven.compiler.target>1.8</maven.compiler.target>
2424
<encoding>UTF-8</encoding>
2525

26-
<sansa.version>0.4.0</sansa.version>
26+
<sansa.rdf.version>0.5.0</sansa.rdf.version>
2727
<sansa.datalake.version>0.0.1-SNAPSHOT</sansa.datalake.version>
2828

2929
<scala.version>2.11.11</scala.version>
@@ -32,11 +32,11 @@
3232

3333
<scala.version.suffix>_${scala.binary.version}</scala.version.suffix>
3434

35-
<spark.version>2.3.1</spark.version>
36-
<flink.version>1.5.0</flink.version>
35+
<spark.version>2.4.0</spark.version>
36+
<flink.version>1.7.0</flink.version>
3737

38-
<jena.version>3.7.0</jena.version>
39-
<jsa.subversion>3</jsa.subversion>
38+
<jena.version>3.9.0</jena.version>
39+
<jsa.subversion>1</jsa.subversion>
4040

4141
<jsa.version>${jena.version}-${jsa.subversion}</jsa.version>
4242
<scalastyle.config.path>${project.basedir}/scalastyle-config.xml</scalastyle.config.path>
@@ -212,19 +212,19 @@
212212
<dependency>
213213
<groupId>${project.groupId}</groupId>
214214
<artifactId>sansa-rdf-common${scala.version.suffix}</artifactId>
215-
<version>${sansa.version}</version>
215+
<version>${sansa.rdf.version}</version>
216216
</dependency>
217217

218218
<dependency>
219219
<groupId>${project.groupId}</groupId>
220220
<artifactId>sansa-rdf-spark${scala.version.suffix}</artifactId>
221-
<version>${sansa.version}</version>
221+
<version>${sansa.rdf.version}</version>
222222
</dependency>
223223

224224
<dependency>
225225
<groupId>${project.groupId}</groupId>
226226
<artifactId>sansa-rdf-flink${scala.version.suffix}</artifactId>
227-
<version>${sansa.version}</version>
227+
<version>${sansa.rdf.version}</version>
228228
</dependency>
229229

230230

@@ -307,6 +307,12 @@
307307
<version>19.0</version>
308308
</dependency>
309309

310+
<dependency>
311+
<groupId>org.apache.commons</groupId>
312+
<artifactId>commons-compress</artifactId>
313+
<version>1.18</version>
314+
</dependency>
315+
310316
<dependency>
311317
<groupId>org.apache.jena</groupId>
312318
<artifactId>apache-jena-libs</artifactId>

‎sansa-query-flink/src/main/scala/net/sansa_stack/query/flink/sparqlify/BasicTableInfoProviderFlink.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import org.apache.flink.table.api.scala.BatchTableEnvironment
1212
class BasicTableInfoProviderFlink(flinkTable: BatchTableEnvironment)
1313
extends BasicTableInfoProvider {
1414
override def getBasicTableInfo(queryString: String): BasicTableInfo = {
15-
val table = flinkTable.sql(queryString)
15+
val table = flinkTable.sqlQuery(queryString)
1616
val schema = table.getSchema
1717
val types = schema.getTypes
1818
val names = schema.getColumnNames

‎sansa-query-flink/src/main/scala/net/sansa_stack/query/flink/sparqlify/QueryExecutionSparqlifyFlink.scala‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import org.aksw.jena_sparql_api.core.{ QueryExecutionBaseSelect, QueryExecutionF
1010
import org.aksw.jena_sparql_api.utils.ResultSetUtils
1111
import org.aksw.sparqlify.core.domain.input.SparqlSqlStringRewrite
1212
import org.aksw.sparqlify.core.interfaces.SparqlSqlStringRewriter
13-
import org.apache.flink .api.java.typeutils.runtime.kryo.KryoSerializer
13+
import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer
1414
import org.apache.flink.api.scala.{ ExecutionEnvironment, _ }
1515
import org.apache.flink.table.api.scala.{ BatchTableEnvironment, _ }
1616
import org.apache.flink.types.Row
@@ -45,6 +45,9 @@ class QueryExecutionSparqlifyFlink(
4545

4646
override def getTimeout1(): Long = -1
4747

48+
override def execJson(): org.apache.jena.atlas.json.JsonArray = throw new UnsupportedOperationException
49+
override def execJsonItems(): java.util.Iterator[org.apache.jena.atlas.json.JsonObject] = throw new UnsupportedOperationException
50+
4851
}
4952

5053
object QueryExecutionSparqlifyFlink {
@@ -55,7 +58,7 @@ object QueryExecutionSparqlifyFlink {
5558

5659
println("SQL Query: " + sqlQueryStr)
5760

58-
val dataset = flinkTable.sql(sqlQueryStr)
61+
val dataset = flinkTable.sqlQuery(sqlQueryStr)
5962
// System.out.println("SqlQueryStr: " + sqlQueryStr);
6063
// System.out.println("VarDef: " + rewrite.getVarDefinition());
6164
val rowMapper = new FlinkRowMapperSparqlify(varDef, dataset.getSchema.getColumnNames)

‎sansa-query-spark/src/main/java/net/sansa_stack/query/spark/sparqlify/QueryExecutionSparqlifySpark.java‎

Lines changed: 45 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
import org.aksw.jena_sparql_api.utils.ResultSetUtils;
1010
import org.aksw.sparqlify.core.domain.input.SparqlSqlStringRewrite;
1111
import org.aksw.sparqlify.core.interfaces.SparqlSqlStringRewriter;
12+
import org.apache.jena.atlas.json.JsonArray;
13+
import org.apache.jena.atlas.json.JsonObject;
1214
import org.apache.jena.query.Query;
1315
import org.apache.jena.query.QueryExecution;
1416
import org.apache.jena.query.ResultSet;
@@ -17,43 +19,47 @@
1719
import org.apache.spark.api.java.JavaRDD;
1820
import org.apache.spark.sql.SparkSession;
1921

20-
public class QueryExecutionSparqlifySpark
21-
extends QueryExecutionBaseSelect
22-
{
23-
public QueryExecutionSparqlifySpark(
24-
Query query,
25-
QueryExecutionFactory subFactory,
26-
SparkSession sparkSession,
27-
SparqlSqlStringRewriter sparqlSqlRewriter
28-
) {
29-
super(query, subFactory);
30-
this.sparkSession = sparkSession;
31-
this.sparqlSqlRewriter = sparqlSqlRewriter;
32-
}
33-
34-
protected SparkSession sparkSession;
35-
protected SparqlSqlStringRewriter sparqlSqlRewriter;
36-
37-
@Override
38-
protected ResultSetCloseable executeCoreSelect(Query query) {
39-
SparqlSqlStringRewrite rewrite = sparqlSqlRewriter.rewrite(query);
40-
List<Var> resultVars = rewrite.getProjectionOrder();
41-
42-
JavaRDD<Binding> rdd = QueryExecutionUtilsSpark.createQueryExecution(sparkSession, rewrite, query);
43-
Iterator<Binding> it = rdd.toLocalIterator();
44-
45-
ResultSet tmp = ResultSetUtils.create2(resultVars, it);
46-
ResultSetCloseable result = new ResultSetCloseable(tmp);
47-
return result;
48-
}
49-
50-
@Override
51-
protected QueryExecution executeCoreSelectX(Query query) {
52-
throw new UnsupportedOperationException();
53-
}
54-
55-
@Override
56-
public long getTimeout1() {
57-
return -1;
58-
}
22+
public class QueryExecutionSparqlifySpark extends QueryExecutionBaseSelect {
23+
public QueryExecutionSparqlifySpark(Query query, QueryExecutionFactory subFactory, SparkSession sparkSession,
24+
SparqlSqlStringRewriter sparqlSqlRewriter) {
25+
super(query, subFactory);
26+
this.sparkSession = sparkSession;
27+
this.sparqlSqlRewriter = sparqlSqlRewriter;
28+
}
29+
30+
protected SparkSession sparkSession;
31+
protected SparqlSqlStringRewriter sparqlSqlRewriter;
32+
33+
@Override
34+
protected ResultSetCloseable executeCoreSelect(Query query) {
35+
SparqlSqlStringRewrite rewrite = sparqlSqlRewriter.rewrite(query);
36+
List<Var> resultVars = rewrite.getProjectionOrder();
37+
38+
JavaRDD<Binding> rdd = QueryExecutionUtilsSpark.createQueryExecution(sparkSession, rewrite, query);
39+
Iterator<Binding> it = rdd.toLocalIterator();
40+
41+
ResultSet tmp = ResultSetUtils.create2(resultVars, it);
42+
ResultSetCloseable result = new ResultSetCloseable(tmp);
43+
return result;
44+
}
45+
46+
@Override
47+
protected QueryExecution executeCoreSelectX(Query query) {
48+
throw new UnsupportedOperationException();
49+
}
50+
51+
@Override
52+
public long getTimeout1() {
53+
return -1;
54+
}
55+
56+
@Override
57+
public JsonArray execJson() {
58+
throw new UnsupportedOperationException();
59+
}
60+
61+
@Override
62+
public Iterator<JsonObject> execJsonItems() {
63+
throw new UnsupportedOperationException();
64+
}
5965
}

‎sansa-query-spark/src/main/scala/net/sansa_stack/query/spark/package.scala‎

Lines changed: 2 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -68,34 +68,12 @@ package object query {
6868

6969
implicit class Semantic(partitions: RDD[String]) extends Serializable {
7070

71-
val symbol = Map(
72-
"space" -> " " * 5,
73-
"blank" -> " ",
74-
"tabs" -> "\t",
75-
"newline" -> "\n",
76-
"colon" -> ":",
77-
"comma" -> ",",
78-
"hash" -> "#",
79-
"slash" -> "/",
80-
"question-mark" -> "?",
81-
"exclamation-mark" -> "!",
82-
"curly-bracket-left" -> "{",
83-
"curly-bracket-right" -> "}",
84-
"round-bracket-left" -> "(",
85-
"round-bracket-right" -> ")",
86-
"less-than" -> "<",
87-
"greater-than" -> ">",
88-
"at" -> "@",
89-
"dot" -> ".",
90-
"dots" -> "...",
91-
"asterisk" -> "*",
92-
"up-arrows" -> "^^")
93-
9471
/**
9572
* semantic partition of and RDF graph
9673
*/
9774
def sparql(sparqlQuery: String)(input: String, output: String, numOfFilesPartition: Int): Unit = {
98-
new QuerySystem(symbol, partitions,
75+
new QuerySystem(
76+
partitions,
9977
input,
10078
output,
10179
numOfFilesPartition).run()

‎sansa-query-spark/src/main/scala/net/sansa_stack/query/spark/semantic/SparqlQuerySystem.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,19 +11,18 @@ import scala.util.Try
1111

1212
import com.google.common.collect.ArrayListMultimap
1313
import net.sansa_stack.query.spark.semantic.utils.Helpers._
14+
import net.sansa_stack.rdf.common.partition.utils.Symbols
1415
import org.apache.spark.rdd._
1516

1617
/*
1718
* QuerySystem - query on semantic partition data
1819
*
19-
* @symbol - list of symbols.
2020
* @partitionData - a RDD of n-triples (formatted).
2121
* @queryInputPath - query file path.
2222
* @queryResultPath - path for output result.
2323
* @numOfFilesPartition - total number of files to save the partition data.
2424
*/
2525
class QuerySystem(
26-
symbol: Map[String, String],
2726
partitionData: RDD[String],
2827
queryInputPath: String,
2928
queryResultPath: String,
@@ -40,6 +39,7 @@ class QuerySystem(
4039
var workingTripleRDD: RDD[(String, List[String])] = _
4140
var workingPartialRDD: RDD[(String, List[String])] = _
4241
var unionOutputRDD: RDD[String] = _
42+
val symbol = Symbols.symbol
4343

4444
def run(): Unit = {
4545
// parse queries

0 commit comments

Comments
 (0)