Skip to content

Commit ba9f61a

Browse files
JohannesJanderJohannesJander
committed
Work on foreign keys, API changes in regards to iterators (necessary b/c return types were partly wrong), more robust BeanSchema/BeanIterator
1 parent 8bfbf39 commit ba9f61a

10 files changed

Lines changed: 258 additions & 51 deletions

File tree

‎src/main/java/io/frictionlessdata/datapackage/resource/AbstractReferencebasedResource.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
import com.fasterxml.jackson.annotation.JsonIgnore;
44
import com.fasterxml.jackson.databind.JsonNode;
55
import io.frictionlessdata.tableschema.Table;
6-
import io.frictionlessdata.tableschema.datasourceformat.DataSourceFormat;
6+
import io.frictionlessdata.tableschema.tabledatasource.TableDataSource;
77
import io.frictionlessdata.tableschema.util.JsonUtil;
88

99
import java.util.ArrayList;
@@ -53,11 +53,11 @@ public Collection<T> getPaths() {
5353
public Set<String> getDatafileNamesForWriting() {
5454
List<String> paths = new ArrayList<>(((FilebasedResource)this).getReferencesAsStrings());
5555
return paths.stream().map((p) -> {
56-
if (p.toLowerCase().endsWith("."+ DataSourceFormat.Format.FORMAT_CSV.getLabel())){
57-
int i = p.toLowerCase().indexOf("."+DataSourceFormat.Format.FORMAT_CSV.getLabel());
56+
if (p.toLowerCase().endsWith("."+ TableDataSource.Format.FORMAT_CSV.getLabel())){
57+
int i = p.toLowerCase().indexOf("."+TableDataSource.Format.FORMAT_CSV.getLabel());
5858
return p.substring(0, i);
59-
} else if (p.toLowerCase().endsWith("."+DataSourceFormat.Format.FORMAT_JSON.getLabel())){
60-
int i = p.toLowerCase().indexOf("."+DataSourceFormat.Format.FORMAT_JSON.getLabel());
59+
} else if (p.toLowerCase().endsWith("."+TableDataSource.Format.FORMAT_JSON.getLabel())){
60+
int i = p.toLowerCase().indexOf("."+TableDataSource.Format.FORMAT_JSON.getLabel());
6161
return p.substring(0, i);
6262
}
6363
return p;

‎src/main/java/io/frictionlessdata/datapackage/resource/AbstractResource.java‎

Lines changed: 47 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,12 @@
77
import com.fasterxml.jackson.databind.node.ObjectNode;
88
import io.frictionlessdata.datapackage.Dialect;
99
import io.frictionlessdata.datapackage.JSONBase;
10+
import io.frictionlessdata.datapackage.Package;
1011
import io.frictionlessdata.datapackage.Profile;
1112
import io.frictionlessdata.datapackage.exceptions.DataPackageException;
1213
import io.frictionlessdata.tableschema.Table;
13-
import io.frictionlessdata.tableschema.datasourceformat.DataSourceFormat;
14+
import io.frictionlessdata.tableschema.tabledatasource.TableDataSource;
15+
import io.frictionlessdata.tableschema.fk.ForeignKey;
1416
import io.frictionlessdata.tableschema.io.FileReference;
1517
import io.frictionlessdata.tableschema.io.URLFileReference;
1618
import io.frictionlessdata.tableschema.iterator.BeanIterator;
@@ -59,21 +61,21 @@ public abstract class AbstractResource<T,C> extends JSONBase implements Resource
5961

6062
@Override
6163
public Iterator<Object[]> objectArrayIterator() throws Exception{
62-
return this.objectArrayIterator(false, false, false);
64+
return this.objectArrayIterator(false, false);
6365
}
6466

6567
@Override
66-
public Iterator<Object[]> objectArrayIterator(boolean keyed, boolean extended, boolean relations) throws Exception{
68+
public Iterator<Object[]> objectArrayIterator(boolean extended, boolean relations) throws Exception{
6769
ensureDataLoaded();
6870
Iterator<Object[]>[] tableIteratorArray = new TableIterator[tables.size()];
6971
int cnt = 0;
7072
for (Table table : tables) {
71-
tableIteratorArray[cnt++] = table.iterator(keyed, extended, true, relations);
73+
tableIteratorArray[cnt++] = (Iterator)table.iterator(false, extended, true, relations);
7274
}
7375
return new IteratorChain(tableIteratorArray);
7476
}
7577

76-
private Iterator stringArrayIterator(boolean relations) throws Exception{
78+
private Iterator<String[]> stringArrayIterator(boolean relations) throws Exception{
7779
ensureDataLoaded();
7880
Iterator[] tableIteratorArray = new TableIterator[tables.size()];
7981
int cnt = 0;
@@ -95,7 +97,7 @@ public Iterator<String[]> stringArrayIterator() throws Exception{
9597
}
9698

9799
@Override
98-
public Iterator<Map<String, Object>> mappedIterator(boolean relations) throws Exception{
100+
public Iterator<Map<String, Object>> mappingIterator(boolean relations) throws Exception{
99101
ensureDataLoaded();
100102
Iterator<Map<String, Object>>[] tableIteratorArray = new TableIterator[tables.size()];
101103
int cnt = 0;
@@ -126,6 +128,22 @@ public List<String[]> getData() throws Exception{
126128
return retVal;
127129
}
128130

131+
@Override
132+
public List<Map<String, Object>> getMappedData(boolean relations) throws Exception {
133+
List<Map<String, Object>> retVal = new ArrayList<>();
134+
ensureDataLoaded();
135+
Iterator[] tableIteratorArray = new TableIterator[tables.size()];
136+
int cnt = 0;
137+
for (Table table : tables) {
138+
tableIteratorArray[cnt++] = table.iterator(true, false, true, relations);
139+
}
140+
Iterator iter = new IteratorChain<>(tableIteratorArray);
141+
while (iter.hasNext()) {
142+
retVal.add((Map)iter.next());
143+
}
144+
return retVal;
145+
}
146+
129147
/**
130148
* Most customizable method to retrieve all data in a Resource. Parameters match those in
131149
* {@link io.frictionlessdata.tableschema.Table#iterator(boolean, boolean, boolean, boolean)}. Data can be
@@ -144,12 +162,12 @@ public List<String[]> getData() throws Exception{
144162
* @return List of data objects
145163
* @throws Exception if reading data fails
146164
*/
147-
public List<Object[]> getData(boolean keyed, boolean extended, boolean cast, boolean relations) throws Exception{
148-
List<Object[]> retVal = new ArrayList<>();
165+
public List<Object> getData(boolean keyed, boolean extended, boolean cast, boolean relations) throws Exception{
166+
List<Object> retVal = new ArrayList<>();
149167
ensureDataLoaded();
150168
Iterator iter;
151169
if (cast) {
152-
iter = objectArrayIterator(keyed, extended, relations);
170+
iter = objectArrayIterator(extended, relations);
153171
} else {
154172
iter = stringArrayIterator(relations);
155173
}
@@ -186,6 +204,21 @@ public List<Table> getTables() throws Exception {
186204
return tables;
187205
}
188206

207+
public void checkRelations() throws Exception {
208+
if (null != schema) {
209+
for (ForeignKey fk : schema.getForeignKeys()) {
210+
fk.validate();
211+
fk.getReference().validate();
212+
}
213+
for (ForeignKey fk : schema.getForeignKeys()) {
214+
if (null != fk.getReference().getResource()) {
215+
//Package pkg = new Package(fk.getReference().getDatapackage(), true);
216+
// TODO fix this
217+
}
218+
}
219+
}
220+
}
221+
189222
/**
190223
* Get JSON representation of the object.
191224
* @return a JSONObject representing the properties of this object
@@ -570,16 +603,16 @@ public void setShouldSerializeToFile(boolean serializeToFile) {
570603

571604
@Override
572605
public void setSerializationFormat(String format) {
573-
if ((format.equals(DataSourceFormat.Format.FORMAT_JSON.getLabel()))
574-
|| format.equals(DataSourceFormat.Format.FORMAT_CSV.getLabel())) {
606+
if ((format.equals(TableDataSource.Format.FORMAT_JSON.getLabel()))
607+
|| format.equals(TableDataSource.Format.FORMAT_CSV.getLabel())) {
575608
this.serializationFormat = format;
576609
} else
577610
throw new DataPackageException("Serialization format "+format+" is unknown");
578611
}
579612

580613
/**
581614
* if an explicit serialisation format was set, return this. Alternatively return the default
582-
* {@link io.frictionlessdata.tableschema.datasourceformat.DataSourceFormat.Format} as a String
615+
* {@link io.frictionlessdata.tableschema.tabledatasource.TableDataSource.Format} as a String
583616
* @return Serialisation format, either "csv" or "json"
584617
*/
585618
@JsonIgnore
@@ -622,9 +655,9 @@ public void writeData(Path outputDir) throws Exception {
622655
}
623656
Files.deleteIfExists(p);
624657
try (Writer wr = Files.newBufferedWriter(p, StandardCharsets.UTF_8)) {
625-
if (serializationFormat.equals(DataSourceFormat.Format.FORMAT_CSV.getLabel())) {
658+
if (serializationFormat.equals(TableDataSource.Format.FORMAT_CSV.getLabel())) {
626659
t.writeCsv(wr, lDialect.toCsvFormat());
627-
} else if (serializationFormat.equals(DataSourceFormat.Format.FORMAT_JSON.getLabel())) {
660+
} else if (serializationFormat.equals(TableDataSource.Format.FORMAT_JSON.getLabel())) {
628661
wr.write(t.asJson());
629662
}
630663
}

‎src/main/java/io/frictionlessdata/datapackage/resource/FilebasedResource.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
import com.fasterxml.jackson.annotation.JsonInclude.Include;
66
import io.frictionlessdata.datapackage.exceptions.DataPackageException;
77
import io.frictionlessdata.tableschema.Table;
8-
import io.frictionlessdata.tableschema.datasourceformat.DataSourceFormat;
8+
import io.frictionlessdata.tableschema.tabledatasource.TableDataSource;
99

1010
import java.io.File;
1111
import java.nio.file.Path;
@@ -58,17 +58,17 @@ public FilebasedResource(Resource fromResource, Collection<File> paths) throws E
5858
private static String sniffFormat(Collection<File> paths) {
5959
Set<String> foundFormats = new HashSet<>();
6060
paths.forEach((p) -> {
61-
if (p.getName().toLowerCase().endsWith(DataSourceFormat.Format.FORMAT_CSV.getLabel())) {
62-
foundFormats.add(DataSourceFormat.Format.FORMAT_CSV.getLabel());
63-
} else if (p.getName().toLowerCase().endsWith(DataSourceFormat.Format.FORMAT_JSON.getLabel())) {
64-
foundFormats.add(DataSourceFormat.Format.FORMAT_JSON.getLabel());
61+
if (p.getName().toLowerCase().endsWith(TableDataSource.Format.FORMAT_CSV.getLabel())) {
62+
foundFormats.add(TableDataSource.Format.FORMAT_CSV.getLabel());
63+
} else if (p.getName().toLowerCase().endsWith(TableDataSource.Format.FORMAT_JSON.getLabel())) {
64+
foundFormats.add(TableDataSource.Format.FORMAT_JSON.getLabel());
6565
}
6666
});
6767
if (foundFormats.size() > 1) {
6868
throw new DataPackageException("Resources cannot be mixed JSON/CSV");
6969
}
7070
if (foundFormats.isEmpty())
71-
return DataSourceFormat.Format.FORMAT_CSV.getLabel();
71+
return TableDataSource.Format.FORMAT_CSV.getLabel();
7272
return foundFormats.iterator().next();
7373
}
7474

‎src/main/java/io/frictionlessdata/datapackage/resource/JSONDataResource.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
package io.frictionlessdata.datapackage.resource;
22

33
import com.fasterxml.jackson.databind.node.ArrayNode;
4-
import io.frictionlessdata.tableschema.datasourceformat.DataSourceFormat;
4+
import io.frictionlessdata.tableschema.tabledatasource.TableDataSource;
55
import io.frictionlessdata.tableschema.util.JsonUtil;
66

77
public class JSONDataResource<C> extends AbstractDataResource<ArrayNode,C> {
@@ -13,6 +13,6 @@ public JSONDataResource(String name, String json) {
1313

1414
@Override
1515
String getResourceFormat() {
16-
return DataSourceFormat.Format.FORMAT_JSON.getLabel();
16+
return TableDataSource.Format.FORMAT_JSON.getLabel();
1717
}
1818
}

‎src/main/java/io/frictionlessdata/datapackage/resource/Resource.java‎

Lines changed: 37 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,23 @@ public interface Resource<T,C> {
4444

4545
String getJson();
4646

47+
/**
48+
* Read all data from a Resource, each row as Map objects. This can be used for smaller datapackages,
49+
* but for huge or unknown sizes, reading via iterator is preferred.
50+
*
51+
* The method returns Map&lt;String,Object&gt; where key is the header name, and val is the data.
52+
* It can be configured to return table rows with relations to other data sources resolved
53+
*
54+
* The method uses Iterators provided by {@link Table} class, and is roughly implemented after
55+
* https://github.com/frictionlessdata/tableschema-py/blob/master/tableschema/table.py
56+
*
57+
* @param relations true: follow relations
58+
* @return A list of table rows.
59+
* @throws Exception if parsing the data fails
60+
*
61+
*/
62+
List<Map<String, Object>> getMappedData(boolean relations) throws Exception;
63+
4764
/**
4865
* Read all data from a Resource. This can be used for smaller datapackages, but for huge or unknown
4966
* sizes, reading via iterator is preferred.
@@ -70,7 +87,7 @@ public interface Resource<T,C> {
7087
* @throws Exception if parsing the data fails
7188
*
7289
*/
73-
List<Object[]> getData(boolean cast, boolean keyed, boolean extended, boolean relations) throws Exception;
90+
List<Object> getData(boolean cast, boolean keyed, boolean extended, boolean relations) throws Exception;
7491

7592
/**
7693
* Read all data from a Resource. This can be used for smaller datapackages, but for huge or unknown
@@ -118,15 +135,15 @@ public interface Resource<T,C> {
118135
* @return Row Iterator
119136
* @throws Exception if parsing the data fails
120137
*/
121-
Iterator<Object[]> objectArrayIterator(boolean keyed, boolean extended, boolean relations) throws Exception;
138+
Iterator<Object[]> objectArrayIterator(boolean extended, boolean relations) throws Exception;
122139

123140

124141
/**
125142
* Returns an Iterator that returns rows as a Map&lt;key,val&gt; where key is the header name, and val is the data
126143
* @return Row Iterator
127144
* @throws Exception if parsing the data fails
128145
*/
129-
Iterator<Map<String, Object>> mappedIterator(boolean relations) throws Exception;
146+
Iterator<Map<String, Object>> mappingIterator(boolean relations) throws Exception;
130147

131148
/**
132149
* Returns an Iterator that returns rows as bean-arrays.
@@ -310,6 +327,8 @@ public interface Resource<T,C> {
310327

311328
String getSerializationFormat();
312329

330+
void checkRelations() throws Exception;
331+
313332
/**
314333
* Recreate a Resource object from a JSON descriptor, a base path to resolve relative file paths against
315334
* and a flag that tells us whether we are reading from inside a ZIP archive.
@@ -339,22 +358,30 @@ static AbstractResource build(ObjectNode resourceJson, Object basePath, boolean
339358
if (resource instanceof FilebasedResource) {
340359
((FilebasedResource)resource).setIsInArchive(isArchivePackage);
341360
}
342-
} else if (data != null && format != null){
343-
if (format.equals(Resource.FORMAT_JSON))
344-
resource = new JSONDataResource(name, ((ArrayNode) data).toString());
361+
// inlined data
362+
} else if (data != null){
363+
if (null == format) {
364+
if (!(data instanceof ArrayNode)) {
365+
// from the spec: " a JSON string - in this case the format or
366+
// mediatype properties MUST be provided
367+
// https://specs.frictionlessdata.io/data-resource/#data-inline-data
368+
throw new DataPackageException(
369+
"Invalid Resource. The format property cannot be null for inlined CSV data.");
370+
}
371+
resource = new JSONDataResource(name, data.toString());
372+
} else if (format.equals(Resource.FORMAT_JSON))
373+
resource = new JSONDataResource(name, data.toString());
345374
else if (format.equals(Resource.FORMAT_CSV))
346375
resource = new CSVDataResource(name, data.toString());
347376
} else {
348-
DataPackageException dpe = new DataPackageException(
377+
throw new DataPackageException(
349378
"Invalid Resource. The path property or the data and format properties cannot be null.");
350-
throw dpe;
351379
}
352380
resource.setDialect(dialect);
353381
JSONBase.setFromJson(resourceJson, resource, schema);
354382
return resource;
355383
}
356384

357-
358385
static AbstractResource build(String name, Collection pathOrUrl, Object basePath) throws MalformedURLException {
359386
if (pathOrUrl != null) {
360387
List<File> files = new ArrayList<>();
@@ -379,7 +406,7 @@ static AbstractResource build(String name, Collection pathOrUrl, Object basePath
379406
for (String s : strings) {
380407
if (basePath instanceof URL) {
381408
/*
382-
* We have a URL fragment, that is not valid on its own.
409+
* We have a URL fragment that is not valid on its own.
383410
* According to https://github.com/frictionlessdata/specs/issues/652 ,
384411
* URL fragments should be resolved relative to the base URL
385412
*/

‎src/test/java/io/frictionlessdata/datapackage/ForeignKeysTest.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
package io.frictionlessdata.datapackage;
2+
3+
import io.frictionlessdata.datapackage.resource.Resource;
4+
import org.junit.jupiter.api.DisplayName;
5+
import org.junit.jupiter.api.Test;
6+
7+
import java.nio.file.Path;
8+
9+
public class ForeignKeysTest {
10+
11+
@Test
12+
@DisplayName("Test that a schema can be defined via a URL")
13+
// Test for https://github.com/frictionlessdata/specs/issues/645
14+
void testValidationURLAsSchemaReference() throws Exception{
15+
Path resourcePath = TestUtil.getResourcePath("/fixtures/datapackages/foreign-keys.json");
16+
Package pkg = new Package(resourcePath, true);
17+
System.out.println(pkg);
18+
Resource teams = pkg.getResource("teams");
19+
teams.checkRelations();
20+
}
21+
}

‎src/test/java/io/frictionlessdata/datapackage/TestUtil.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,18 @@
11
package io.frictionlessdata.datapackage;
22

3+
import java.io.File;
34
import java.net.URL;
45
import java.nio.file.Path;
56
import java.nio.file.Paths;
67

78
public class TestUtil {
89

10+
public static File getTestDataDirectory()throws Exception {
11+
URL u = TestUtil.class.getResource("/fixtures/multi_data_datapackage.json");
12+
Path path = Paths.get(u.toURI());
13+
return path.getParent().getParent().toFile();
14+
}
15+
916
public static Path getBasePath() {
1017
try {
1118
String pathName = "/fixtures/multi_data_datapackage.json";

0 commit comments

Comments
 (0)