Skip to content

Commit 53f046e

Browse files
testowertestower
authored
Merge pull request #41 from entur/custom-validation-rules
Support for custom rules by schema patching
2 parents bdb3c2a + ed4813a commit 53f046e

25 files changed

Lines changed: 673 additions & 149 deletions

‎README.md‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,41 @@
33
# gbfs-validator-java
44

55
Validate GBFS feeds. Intended as Java native alternative to https://github.com/MobilityData/gbfs-validator.
6+
7+
Uses the official json schema to validate files.
8+
9+
## Additional validation rules
10+
11+
The interface `CustomRuleSchemaPatcher` enables adding additional rules dynamically by schema patching:
12+
13+
JSONObject addRule(JSONObject rawSchema, Map<String, JSONObject> feeds);
14+
15+
The raw schema along with a map of the data feeds is passed to this method. The patched schema should be returned.
16+
17+
List of additional rules:
18+
19+
* `VehicleTypeDefaultPricingPlanIdExistsInSystemPricingPlans`
20+
* `VehicleTypeIdsInVehicleTypesAvailableExistsInVehicleTypes`
21+
* `VehicleTypesAvailableRequiredWhenVehicleTypesExist`
22+
* `VehicleTypeIdRequiredInVehicleStatusWhenVehicleTypesExist`
23+
* `CurrentRangeMetersIsRequiredInVehicleStatusForMotorizedVehicles`
24+
25+
Planned rules:
26+
27+
* if free_bike_status / vehicle_status or station_information has rental uris then system_information must have store_uri in rental_apps (ios and / or android)
28+
29+
## Non-schema rules:
30+
31+
Some rules can't be validated with json schema:
32+
33+
Existing rules:
34+
35+
* All version of gbfs require the system_information endpoint.
36+
* In addition, gbfs endpoint is required as of v2.0.
37+
38+
Planned rules:
39+
40+
* Either station_information or station_status is required if the other is present
41+
* vehicle_types is required vehicle types are referenced in other files
42+
* system_pricing_plans is required if pricing plans are referenced in other files
43+

‎pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,11 @@
9595
<artifactId>slf4j-api</artifactId>
9696
<version>${slf4j.version}</version>
9797
</dependency>
98+
<dependency>
99+
<groupId>com.jayway.jsonpath</groupId>
100+
<artifactId>json-path</artifactId>
101+
<version>2.8.0</version>
102+
</dependency>
98103

99104
<!-- test dependencies -->
100105
<dependency>

‎src/main/java/org/entur/gbfs/validation/validator/FileValidator.java‎

Lines changed: 30 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -20,19 +20,14 @@
2020

2121
import org.entur.gbfs.validation.model.FileValidationError;
2222
import org.entur.gbfs.validation.model.FileValidationResult;
23-
import org.entur.gbfs.validation.versions.Version;
24-
import org.entur.gbfs.validation.versions.VersionFactory;
25-
import org.everit.json.schema.Schema;
23+
import org.entur.gbfs.validation.validator.versions.Version;
24+
import org.entur.gbfs.validation.validator.versions.VersionFactory;
2625
import org.everit.json.schema.ValidationException;
27-
import org.everit.json.schema.loader.SchemaLoader;
2826
import org.json.JSONObject;
29-
import org.json.JSONTokener;
3027
import org.slf4j.Logger;
3128
import org.slf4j.LoggerFactory;
3229

33-
import java.io.InputStream;
3430
import java.util.Collections;
35-
import java.util.HashMap;
3631
import java.util.List;
3732
import java.util.Map;
3833
import java.util.Optional;
@@ -42,58 +37,52 @@
4237
public class FileValidator {
4338
private static final Logger logger = LoggerFactory.getLogger(FileValidator.class);
4439
private final Version version;
45-
private final Map<String, Schema> schemas;
46-
4740
private static final Map<String, FileValidator> FILE_VALIDATORS = new ConcurrentHashMap<>();
4841

42+
4943
public static FileValidator getFileValidator(
5044
String detectedVersion
5145
) {
5246
if (FILE_VALIDATORS.containsKey(detectedVersion)) {
5347
return FILE_VALIDATORS.get(detectedVersion);
5448
} else {
5549
Version version = VersionFactory.createVersion(detectedVersion);
56-
Map<String, Schema> schemas = FileValidator.getSchemas(version);
57-
FileValidator fileValidator = new FileValidator(version, schemas);
50+
51+
FileValidator fileValidator = new FileValidator(version);
5852
FILE_VALIDATORS.put(detectedVersion, fileValidator);
5953
return fileValidator;
6054
}
6155
}
6256

63-
private FileValidator(
64-
Version version,
65-
Map<String, Schema> schemas
57+
protected FileValidator(
58+
Version version
6659
) {
6760
this.version = version;
68-
this.schemas = schemas;
6961
}
7062

71-
public FileValidationResult validate(String feedName, JSONObject feed) {
72-
if (schemas.containsKey(feedName)) {
73-
return validate(schemas.get(feedName), feed, feedName);
74-
}
75-
76-
logger.warn("Schema not found for gbfs feed={} version={}", feedName, version.getVersion());
77-
return null;
78-
}
63+
public FileValidationResult validate(String feedName, Map<String, JSONObject> feedMap) {
64+
if (version.getFileNames().contains(feedName)) {
65+
JSONObject feed = feedMap.get(feedName);
66+
FileValidationResult fileValidationResult = new FileValidationResult();
67+
fileValidationResult.setFile(feedName);
68+
fileValidationResult.setRequired(isRequired(feedName));
69+
fileValidationResult.setExists(feed != null);
70+
fileValidationResult.setSchema(version.getSchema(feedName, feedMap).toString());
71+
fileValidationResult.setFileContents(Optional.ofNullable(feed).map(JSONObject::toString).orElse(null));
72+
fileValidationResult.setVersion(version.getVersionString());
73+
74+
try {
75+
version.validate(feedName, feedMap);
76+
} catch (ValidationException validationException) {
77+
fileValidationResult.setErrors(mapToValidationErrors(validationException));
78+
fileValidationResult.setErrorsCount(validationException.getViolationCount());
79+
}
7980

80-
private FileValidationResult validate(Schema schema, JSONObject feed, String feedName) {
81-
FileValidationResult fileValidationResult = new FileValidationResult();
82-
fileValidationResult.setFile(feedName);
83-
fileValidationResult.setRequired(isRequired(feedName));
84-
fileValidationResult.setExists(feed != null);
85-
fileValidationResult.setSchema(schema.toString());
86-
fileValidationResult.setFileContents(Optional.ofNullable(feed).map(JSONObject::toString).orElse(null));
87-
fileValidationResult.setVersion(version.getVersion());
88-
89-
try {
90-
schema.validate(feed);
91-
} catch (ValidationException validationException) {
92-
fileValidationResult.setErrors(mapToValidationErrors(validationException));
93-
fileValidationResult.setErrorsCount(validationException.getViolationCount());
81+
return fileValidationResult;
9482
}
9583

96-
return fileValidationResult;
84+
logger.warn("Schema not found for gbfs feed={} version={}", feedName, version.getVersionString());
85+
return null;
9786
}
9887

9988
List<FileValidationError> mapToValidationErrors(ValidationException validationException) {
@@ -115,39 +104,11 @@ private boolean isRequired(String feedName) {
115104
return version.isFileRequired(feedName);
116105
}
117106

118-
protected static Map<String, Schema> getSchemas(Version version) {
119-
Map<String, Schema> schemas = new HashMap<>();
120-
version.getFeeds().forEach(feed -> {
121-
Schema schema = loadSchema(version.getVersion(), feed);
122-
if (schema != null) {
123-
schemas.put(feed, schema);
124-
}
125-
});
126-
return schemas;
127-
}
128-
129-
protected static Schema loadSchema(String version, String feedName) {
130-
InputStream inputStream = FileValidator.class.getClassLoader().getResourceAsStream("schema/v"+version+"/"+feedName+".json");
131-
132-
if (inputStream == null) {
133-
logger.warn("Unable to load schema version={} feedName={}", version, feedName);
134-
return null;
135-
}
136-
137-
JSONObject rawSchema = new JSONObject(new JSONTokener(inputStream));
138-
SchemaLoader schemaLoader = SchemaLoader.builder()
139-
.enableOverrideOfBuiltInFormatValidators()
140-
.addFormatValidator(new URIFormatValidator())
141-
.schemaJson(rawSchema)
142-
.build();
143-
144-
return schemaLoader.load().build();
145-
}
146107

147108
public void validateMissingFile(FileValidationResult fvr) {
148-
if (version.getFeeds().contains(fvr.getFile())) {
149-
fvr.setVersion(version.getVersion());
150-
fvr.setSchema(schemas.get(fvr.getFile()).toString());
109+
if (version.getFileNames().contains(fvr.getFile())) {
110+
fvr.setVersion(version.getVersionString());
111+
fvr.setSchema(version.getSchema(fvr.getFile()).toString());
151112
fvr.setRequired(version.isFileRequired(fvr.getFile()));
152113
}
153114
}

‎src/main/java/org/entur/gbfs/validation/validator/GbfsJsonValidator.java‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,10 @@
2222
import org.entur.gbfs.validation.model.FileValidationResult;
2323
import org.entur.gbfs.validation.model.ValidationResult;
2424
import org.entur.gbfs.validation.model.ValidationSummary;
25-
import org.entur.gbfs.validation.versions.Version;
26-
import org.entur.gbfs.validation.versions.VersionFactory;
25+
import org.entur.gbfs.validation.validator.versions.Version;
26+
import org.entur.gbfs.validation.validator.versions.VersionFactory;
2727
import org.json.JSONObject;
28+
import org.json.JSONTokener;
2829
import org.slf4j.Logger;
2930
import org.slf4j.LoggerFactory;
3031

@@ -71,12 +72,12 @@ public ValidationResult validate(Map<String, InputStream> rawFeeds) {
7172
ValidationSummary summary = new ValidationSummary();
7273
Map<String, FileValidationResult> fileValidations = new HashMap<>();
7374

74-
FEEDS.forEach(feed-> fileValidations.put(feed, validateFile(feed, feedMap.get(feed))));
75+
FEEDS.forEach(feed-> fileValidations.put(feed, validateFile(feed, feedMap)));
7576

7677
Version version = findVersion(fileValidations);
7778
handleMissingFiles(fileValidations, version);
7879

79-
summary.setVersion(version.getVersion());
80+
summary.setVersion(version.getVersionString());
8081
summary.setTimestamp(System.currentTimeMillis());
8182
summary.setErrorsCount(
8283
fileValidations.values().stream()
@@ -91,11 +92,11 @@ public ValidationResult validate(Map<String, InputStream> rawFeeds) {
9192

9293
@Override
9394
public FileValidationResult validateFile(String fileName, InputStream file) {
94-
return validateFile(fileName, parseFeed(file));
95+
return validateFile(fileName, Map.of(fileName, new JSONObject(new JSONTokener(file))));
9596
}
9697

9798
private void handleMissingFiles(Map<String, FileValidationResult> fileValidations, Version version) {
98-
FileValidator fileValidator = FileValidator.getFileValidator(version.getVersion());
99+
FileValidator fileValidator = FileValidator.getFileValidator(version.getVersionString());
99100
fileValidations.values().stream()
100101
.filter(fvr -> !fvr.isExists())
101102
.forEach(fileValidator::validateMissingFile);
@@ -117,8 +118,8 @@ private Version findVersion(Map<String, FileValidationResult> fileValidations) {
117118
);
118119
}
119120

120-
private FileValidationResult validateFile(String feedName, JSONObject feed) {
121-
121+
private FileValidationResult validateFile(String feedName, Map<String, JSONObject> feedMap) {
122+
JSONObject feed = feedMap.get(feedName);
122123
if (feed == null) {
123124
FileValidationResult result = new FileValidationResult();
124125
result.setFile(feedName);
@@ -135,7 +136,7 @@ private FileValidationResult validateFile(String feedName, JSONObject feed) {
135136

136137
// find correct file validator
137138
FileValidator fileValidator = FileValidator.getFileValidator(detectedVersion);
138-
return fileValidator.validate(feedName, feed);
139+
return fileValidator.validate(feedName, feedMap);
139140
}
140141

141142
private Map<String, JSONObject> parseFeeds(Map<String, InputStream> rawFeeds) {

‎src/main/java/org/entur/gbfs/validation/validator/rules/CurrentRangeMetersIsRequiredInVehicleStatusForMotorizedVehicles.java‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
/*
2+
*
3+
* *
4+
* *
5+
* * * Licensed under the EUPL, Version 1.2 or – as soon they will be approved by
6+
* * * the European Commission - subsequent versions of the EUPL (the "Licence");
7+
* * * You may not use this work except in compliance with the Licence.
8+
* * * You may obtain a copy of the Licence at:
9+
* * *
10+
* * * https://joinup.ec.europa.eu/software/page/eupl
11+
* * *
12+
* * * Unless required by applicable law or agreed to in writing, software
13+
* * * distributed under the Licence is distributed on an "AS IS" basis,
14+
* * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* * * See the Licence for the specific language governing permissions and
16+
* * * limitations under the Licence.
17+
* *
18+
*
19+
*/
20+
21+
package org.entur.gbfs.validation.validator.rules;
22+
23+
import com.jayway.jsonpath.DocumentContext;
24+
import com.jayway.jsonpath.Filter;
25+
import com.jayway.jsonpath.JsonPath;
26+
import org.json.JSONArray;
27+
import org.json.JSONObject;
28+
29+
import java.util.List;
30+
import java.util.Map;
31+
32+
import static com.jayway.jsonpath.Criteria.where;
33+
34+
/**
35+
* It is required to provide the current_range_meters property in vehicle_status for motorized vehicles
36+
*/
37+
public class CurrentRangeMetersIsRequiredInVehicleStatusForMotorizedVehicles implements CustomRuleSchemaPatcher {
38+
39+
private final String fileName;
40+
41+
public CurrentRangeMetersIsRequiredInVehicleStatusForMotorizedVehicles(String fileName) {
42+
this.fileName = fileName;
43+
}
44+
45+
private static final Filter motorizedVehicleTypesFilter = Filter.filter(
46+
where("propulsion_type").in(
47+
List.of(
48+
"electric_assist", "electric", "combustion"
49+
)
50+
)
51+
);
52+
private static final String BIKE_ITEMS_SCHEMA_PATH = "$.properties.data.properties.bikes.items";
53+
private static final String VEHICLE_ITEMS_SCHEMA_PATH = "$.properties.data.properties.vehicles.items";
54+
55+
@Override
56+
public DocumentContext addRule(DocumentContext rawSchemaDocumentContext, Map<String, JSONObject> feeds) {
57+
JSONObject vehicleTypesFeed = feeds.get("vehicle_types");
58+
59+
JSONArray motorizedVehicleTypeIds = null;
60+
61+
if (vehicleTypesFeed != null) {
62+
motorizedVehicleTypeIds = JsonPath.parse(vehicleTypesFeed)
63+
.read("$.data.vehicle_types[?].vehicle_type_id", motorizedVehicleTypesFilter);
64+
}
65+
66+
String schemaPath = VEHICLE_ITEMS_SCHEMA_PATH;
67+
68+
if (fileName.equals("free_bike_status")) {
69+
schemaPath = BIKE_ITEMS_SCHEMA_PATH;
70+
}
71+
72+
JSONObject bikeItemsSchema = rawSchemaDocumentContext.read(schemaPath);
73+
74+
if (motorizedVehicleTypeIds != null && motorizedVehicleTypeIds.length() > 0) {
75+
bikeItemsSchema.put("errorMessage", new JSONObject().put("required", new JSONObject().put("vehicle_type_id", "'vehicle_type_id' is required for this vehicle type")));
76+
bikeItemsSchema
77+
.put("if",
78+
new JSONObject()
79+
.put("properties", new JSONObject().put("vehicle_type_id", new JSONObject().put("enum", motorizedVehicleTypeIds)))
80+
81+
// "required" so it only trigger "then" when "vehicle_type_id" is present.
82+
.put("required", new JSONArray().put("vehicle_type_id"))
83+
)
84+
.put("then", new JSONObject().put("required", new JSONArray().put("current_range_meters")));
85+
}
86+
87+
return rawSchemaDocumentContext.set(schemaPath, bikeItemsSchema);
88+
}
89+
}

‎src/main/java/org/entur/gbfs/validation/validator/rules/CustomRuleSchemaPatcher.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
/*
2+
*
3+
* *
4+
* *
5+
* * * Licensed under the EUPL, Version 1.2 or – as soon they will be approved by
6+
* * * the European Commission - subsequent versions of the EUPL (the "Licence");
7+
* * * You may not use this work except in compliance with the Licence.
8+
* * * You may obtain a copy of the Licence at:
9+
* * *
10+
* * * https://joinup.ec.europa.eu/software/page/eupl
11+
* * *
12+
* * * Unless required by applicable law or agreed to in writing, software
13+
* * * distributed under the Licence is distributed on an "AS IS" basis,
14+
* * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* * * See the Licence for the specific language governing permissions and
16+
* * * limitations under the Licence.
17+
* *
18+
*
19+
*/
20+
21+
package org.entur.gbfs.validation.validator.rules;
22+
23+
import com.jayway.jsonpath.DocumentContext;
24+
import org.json.JSONObject;
25+
26+
import java.util.Map;
27+
28+
public interface CustomRuleSchemaPatcher {
29+
DocumentContext addRule(DocumentContext rawSchemaDocumentContext, Map<String, JSONObject> feeds);
30+
}

0 commit comments

Comments
 (0)