Skip to content

Commit 539e019

Browse files
mprinsmprins
committed
HTM-1964: Add GeoJSON extract format (#1702)
For now use GeoTools 35-SNAPSHOT to make geojson export work, see https://osgeo-org.atlassian.net/browse/GEOT-7894
1 parent 6adf124 commit 539e019

4 files changed

Lines changed: 101 additions & 19 deletions

File tree

‎pom.xml‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ SPDX-License-Identifier: MIT
101101
<maven.compiler.target>${java.version}</maven.compiler.target>
102102
<maven.compiler.release>${java.version}</maven.compiler.release>
103103
<project.build.outputTimestamp>2026-05-05T11:48:40Z</project.build.outputTimestamp>
104-
<geotools.version>34.3</geotools.version>
104+
<geotools.version>35-SNAPSHOT</geotools.version>
105105
<jts.version>1.20.0</jts.version>
106106
<okhttp.version>5.3.2</okhttp.version>
107107
<greenmail.version>2.1.8</greenmail.version>
@@ -324,6 +324,10 @@ SPDX-License-Identifier: MIT
324324
<artifactId>gt-excel-writer</artifactId>
325325
<version>[35-SNAPSHOT,)</version>
326326
</dependency>
327+
<dependency>
328+
<groupId>org.geotools</groupId>
329+
<artifactId>gt-geojson-store</artifactId>
330+
</dependency>
327331
<dependency>
328332
<groupId>org.geotools</groupId>
329333
<artifactId>gt-http</artifactId>
@@ -696,6 +700,14 @@ SPDX-License-Identifier: MIT
696700
<name>B3Partners public repository</name>
697701
<url>https://repo.b3p.nl/nexus/repository/public/</url>
698702
</repository>
703+
<repository>
704+
<snapshots>
705+
<enabled>true</enabled>
706+
</snapshots>
707+
<id>OSGeo-snapshots</id>
708+
<name>Snapshots hosted by OSGeo</name>
709+
<url>https://repo.osgeo.org/repository/snapshot/</url>
710+
</repository>
699711
</repositories>
700712
<pluginRepositories />
701713
<build>

‎src/main/java/org/tailormap/api/controller/LayerExtractController.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -267,10 +267,10 @@ private void validateExcelLimits(TMFeatureType featureType, Set<String> attribut
267267
}
268268

269269
public enum ExtractOutputFormat {
270-
CSV("csv", "csv"),
271-
GEOJSON("geojson", "json"),
272-
XLSX("xlsx", "xlsx"),
273-
SHAPE("shape", "zip");
270+
CSV("csv", ".csv"),
271+
GEOJSON("geojson", ".geojson"),
272+
XLSX("xlsx", ".xlsx"),
273+
SHAPE("shape", ".zip");
274274

275275
private final String value;
276276
private final String extension;

‎src/main/java/org/tailormap/api/service/CreateLayerExtractService.java‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import org.geotools.data.DataUtilities;
3737
import org.geotools.data.DefaultTransaction;
3838
import org.geotools.data.csv.CSVDataStoreFactory;
39+
import org.geotools.data.geojson.store.GeoJSONDataStoreFactory;
3940
import org.geotools.factory.CommonFactoryFinder;
4041
import org.geotools.feature.SchemaException;
4142
import org.geotools.filter.text.cql2.CQLException;
@@ -183,11 +184,11 @@ public String createExtractFilename(
183184
if (cleanFTName.contains(":")) {
184185
// clip off the WFS namespace part
185186
cleanFTName = cleanFTName.substring(cleanFTName.lastIndexOf(":") + 1);
186-
// remove: . _ which are used as separators in the filename and could cause issues when parsing the filename
187-
// later on
187+
// remove: '.' and '_' which are used as separators in the filename and could cause issues when parsing the
188+
// filename later on
188189
cleanFTName = cleanFTName.replaceAll("[._]", "");
189190
}
190-
return "%s_%s_%s.%s".formatted(cleanFTName, clientId, UUIDv7.randomV7(), outputFormat.getExtension());
191+
return "%s_%s_%s%s".formatted(cleanFTName, clientId, UUIDv7.randomV7(), outputFormat.getExtension());
191192
}
192193

193194
@Async("extractTaskExecutor")
@@ -241,7 +242,10 @@ public void createLayerExtract(
241242
SimpleFeatureType fType =
242243
DataUtilities.createSubType(inputFeatureSource.getSchema(), attributes.toArray(new String[0]));
243244
outputDataStore.createSchema(fType);
244-
245+
// as a workaround for https://osgeo-org.atlassian.net/browse/GEOT-7894 we could instead call
246+
// if (outputDataStore.getFeatureSource(fType.getName()) instanceof SimpleFeatureStore featureStore) {
247+
// but I'd rather wait for a release of geotools with a fix for that issue, because it does not work with
248+
// the CSV store
245249
if (outputDataStore.getFeatureSource() instanceof SimpleFeatureStore featureStore) {
246250
featureStore.setTransaction(outputTransaction);
247251
featureStore.addFeatureListener(event -> {
@@ -261,12 +265,13 @@ public void createLayerExtract(
261265
});
262266
featureStore.addFeatures(inputFeatureSource.getFeatures(q));
263267
outputTransaction.commit();
268+
this.emitProgress(clientId, outputFileName, 100, true, "Extract completed successfully");
269+
outputDataStore.dispose();
264270
} else {
271+
outputDataStore.dispose();
265272
this.emitError(clientId, "Output datastore is not a SimpleFeatureStore, cannot write features");
266273
logger.error("Output datastore is not a SimpleFeatureStore, cannot write features");
267274
}
268-
outputDataStore.dispose();
269-
this.emitProgress(clientId, outputFileName, 100, true, "Extract completed successfully");
270275
} catch (IOException | CQLException | SchemaException e) {
271276
emitError(clientId, e.getMessage());
272277
logger.error("Creating extract failed", e);
@@ -324,12 +329,16 @@ private FileDataStore getExtractDataStore(
324329
ExcelDataStoreFactory.FILE_PARAM.key,
325330
outputFile,
326331
ExcelDataStoreFactory.SHEET_PARAM.key,
327-
// typeName could hve a prefix; for Excel sheet names ':' is disallowed, max length is 31
332+
// typeName could have a prefix; for Excel sheet names ':' is disallowed, max length is 31
328333
typeName.substring(typeName.lastIndexOf(":") + 1, Math.min(typeName.length(), 31)));
329334
return (FileDataStore) new ExcelDataStoreFactory().createNewDataStore(params);
330335
}
336+
case GEOJSON -> {
337+
Map<String, Serializable> params = Map.of(GeoJSONDataStoreFactory.FILE_PARAM.key, outputFile);
338+
return (FileDataStore) new GeoJSONDataStoreFactory().createNewDataStore(params);
339+
}
331340
// TODO implement
332-
case GEOJSON, SHAPE -> {
341+
case SHAPE -> {
333342
emitError(clientId, "Output format " + extractOutputFormat + " is not yet supported");
334343
logger.error("Output format {} is not yet supported", extractOutputFormat);
335344
throw new IOException("Unsupported output format: " + extractOutputFormat);

‎src/test/java/org/tailormap/api/controller/LayerExtractControllerIntegrationTest.java‎

Lines changed: 67 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import static java.util.concurrent.TimeUnit.SECONDS;
1010
import static org.hamcrest.MatcherAssert.assertThat;
1111
import static org.hamcrest.Matchers.containsString;
12+
import static org.hamcrest.Matchers.endsWith;
1213
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
1314
import static org.hamcrest.Matchers.not;
1415
import static org.hamcrest.Matchers.startsWith;
@@ -17,9 +18,12 @@
1718
import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
1819
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
1920
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
21+
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
2022
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.request;
2123
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
2224
import static org.tailormap.api.TestRequestProcessor.setServletPath;
25+
import static org.tailormap.api.controller.LayerExtractController.ExtractOutputFormat.CSV;
26+
import static org.tailormap.api.controller.LayerExtractController.ExtractOutputFormat.GEOJSON;
2327
import static org.tailormap.api.controller.TestUrls.layerBegroeidTerreindeelPostgis;
2428
import static org.tailormap.api.controller.TestUrls.layerProxiedWithAuthInPublicApp;
2529

@@ -91,7 +95,7 @@ void should_export_large_filter_to_csv() throws Exception {
9195
.with(setServletPath(extractUrl))
9296
.with(csrf())
9397
.param("attributes", "")
94-
.param("outputFormat", "csv")
98+
.param("outputFormat", CSV.getValue())
9599
.param("filter", StaticTestData.get("large_cql_filter"))
96100
.acceptCharset(StandardCharsets.UTF_8)
97101
.characterEncoding(StandardCharsets.UTF_8)
@@ -115,7 +119,7 @@ void should_export_large_filter_to_csv() throws Exception {
115119
assertThat(lastCompletedEventJson.length(), greaterThanOrEqualTo(100));
116120

117121
final String extractedDownloadId = getDownloadId(lastCompletedEventJson);
118-
assertThat(extractedDownloadId, containsString(".csv"));
122+
assertThat(extractedDownloadId, endsWith(CSV.getExtension()));
119123

120124
final String downloadUrl = apiBasePath + layerBegroeidTerreindeelPostgis + downloadPath + extractedDownloadId;
121125
MvcResult download = mockMvc.perform(get(downloadUrl).with(setServletPath(downloadUrl)))
@@ -145,7 +149,7 @@ void should_export_large_output_to_csv() throws Exception {
145149
.with(setServletPath(extractUrl))
146150
.with(csrf())
147151
.param("attributes", "identificatie, class")
148-
.param("outputFormat", "csv")
152+
.param("outputFormat", CSV.getValue())
149153
.acceptCharset(StandardCharsets.UTF_8)
150154
.characterEncoding(StandardCharsets.UTF_8)
151155
.contentType(MediaType.APPLICATION_FORM_URLENCODED))
@@ -168,7 +172,7 @@ void should_export_large_output_to_csv() throws Exception {
168172
assertThat(lastCompletedEventJson.length(), greaterThanOrEqualTo(100));
169173

170174
final String extractedDownloadId = getDownloadId(lastCompletedEventJson);
171-
assertThat(extractedDownloadId, containsString(".csv"));
175+
assertThat(extractedDownloadId, endsWith(CSV.getExtension()));
172176

173177
final String downloadUrl = apiBasePath + layerBegroeidTerreindeelPostgis + downloadPath + extractedDownloadId;
174178
MvcResult download = mockMvc.perform(get(downloadUrl).with(setServletPath(downloadUrl)))
@@ -210,7 +214,7 @@ void should_export_wfs_to_csv_with_authentication() throws Exception {
210214
.with(setServletPath(extractUrl))
211215
.with(csrf())
212216
.param("attributes", "geom,naam,code")
213-
.param("outputFormat", "csv")
217+
.param("outputFormat", CSV.getValue())
214218
.acceptCharset(StandardCharsets.UTF_8)
215219
.characterEncoding(StandardCharsets.UTF_8)
216220
.contentType(MediaType.APPLICATION_FORM_URLENCODED))
@@ -233,7 +237,7 @@ void should_export_wfs_to_csv_with_authentication() throws Exception {
233237
assertThat(lastCompletedEventJson.length(), greaterThanOrEqualTo(100));
234238

235239
final String extractedDownloadId = getDownloadId(lastCompletedEventJson);
236-
assertThat(extractedDownloadId, containsString(".csv"));
240+
assertThat(extractedDownloadId, endsWith(CSV.getExtension()));
237241

238242
final String downloadUrl = apiBasePath + layerProxiedWithAuthInPublicApp + downloadPath + extractedDownloadId;
239243
MvcResult download = mockMvc.perform(get(downloadUrl).with(setServletPath(downloadUrl)))
@@ -347,6 +351,63 @@ void should_export_large_filter_to_excel() throws Exception {
347351
}
348352
}
349353

354+
@Test
355+
void should_export_large_filter_to_geojson() throws Exception {
356+
final String extractUrl = apiBasePath + layerBegroeidTerreindeelPostgis + extractPath + sseClientId;
357+
mockMvc.perform(post(extractUrl)
358+
.accept(MediaType.APPLICATION_JSON)
359+
.with(setServletPath(extractUrl))
360+
.with(csrf())
361+
.param("attributes", "")
362+
.param("outputFormat", GEOJSON.getValue())
363+
.param("filter", StaticTestData.get("large_cql_filter"))
364+
.acceptCharset(StandardCharsets.UTF_8)
365+
.characterEncoding(StandardCharsets.UTF_8)
366+
.contentType(MediaType.APPLICATION_FORM_URLENCODED))
367+
.andExpect(status().isAccepted());
368+
369+
// The SseEventBus may dispatch events slightly after the POST returns.
370+
// Awaitility polls the buffered SSE response until the expected content appears.
371+
Awaitility.await()
372+
.atMost(10, SECONDS)
373+
.untilAsserted(() -> assertThat(
374+
sseResult.getResponse().getContentAsString(), containsString("Extract task received")));
375+
376+
Awaitility.await().pollInterval(5, SECONDS).atMost(30, SECONDS).untilAsserted(() -> {
377+
final String stream = sseResult.getResponse().getContentAsString();
378+
assertThat(count_completed_messages(stream), greaterThanOrEqualTo(1));
379+
});
380+
381+
final String lastCompletedEventJson =
382+
getLastCompletedEventJson(sseResult.getResponse().getContentAsString());
383+
assertThat(lastCompletedEventJson.length(), greaterThanOrEqualTo(100));
384+
385+
final String extractedDownloadId = getDownloadId(lastCompletedEventJson);
386+
assertThat(extractedDownloadId, endsWith(GEOJSON.getExtension()));
387+
388+
final String downloadUrl = apiBasePath + layerBegroeidTerreindeelPostgis + downloadPath + extractedDownloadId;
389+
mockMvc.perform(get(downloadUrl).with(setServletPath(downloadUrl)))
390+
.andExpect(status().isOk())
391+
.andExpect(result -> {
392+
String contentType = result.getResponse().getContentType();
393+
assertThat(contentType, containsString("application/geo+json"));
394+
395+
String contentDisposition = result.getResponse().getHeader("Content-Disposition");
396+
assertThat(contentDisposition, containsString("attachment; filename="));
397+
assertThat(contentDisposition, containsString(extractedDownloadId));
398+
})
399+
.andExpect(jsonPath("$.type").value("FeatureCollection"))
400+
.andExpect(jsonPath("$.features.length()").value(18))
401+
.andExpect(jsonPath("$.features[0].type").value("Feature"))
402+
.andExpect(jsonPath("$.features[0].geometry").isNotEmpty())
403+
.andExpect(jsonPath("$.features[0].properties.length()").value(13))
404+
.andExpect(jsonPath("$.features[0].properties.bronhouder").value("G0344"))
405+
.andExpect(jsonPath("$.features[0].geometry.type").value("Polygon"))
406+
// no CRS members
407+
.andExpect(jsonPath("$.crs").doesNotHaveJsonPath())
408+
.andExpect(jsonPath("$.features[0].crs").doesNotHaveJsonPath());
409+
}
410+
350411
/**
351412
* Parse the last non-empty line from the SSE stream that looks something like:
352413
* {@code data:{"details":{"message":"Extract task

0 commit comments

Comments
 (0)