Skip to content

Commit d9c823f

Browse files
mprinsmprins
committed
HTM-1977: Implement GeoPackage export using low-level API, which seems more robust
Add --enable-native-access=ALL-UNNAMED for the native sqlite/geopackage driver
1 parent 5ea4edf commit d9c823f

3 files changed

Lines changed: 101 additions & 65 deletions

File tree

‎pom.xml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1017,6 +1017,8 @@ SPDX-License-Identifier: MIT
10171017
See https://github.com/orgs/paketo-buildpacks/discussions/241
10181018
-->
10191019
<BPE_APPEND_JAVA_TOOL_OPTIONS xml:space="preserve"> -XX:MaxDirectMemorySize=256M</BPE_APPEND_JAVA_TOOL_OPTIONS>
1020+
<!-- for GeoPackage support which uses a native driver -->
1021+
<BPE_APPEND_JAVA_TOOL_OPTIONS xml:space="preserve"> --enable-native-access=ALL-UNNAMED</BPE_APPEND_JAVA_TOOL_OPTIONS>
10201022
<!-- Headroom is used by the memory calculator to reduce the max total memory limit. The default is 0%,
10211023
but since Tailormap is usually run with unconstrained container memory, set it to 10% to prevent taking
10221024
too much host memory. Although Tailormap should not exhaust heap memory, reduce it as a preventive safety

‎src/main/java/org/tailormap/api/service/CreateLayerExtractService.java‎

Lines changed: 67 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,25 @@
77

88
import ch.rasc.sse.eventbus.SseEvent;
99
import ch.rasc.sse.eventbus.SseEventBus;
10+
import java.io.File;
11+
import java.io.IOException;
12+
import java.lang.invoke.MethodHandles;
13+
import java.nio.charset.StandardCharsets;
14+
import java.nio.file.Files;
15+
import java.nio.file.Path;
16+
import java.time.Instant;
17+
import java.util.ArrayList;
18+
import java.util.Comparator;
19+
import java.util.List;
20+
import java.util.Map;
21+
import java.util.Objects;
22+
import java.util.Set;
23+
import java.util.concurrent.TimeUnit;
24+
import java.util.concurrent.atomic.AtomicInteger;
25+
import java.util.stream.Stream;
26+
import java.util.zip.ZipEntry;
27+
import java.util.zip.ZipOutputStream;
1028
import org.apache.commons.lang3.StringUtils;
11-
import org.geotools.api.data.DataStore;
1229
import org.geotools.api.data.FeatureEvent;
1330
import org.geotools.api.data.FileDataStore;
1431
import org.geotools.api.data.Query;
@@ -26,7 +43,8 @@
2643
import org.geotools.data.shapefile.ShapefileDumper;
2744
import org.geotools.factory.CommonFactoryFinder;
2845
import org.geotools.feature.SchemaException;
29-
import org.geotools.geopkg.GeoPkgDataStoreFactory;
46+
import org.geotools.geopkg.FeatureEntry;
47+
import org.geotools.geopkg.GeoPackage;
3048
import org.geotools.util.factory.GeoTools;
3149
import org.jspecify.annotations.NonNull;
3250
import org.jspecify.annotations.Nullable;
@@ -50,25 +68,6 @@
5068
import tools.jackson.databind.SerializationFeature;
5169
import tools.jackson.databind.json.JsonMapper;
5270

53-
import java.io.File;
54-
import java.io.IOException;
55-
import java.lang.invoke.MethodHandles;
56-
import java.nio.charset.StandardCharsets;
57-
import java.nio.file.Files;
58-
import java.nio.file.Path;
59-
import java.time.Instant;
60-
import java.util.ArrayList;
61-
import java.util.Comparator;
62-
import java.util.List;
63-
import java.util.Map;
64-
import java.util.Objects;
65-
import java.util.Set;
66-
import java.util.concurrent.TimeUnit;
67-
import java.util.concurrent.atomic.AtomicInteger;
68-
import java.util.stream.Stream;
69-
import java.util.zip.ZipEntry;
70-
import java.util.zip.ZipOutputStream;
71-
7271
@Service
7372
public class CreateLayerExtractService {
7473
private static final Logger logger =
@@ -250,55 +249,65 @@ private void handleGeoPackage(
250249
@NonNull String outputFileName) {
251250

252251
SimpleFeatureSource inputFeatureSource = null;
253-
DataStore outputDataStore = null;
252+
File outputFile = null;
253+
try {
254+
outputFile = getValidatedOutputFile(outputFileName);
255+
if (!logger.isDebugEnabled()) {
256+
// delete in production after JVM exit because the event bus will be reset when the JVM exits, and then
257+
// we
258+
// are unlikely to have a reference to the file anymore.
259+
// In debug/development mode we want to keep the file for inspection.
260+
outputFile.deleteOnExit();
261+
}
262+
} catch (IOException e) {
263+
emitError(clientId, e.getMessage());
264+
logger.error("Creating extract failed", e);
265+
return;
266+
}
267+
268+
try (GeoPackage geopkg = new GeoPackage(outputFile)) {
269+
geopkg.init();
254270

255-
try (Transaction outputTransaction = new DefaultTransaction("tailormap-extract-output")) {
256271
inputFeatureSource = featureSourceFactoryHelper.openGeoToolsFeatureSource(inputTmFeatureType);
257272

258273
Query q = createQuery(inputFeatureSource, attributes, filter, sortBy, sortOrder);
259274

260275
int featCount = getFeatureCount(inputFeatureSource, q);
261276
if (featCount < 0) {
262-
logger.warn("Could not determine feature count for extract, progress reporting will be omitted");
277+
logger.warn("Could not determine feature count for extract, progress reporting will be inaccurate");
263278
}
264-
265-
outputDataStore = new GeoPkgDataStoreFactory()
266-
.createDataStore(Map.of(
267-
GeoPkgDataStoreFactory.DBTYPE.key,
268-
"geopkg",
269-
GeoPkgDataStoreFactory.DATABASE.key,
270-
getValidatedOutputFile(outputFileName),
271-
GeoPkgDataStoreFactory.CONTENTS_ONLY.key,
272-
false));
279+
final boolean hasKnownFeatureCount = featCount > 0;
273280

274281
SimpleFeatureType fType =
275282
DataUtilities.createSubType(inputFeatureSource.getSchema(), attributes.toArray(new String[0]));
276-
outputDataStore.createSchema(fType);
277283

278-
final AtomicInteger featsAdded = new AtomicInteger();
279-
if (outputDataStore.getFeatureSource(fType.getName()) instanceof SimpleFeatureStore featureStore) {
280-
featureStore.setTransaction(outputTransaction);
281-
featureStore.addFeatureListener(event -> {
282-
if (event.getType().equals(FeatureEvent.Type.ADDED)) {
283-
featsAdded.getAndIncrement();
284-
logger.debug("Added feature {}", featsAdded.get());
285-
}
286-
if (featCount > 0) {
287-
if (featsAdded.get() % progressReportInterval == 0) {
288-
this.emitProgress(
289-
clientId,
290-
outputFileName,
291-
(int) ((featsAdded.doubleValue() / featCount) * 100),
292-
false,
293-
null);
294-
}
295-
}
296-
});
297-
featureStore.addFeatures(inputFeatureSource.getFeatures(q));
298-
outputTransaction.commit();
299-
outputDataStore.dispose();
300-
this.emitProgress(clientId, outputFileName, 100, true, "Extract completed successfully");
301-
}
284+
FeatureEntry entry = new FeatureEntry();
285+
entry.setTableName(fType.getTypeName());
286+
entry.setDescription(fType.getTypeName());
287+
288+
AtomicInteger lastProgress = new AtomicInteger(0);
289+
geopkg.add(
290+
entry,
291+
new ProgressReportingFeatureCollection(
292+
inputFeatureSource.getFeatures(q), progressReportInterval, processed -> {
293+
int progress = hasKnownFeatureCount ? (int) ((processed / (double) featCount) * 99) : 0;
294+
lastProgress.set(progress);
295+
String progressMessage = hasKnownFeatureCount
296+
? "Extracting geopackage: %d/%d features processed"
297+
.formatted(processed, featCount)
298+
: "Extracting geopackage: %d features processed".formatted(processed);
299+
this.emitProgress(clientId, outputFileName, progress, false, progressMessage);
300+
}));
301+
302+
this.emitProgress(
303+
clientId,
304+
outputFileName,
305+
Math.max(99, lastProgress.get()),
306+
false,
307+
"Extract geopackage created successfully");
308+
geopkg.createSpatialIndex(entry);
309+
geopkg.close();
310+
this.emitProgress(clientId, outputFileName, 100, true, "Extract completed successfully");
302311
} catch (SchemaException | IOException | IllegalArgumentException e) {
303312
emitError(clientId, e.getMessage());
304313
logger.error("Creating extract failed", e);
@@ -310,9 +319,6 @@ private void handleGeoPackage(
310319
logger.warn("Error disposing datastore for feature source {}", inputFeatureSource.getName(), e);
311320
}
312321
}
313-
if (outputDataStore != null) {
314-
outputDataStore.dispose();
315-
}
316322
}
317323
}
318324

‎src/test/java/org/tailormap/api/controller/LayerExtractControllerIntegrationTest.java‎

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import static org.hamcrest.Matchers.startsWith;
1616
import static org.junit.jupiter.api.Assertions.assertAll;
1717
import static org.junit.jupiter.api.Assertions.assertEquals;
18+
import static org.junit.jupiter.api.Assertions.assertTrue;
1819
import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
1920
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
2021
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
@@ -32,6 +33,12 @@
3233
import java.io.ByteArrayInputStream;
3334
import java.io.InputStream;
3435
import java.nio.charset.StandardCharsets;
36+
import java.nio.file.Files;
37+
import java.nio.file.Path;
38+
import java.sql.Connection;
39+
import java.sql.DriverManager;
40+
import java.sql.PreparedStatement;
41+
import java.sql.ResultSet;
3542
import java.util.HashSet;
3643
import java.util.Set;
3744
import java.util.zip.ZipEntry;
@@ -554,7 +561,7 @@ void should_export_to_geopackage() throws Exception {
554561
.untilAsserted(() -> assertThat(
555562
sseResult.getResponse().getContentAsString(), containsString("Extract task received")));
556563

557-
Awaitility.await().pollInterval(5, SECONDS).atMost(30, SECONDS).untilAsserted(() -> {
564+
Awaitility.await().pollInterval(5, SECONDS).atMost(45, SECONDS).untilAsserted(() -> {
558565
final String stream = sseResult.getResponse().getContentAsString();
559566
assertThat(count_completed_messages(stream), greaterThanOrEqualTo(1));
560567
});
@@ -564,10 +571,10 @@ void should_export_to_geopackage() throws Exception {
564571
assertThat(lastCompletedEventJson.length(), greaterThanOrEqualTo(100));
565572

566573
final String extractedDownloadId = getDownloadId(lastCompletedEventJson);
567-
assertThat(extractedDownloadId, containsString(GEOPACKAGE.getExtension()));
574+
assertThat(extractedDownloadId, endsWith(GEOPACKAGE.getExtension()));
568575

569576
final String downloadUrl = apiBasePath + layerBegroeidTerreindeelPostgis + downloadPath + extractedDownloadId;
570-
mockMvc.perform(get(downloadUrl).with(setServletPath(downloadUrl)))
577+
MvcResult download = mockMvc.perform(get(downloadUrl).with(setServletPath(downloadUrl)))
571578
.andExpect(status().isOk())
572579
.andExpect(result -> {
573580
String contentType = result.getResponse().getContentType();
@@ -576,6 +583,27 @@ void should_export_to_geopackage() throws Exception {
576583
String contentDisposition = result.getResponse().getHeader("Content-Disposition");
577584
assertThat(contentDisposition, containsString("attachment; filename="));
578585
assertThat(contentDisposition, containsString(extractedDownloadId));
579-
});
586+
})
587+
.andReturn();
588+
589+
byte[] geopackageBytes = download.getResponse().getContentAsByteArray();
590+
Path tempFile = Files.createTempFile("tm-extract-", ".gpkg");
591+
592+
try {
593+
Files.write(tempFile, geopackageBytes);
594+
595+
try (Connection connection = DriverManager.getConnection("jdbc:sqlite:" + tempFile);
596+
PreparedStatement statement =
597+
connection.prepareStatement("SELECT COUNT(*) FROM begroeidterreindeel");
598+
ResultSet resultSet = statement.executeQuery()) {
599+
600+
assertTrue(
601+
resultSet.next(), // NOPMD - we just want to check that it returns a row
602+
"Expected COUNT(*) query to return a row");
603+
assertEquals(18, resultSet.getInt(1), "Expected 18 records in begroeidterreindeel table");
604+
}
605+
} finally {
606+
Files.deleteIfExists(tempFile);
607+
}
580608
}
581609
}

0 commit comments

Comments
 (0)