HTM-1977: implement geopackage extract

Author: mprins

.mvn/jvm.config

@@ -11,4 +11,5 @@
 --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
 --add-opens jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED
 --add-opens jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED
---add-modules=jdk.incubator.vector
+--add-modules=jdk.incubator.vector
+--enable-native-access=ALL-UNNAMED

pom.xml

@@ -328,6 +328,10 @@ SPDX-License-Identifier: MIT
             <groupId>org.geotools</groupId>
             <artifactId>gt-geojson-store</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.geotools</groupId>
+            <artifactId>gt-geopkg</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.geotools</groupId>
             <artifactId>gt-http</artifactId>

src/main/java/org/tailormap/api/controller/LayerExtractController.java

@@ -279,6 +279,7 @@ private void validateExcelLimits(TMFeatureType featureType, Set<String> attribut
   }
 
   public enum ExtractOutputFormat {
+    GEOPACKAGE("geopackage", ".gpkg"),
     CSV("csv", ".csv"),
     GEOJSON("geojson", ".geojson"),
     XLSX("xlsx", ".xlsx"),

src/main/java/org/tailormap/api/service/CreateLayerExtractService.java

@@ -7,25 +7,8 @@
 
 import ch.rasc.sse.eventbus.SseEvent;
 import ch.rasc.sse.eventbus.SseEventBus;
-import java.io.File;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.time.Instant;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Stream;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
 import org.apache.commons.lang3.StringUtils;
+import org.geotools.api.data.DataStore;
 import org.geotools.api.data.FeatureEvent;
 import org.geotools.api.data.FileDataStore;
 import org.geotools.api.data.Query;
@@ -43,6 +26,7 @@
 import org.geotools.data.shapefile.ShapefileDumper;
 import org.geotools.factory.CommonFactoryFinder;
 import org.geotools.feature.SchemaException;
+import org.geotools.geopkg.GeoPkgDataStoreFactory;
 import org.geotools.util.factory.GeoTools;
 import org.jspecify.annotations.NonNull;
 import org.jspecify.annotations.Nullable;
@@ -66,6 +50,25 @@
 import tools.jackson.databind.SerializationFeature;
 import tools.jackson.databind.json.JsonMapper;
 
+import java.io.File;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Stream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
 @Service
 public class CreateLayerExtractService {
   private static final Logger logger =
@@ -218,6 +221,9 @@ public void createLayerExtract(
     this.emitProgress(clientId, outputFileName, 0, false, "Starting extract");
 
     switch (extractOutputFormat) {
+      case GEOPACKAGE ->
+        this.handleGeoPackage(
+            clientId, inputTmFeatureType, attributes, filter, sortBy, sortOrder, outputFileName);
       case SHAPE ->
         this.handleWithShapeDumper(
             clientId, inputTmFeatureType, attributes, filter, sortBy, sortOrder, outputFileName);
@@ -234,6 +240,82 @@ public void createLayerExtract(
     }
   }
 
+  private void handleGeoPackage(
+      @NonNull String clientId,
+      @NonNull TMFeatureType inputTmFeatureType,
+      @NonNull Set<String> attributes,
+      Filter filter,
+      String sortBy,
+      SortOrder sortOrder,
+      @NonNull String outputFileName) {
+
+    SimpleFeatureSource inputFeatureSource = null;
+    DataStore outputDataStore = null;
+
+    try (Transaction outputTransaction = new DefaultTransaction("tailormap-extract-output")) {
+      inputFeatureSource = featureSourceFactoryHelper.openGeoToolsFeatureSource(inputTmFeatureType);
+
+      Query q = createQuery(inputFeatureSource, attributes, filter, sortBy, sortOrder);
+
+      int featCount = getFeatureCount(inputFeatureSource, q);
+      if (featCount < 0) {
+        logger.warn("Could not determine feature count for extract, progress reporting will be omitted");
+      }
+
+      outputDataStore = new GeoPkgDataStoreFactory()
+          .createDataStore(Map.of(
+              GeoPkgDataStoreFactory.DBTYPE.key,
+              "geopkg",
+              GeoPkgDataStoreFactory.DATABASE.key,
+              getValidatedOutputFile(outputFileName),
+              GeoPkgDataStoreFactory.CONTENTS_ONLY.key,
+              false));
+
+      SimpleFeatureType fType =
+          DataUtilities.createSubType(inputFeatureSource.getSchema(), attributes.toArray(new String[0]));
+      outputDataStore.createSchema(fType);
+
+      final AtomicInteger featsAdded = new AtomicInteger();
+      if (outputDataStore.getFeatureSource(fType.getName()) instanceof SimpleFeatureStore featureStore) {
+        featureStore.setTransaction(outputTransaction);
+        featureStore.addFeatureListener(event -> {
+          if (event.getType().equals(FeatureEvent.Type.ADDED)) {
+            featsAdded.getAndIncrement();
+            logger.debug("Added feature {}", featsAdded.get());
+          }
+          if (featCount > 0) {
+            if (featsAdded.get() % progressReportInterval == 0) {
+              this.emitProgress(
+                  clientId,
+                  outputFileName,
+                  (int) ((featsAdded.doubleValue() / featCount) * 100),
+                  false,
+                  null);
+            }
+          }
+        });
+        featureStore.addFeatures(inputFeatureSource.getFeatures(q));
+        outputTransaction.commit();
+        outputDataStore.dispose();
+        this.emitProgress(clientId, outputFileName, 100, true, "Extract completed successfully");
+      }
+    } catch (SchemaException | IOException | IllegalArgumentException e) {
+      emitError(clientId, e.getMessage());
+      logger.error("Creating extract failed", e);
+    } finally {
+      if (inputFeatureSource != null) {
+        try {
+          inputFeatureSource.getDataStore().dispose();
+        } catch (Exception e) {
+          logger.warn("Error disposing datastore for feature source {}", inputFeatureSource.getName(), e);
+        }
+      }
+      if (outputDataStore != null) {
+        outputDataStore.dispose();
+      }
+    }
+  }
+
   private void handleSingleFileFormats(
       @NonNull String clientId,
       @NonNull TMFeatureType inputTmFeatureType,

src/main/resources/application.properties

@@ -29,7 +29,7 @@ tailormap-api.features.wfs_count_exact=false
 tailormap-api.feature.info.maxitems=30
 
 # see org.tailormap.api.controller.LayerExtractController.ExtractOutputFormat for valid values
-tailormap-api.extract.allowed-outputformats=csv,geojson,xlsx,shape
+tailormap-api.extract.allowed-outputformats=csv,geojson,xlsx,shape,geopackage
 # any files older than this (in minutes) in the extract output directory will be deleted by a scheduled job, to prevent filling up the disk
 # tailormap-api.extract.cleanup-minutes=120
 # the directory where the extract output files are stored, should be writable by the application

src/test/java/org/tailormap/api/controller/LayerExtractControllerIntegrationTest.java

@@ -24,6 +24,7 @@
 import static org.tailormap.api.TestRequestProcessor.setServletPath;
 import static org.tailormap.api.controller.LayerExtractController.ExtractOutputFormat.CSV;
 import static org.tailormap.api.controller.LayerExtractController.ExtractOutputFormat.GEOJSON;
+import static org.tailormap.api.controller.LayerExtractController.ExtractOutputFormat.GEOPACKAGE;
 import static org.tailormap.api.controller.LayerExtractController.ExtractOutputFormat.SHAPE;
 import static org.tailormap.api.controller.TestUrls.layerBegroeidTerreindeelPostgis;
 import static org.tailormap.api.controller.TestUrls.layerProxiedWithAuthInPublicApp;
@@ -530,4 +531,51 @@ void should_export_large_filter_to_shape() throws Exception {
       assertEquals(6, extensions.size(), "Expected 6 unique file extensions in the shapefile zip");
     }
   }
+
+  @Test
+  void should_export_to_geopackage() throws Exception {
+    final String extractUrl = apiBasePath + layerBegroeidTerreindeelPostgis + extractPath + sseClientId;
+    mockMvc.perform(post(extractUrl)
+            .accept(MediaType.APPLICATION_JSON)
+            .with(setServletPath(extractUrl))
+            .with(csrf())
+            .param("attributes", "")
+            .param("outputFormat", GEOPACKAGE.getValue())
+            .param("filter", StaticTestData.get("large_cql_filter"))
+            .acceptCharset(StandardCharsets.UTF_8)
+            .characterEncoding(StandardCharsets.UTF_8)
+            .contentType(MediaType.APPLICATION_FORM_URLENCODED))
+        .andExpect(status().isAccepted());
+
+    // The SseEventBus may dispatch events slightly after the POST returns.
+    // Awaitility polls the buffered SSE response until the expected content appears.
+    Awaitility.await()
+        .atMost(10, SECONDS)
+        .untilAsserted(() -> assertThat(
+            sseResult.getResponse().getContentAsString(), containsString("Extract task received")));
+
+    Awaitility.await().pollInterval(5, SECONDS).atMost(30, SECONDS).untilAsserted(() -> {
+      final String stream = sseResult.getResponse().getContentAsString();
+      assertThat(count_completed_messages(stream), greaterThanOrEqualTo(1));
+    });
+
+    final String lastCompletedEventJson =
+        getLastCompletedEventJson(sseResult.getResponse().getContentAsString());
+    assertThat(lastCompletedEventJson.length(), greaterThanOrEqualTo(100));
+
+    final String extractedDownloadId = getDownloadId(lastCompletedEventJson);
+    assertThat(extractedDownloadId, containsString(GEOPACKAGE.getExtension()));
+
+    final String downloadUrl = apiBasePath + layerBegroeidTerreindeelPostgis + downloadPath + extractedDownloadId;
+    mockMvc.perform(get(downloadUrl).with(setServletPath(downloadUrl)))
+        .andExpect(status().isOk())
+        .andExpect(result -> {
+          String contentType = result.getResponse().getContentType();
+          assertThat(contentType, containsString("application/geopackage+sqlite3"));
+
+          String contentDisposition = result.getResponse().getHeader("Content-Disposition");
+          assertThat(contentDisposition, containsString("attachment; filename="));
+          assertThat(contentDisposition, containsString(extractedDownloadId));
+        });
+  }
 }

src/test/resources/application.properties

@@ -3,7 +3,7 @@ tailormap-api.admin.base-path=/api/admin
 management.endpoints.web.base-path=/api/actuator
 tailormap-api.new-admin-username=tm-admin
 # see org.tailormap.api.controller.LayerExtractController.ExtractOutputFormat for valid values
-tailormap-api.extract.allowed-outputformats=csv,geojson,xlsx,shape
+tailormap-api.extract.allowed-outputformats=csv,geojson,xlsx,shape,geopackage
 # the number of features after which a progress report is sent back to the viewer, to update the progress bar
 tailormap-api.extract.progress-report-interval=10
 # any files older than this (in minutes) in the extract output directory will be deleted by a scheduled job, to prevent filling up the disk