HTM-1960: Custom extract functionality for layer attributes

.mvn/jvm.config

@@ -11,4 +11,5 @@
 --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
 --add-opens jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED
 --add-opens jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED
---add-modules=jdk.incubator.vector
+--add-modules=jdk.incubator.vector
+--enable-native-access=ALL-UNNAMED

README.md

@@ -217,6 +217,23 @@ mvn -Pdeveloping,postgresql verify -Dspring-boot.run.profiles=dev,populate-testd
 * You can use `mvn -U org.codehaus.mojo:versions-maven-plugin:display-dependency-updates` to search for dependency
   updates
 
+#### SSE streams
+
+We have 2 SSE streams available, one in the admin API at: `api/admin/events/{clientId}` and one in the viewer API 
+at: `api/events/{clientId}`. When using these streams you must make sure that you are using/injecting the correct
+`SseEventBus` for each API.
+
+For the admin use the default `eventBus` bean, inject using: `SseEventBus eventBus` (defined using the 
+`@EnableSseEventBus` annotation in the `TailormapConfig` class).
+For the viewer use the `viewerSseEventBus` bean, inject using: `@Qualifier("viewerSseEventBus") SseEventBus eventBus` 
+(defined in the `TailormapConfig` class). 
+See: 
+- [ServerSentEventsAdminController](src/main/java/org/tailormap/api/controller/admin/ServerSentEventsAdminController.java) for the admin configuration
+- [ServerSentEventsController](src/main/java/org/tailormap/api/controller/ServerSentEventsController.java) for the viewer configuration
+- [TailormapConfig](src/main/java/org/tailormap/api/configuration/TailormapConfig.java) for the bean definitions
+
+If you inject the wrong one you may not receive the events you want, and you risk sending administrative events to the viewer.
+
 ## Releasing
 
 ### Prerequisites

build/ci/docker-compose.yml

@@ -181,4 +181,4 @@ services:
       start_period: 60s
       interval: 15s
       timeout: 5s
-      retries: 3
+      retries: 3

build/qa/PMD-ruleset_for_TM.xml

@@ -113,7 +113,7 @@ SPDX-License-Identifier: MIT
     <rule ref="category/java/errorprone.xml/MoreThanOneLogger"/>
     <rule ref="category/java/bestpractices.xml/UnitTestShouldIncludeAssert">
         <properties>
-            <property name="extraAssertMethodNames" value="andExpect"/>
+            <property name="extraAssertMethodNames" value="andExpect, untilAsserted"/>
         </properties>
     </rule>
     <rule ref="category/java/bestpractices.xml/SimplifiableTestAssertion"/>

pom.xml

@@ -311,10 +311,28 @@ SPDX-License-Identifier: MIT
             <groupId>org.geotools</groupId>
             <artifactId>gt-cql</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.geotools</groupId>
+            <artifactId>gt-csv</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.geotools</groupId>
             <artifactId>gt-epsg-hsql</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.geotools</groupId>
+            <artifactId>gt-excel-writer</artifactId>
+            <!-- gt-excel-writer is not in the gt-bom, so we need to specify the version here -->
+            <version>${geotools.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.geotools</groupId>
+            <artifactId>gt-geojson-store</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.geotools</groupId>
+            <artifactId>gt-geopkg</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.geotools</groupId>
             <artifactId>gt-http</artifactId>
@@ -335,6 +353,10 @@ SPDX-License-Identifier: MIT
             <groupId>org.geotools</groupId>
             <artifactId>gt-referencing</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.geotools</groupId>
+            <artifactId>gt-shapefile</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.geotools</groupId>
             <artifactId>gt-wfs-ng</artifactId>
@@ -679,6 +701,14 @@ SPDX-License-Identifier: MIT
             <name>Releases hosted by OSGeo</name>
             <url>https://repo.osgeo.org/repository/release/</url>
         </repository>
+        <repository>
+            <snapshots>
+                <enabled>true</enabled>
+            </snapshots>
+            <id>repo.b3p.nl</id>
+            <name>B3Partners public repository</name>
+            <url>https://repo.b3p.nl/nexus/repository/public/</url>
+        </repository>
     </repositories>
     <pluginRepositories />
     <build>
@@ -974,12 +1004,14 @@ SPDX-License-Identifier: MIT
                              alternatively, use environment variable BPL_JVM_CLASS_ADJUSTMENT when deploying the docker container
                              -->
                             <BPE_DEFAULT_BPL_JVM_CLASS_ADJUSTMENT>120%</BPE_DEFAULT_BPL_JVM_CLASS_ADJUSTMENT>
-                            <!-- JVM default is the same as -Xmx. However, the Paketo Java Buildpack memory calculator sets it to
-                             10M by default, which is too low causing OOM in our application (Netty Solr client and Hikari use more
-                             than 10MB in direct buffer pools after some time), set it to 256M
-                             See https://github.com/orgs/paketo-buildpacks/discussions/241
-                             -->
-                            <BPE_APPEND_JAVA_TOOL_OPTIONS xml:space="preserve"> -XX:MaxDirectMemorySize=256M</BPE_APPEND_JAVA_TOOL_OPTIONS>
+                            <!--
+                             - JVM default is the same as -Xmx. However, the Paketo Java Buildpack memory calculator sets it to
+                               10M by default, which is too low causing OOM in our application (Netty Solr client and Hikari use more
+                               than 10MB in direct buffer pools after some time), set it to 256M
+                               See https://github.com/orgs/paketo-buildpacks/discussions/241
+
+                             - Enable native access for GeoPackage support which uses a native driver -->
+                            <BPE_APPEND_JAVA_TOOL_OPTIONS xml:space="preserve"> -XX:MaxDirectMemorySize=256M --enable-native-access=ALL-UNNAMED</BPE_APPEND_JAVA_TOOL_OPTIONS>
                             <!-- Headroom is used by the memory calculator to reduce the max total memory limit. The default is 0%,
                              but since Tailormap is usually run with unconstrained container memory, set it to 10% to prevent taking
                              too much host memory. Although Tailormap should not exhaust heap memory, reduce it as a preventive safety

src/main/java/org/tailormap/api/configuration/AsyncConfig.java

@@ -28,4 +28,16 @@ public Executor passwordResetTaskExecutor() {
     executor.initialize();
     return executor;
   }
+
+  @Bean(name = "extractTaskExecutor")
+  public Executor extractTaskExecutor() {
+    ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
+    executor.setCorePoolSize(1);
+    executor.setMaxPoolSize(10);
+    executor.setQueueCapacity(100);
+    executor.setThreadNamePrefix("create-extract-");
+    executor.setWaitForTasksToCompleteOnShutdown(false);
+    executor.initialize();
+    return executor;
+  }
 }

src/main/java/org/tailormap/api/configuration/TailormapConfig.java

@@ -5,8 +5,23 @@
  */
 package org.tailormap.api.configuration;
 
+import ch.rasc.sse.eventbus.DataObjectConverter;
+import ch.rasc.sse.eventbus.DefaultDataObjectConverter;
+import ch.rasc.sse.eventbus.DefaultSubscriptionRegistry;
+import ch.rasc.sse.eventbus.DistributedEventBus;
+import ch.rasc.sse.eventbus.JacksonDataObjectConverter;
+import ch.rasc.sse.eventbus.ReplayStore;
+import ch.rasc.sse.eventbus.SseEventBus;
+import ch.rasc.sse.eventbus.SubscriptionRegistry;
 import ch.rasc.sse.eventbus.config.EnableSseEventBus;
+import ch.rasc.sse.eventbus.config.SseEventBusConfigurer;
+import ch.rasc.sse.eventbus.observation.SseEventBusObservationConvention;
+import io.micrometer.observation.ObservationRegistry;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Locale;
+import org.jspecify.annotations.Nullable;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
@@ -15,6 +30,7 @@
 import org.springframework.scheduling.annotation.EnableScheduling;
 import org.springframework.web.servlet.LocaleResolver;
 import org.springframework.web.servlet.i18n.AcceptHeaderLocaleResolver;
+import tools.jackson.databind.ObjectMapper;
 
 @Configuration
 @EnableConfigurationProperties
@@ -42,4 +58,46 @@ public LocaleResolver localeResolver() {
     resolver.setDefaultLocale(Locale.of(defaultLanguage));
     return resolver;
   }
+
+  /**
+   * Define a new viewer SseEventBus bean for viewer-specific SSE traffic.
+   *
+   * @return the viewerSseEventBus instance
+   */
+  @Bean("viewerSseEventBus")
+  public SseEventBus viewerSseEventBus(
+      @Autowired(required = false) @Nullable SseEventBusConfigurer configurer,
+      @Autowired(required = false) @Nullable ObjectMapper objectMapper,
+      @Autowired(required = false) @Nullable List<DataObjectConverter> dataObjectConverters,
+      @Autowired(required = false) @Nullable SubscriptionRegistry subscriptionRegistry,
+      @Autowired(required = false) @Nullable ReplayStore replayStore,
+      @Autowired(required = false) @Nullable ObservationRegistry observationRegistry,
+      @Autowired(required = false) @Nullable SseEventBusObservationConvention observationConvention,
+      @Autowired(required = false) @Nullable DistributedEventBus distributedEventBus) {
+
+    // Apply same defaults as DefaultSseEventBusConfiguration
+    SseEventBusConfigurer config = configurer != null
+        ? configurer
+        : new SseEventBusConfigurer() {
+          /* defaults */
+        };
+
+    SubscriptionRegistry registry =
+        subscriptionRegistry != null ? subscriptionRegistry : new DefaultSubscriptionRegistry();
+
+    ReplayStore store = replayStore != null ? replayStore : config.replayStore();
+
+    List<DataObjectConverter> converters =
+        dataObjectConverters != null ? new ArrayList<>(dataObjectConverters) : new ArrayList<>();
+    if (converters.isEmpty()) {
+      if (objectMapper != null) {
+        converters.add(new JacksonDataObjectConverter(objectMapper));
+      } else {
+        converters.add(new DefaultDataObjectConverter());
+      }
+    }
+
+    return new SseEventBus(
+        config, registry, converters, store, observationRegistry, observationConvention, distributedEventBus);
+  }
 }

src/main/java/org/tailormap/api/configuration/base/WebMvcConfig.java

@@ -16,6 +16,7 @@
 import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
 import org.springframework.web.servlet.resource.EncodedResourceResolver;
 import org.tailormap.api.configuration.CaseInsensitiveEnumConverter;
+import org.tailormap.api.controller.LayerExtractController;
 import org.tailormap.api.persistence.json.GeoServiceProtocol;
 import org.tailormap.api.scheduling.TaskType;
 
@@ -63,5 +64,9 @@ public void addFormatters(@NonNull FormatterRegistry registry) {
         String.class, GeoServiceProtocol.class, new CaseInsensitiveEnumConverter<>(GeoServiceProtocol.class));
 
     registry.addConverter(String.class, TaskType.class, new CaseInsensitiveEnumConverter<>(TaskType.class));
+    registry.addConverter(
+        String.class,
+        LayerExtractController.ExtractOutputFormat.class,
+        new CaseInsensitiveEnumConverter<>(LayerExtractController.ExtractOutputFormat.class));
   }
 }