diff --git a/pom.xml b/pom.xml index 98d2cd59d2..3029f6ee6b 100644 --- a/pom.xml +++ b/pom.xml @@ -101,7 +101,7 @@ SPDX-License-Identifier: MIT ${java.version} ${java.version} 2026-06-12T09:30:45Z - 35-RC + 35.0 1.20.0 5.4.0 2.1.8 diff --git a/src/main/java/org/tailormap/api/TailormapApiApplication.java b/src/main/java/org/tailormap/api/TailormapApiApplication.java index 84d01f53b4..8196fe3dd1 100644 --- a/src/main/java/org/tailormap/api/TailormapApiApplication.java +++ b/src/main/java/org/tailormap/api/TailormapApiApplication.java @@ -6,12 +6,14 @@ package org.tailormap.api; +import org.geotools.util.factory.GeoTools; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; @SpringBootApplication(scanBasePackages = {"org.tailormap.api.configuration.base"}) public class TailormapApiApplication { - public static void main(String[] args) { + static void main(String[] args) { + GeoTools.init(); SpringApplication.run(TailormapApiApplication.class, args); } } diff --git a/src/main/java/org/tailormap/api/geotools/featuresources/WFSFeatureSourceHelper.java b/src/main/java/org/tailormap/api/geotools/featuresources/WFSFeatureSourceHelper.java index f19f21d375..b170abda23 100644 --- a/src/main/java/org/tailormap/api/geotools/featuresources/WFSFeatureSourceHelper.java +++ b/src/main/java/org/tailormap/api/geotools/featuresources/WFSFeatureSourceHelper.java @@ -15,6 +15,7 @@ import org.geotools.data.wfs.WFSDataStoreFactory; import org.geotools.data.wfs.internal.FeatureTypeInfo; import org.geotools.util.PreventLocalEntityResolver; +import org.geotools.util.factory.Hints; import org.springframework.util.LinkedCaseInsensitiveMap; import org.springframework.web.util.UriComponentsBuilder; import org.tailormap.api.geotools.wfs.SimpleWFSHelper; @@ -30,6 +31,8 @@ public DataStore createDataStore(TMFeatureSource tmfs, Integer timeout) throws I Map params = new HashMap<>(); params.put(WFSDataStoreFactory.ENTITY_RESOLVER.key, PreventLocalEntityResolver.INSTANCE); + Hints.putSystemDefault(Hints.ENTITY_RESOLVER, params.get(WFSDataStoreFactory.ENTITY_RESOLVER.key)); + if (timeout != null) { params.put(WFSDataStoreFactory.TIMEOUT.key, timeout); } diff --git a/src/main/java/org/tailormap/api/geotools/wfs/SimpleWFSHelper.java b/src/main/java/org/tailormap/api/geotools/wfs/SimpleWFSHelper.java index 2362f42d0c..58df514a4e 100644 --- a/src/main/java/org/tailormap/api/geotools/wfs/SimpleWFSHelper.java +++ b/src/main/java/org/tailormap/api/geotools/wfs/SimpleWFSHelper.java @@ -8,6 +8,7 @@ import static org.tailormap.api.util.HttpProxyUtil.setHttpBasicAuthenticationHeader; import java.io.InputStream; +import java.io.Reader; import java.lang.invoke.MethodHandles; import java.net.URI; import java.net.URLEncoder; @@ -24,16 +25,10 @@ import javax.xml.XMLConstants; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; import javax.xml.xpath.XPath; import javax.xml.xpath.XPathConstants; import javax.xml.xpath.XPathFactory; -import org.geotools.http.HTTPClient; -import org.geotools.http.SimpleHttpClient; -import org.geotools.ows.wms.LayerDescription; -import org.geotools.ows.wms.WMS1_1_1; -import org.geotools.ows.wms.WebMapServer; -import org.geotools.ows.wms.request.DescribeLayerRequest; -import org.geotools.ows.wms.response.DescribeLayerResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.util.LinkedMultiValueMap; @@ -44,6 +39,8 @@ import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.NodeList; +import org.xml.sax.EntityResolver; +import org.xml.sax.InputSource; public class SimpleWFSHelper { private static final Logger logger = @@ -82,7 +79,7 @@ public static URI getOGCRequestURL( // We need to encode the parameters manually because UriComponentsBuilder annoyingly does not // encode '+' as used in mime types for output formats, see // https://stackoverflow.com/questions/18138011 - parameters.replaceAll((key, values) -> values.stream() + parameters.replaceAll((unusedKey, values) -> values.stream() .map(s -> URLEncoder.encode(s, StandardCharsets.UTF_8)) .collect(Collectors.toList())); params.addAll(parameters); @@ -119,17 +116,7 @@ public static List getOutputFormats( HttpResponse response = httpClient.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofInputStream()); - // Parse capabilities in DOM - - DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance(); - documentBuilderFactory.setNamespaceAware(true); - documentBuilderFactory.setExpandEntityReferences(false); - documentBuilderFactory.setValidating(false); - documentBuilderFactory.setXIncludeAware(false); - documentBuilderFactory.setCoalescing(true); - documentBuilderFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); - DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder(); - Document doc = documentBuilder.parse(response.body()); + Document doc = parseSecureXml(response.body()); // WFS 1.1.0 and WFS 2.0.0 use different namespaces, but the same local element names boolean wfs2 = "2.0.0".equals(doc.getDocumentElement().getAttribute("version")); @@ -188,31 +175,15 @@ public static SimpleWFSLayerDescription describeWMSLayer( public static Map describeWMSLayers( String url, String username, String password, List layers) { - try { - HTTPClient client = new SimpleHttpClient(); - client.setUser(username); - client.setPassword(password); - client.setConnectTimeout(TIMEOUT); - client.setReadTimeout(TIMEOUT); - WebMapServer wms = new WebMapServer(new URI(url).toURL(), client); - // Directly create WMS 1.1.1 request. Creating it from WebMapServer errors with GeoServer - // about unsupported request in capabilities unless we override WebMapServer to set up - // specifications. - DescribeLayerRequest describeLayerRequest = new WMS1_1_1().createDescribeLayerRequest(new URI(url).toURL()); - // XXX Otherwise GeoTools will send VERSION=1.1.0... - describeLayerRequest.setProperty("VERSION", "1.1.1"); - describeLayerRequest.setLayers(String.join(",", layers)); - // GeoTools will throw a ClassCastException when a WMS ServiceException is returned - DescribeLayerResponse describeLayerResponse = wms.issueRequest(describeLayerRequest); - - Map descriptions = new HashMap<>(); - for (LayerDescription ld : describeLayerResponse.getLayerDescs()) { - String wfsUrl = getWfsUrl(ld, wms); - - if (wfsUrl != null && ld.getQueries() != null && ld.getQueries().length != 0) { - descriptions.put(ld.getName(), new SimpleWFSLayerDescription(wfsUrl, List.of(ld.getQueries()))); - } - } + try (HttpClient httpClient = getDefaultHttpClient(); ) { + URI describeLayerUri = getDescribeLayerRequestUrl(url, layers); + HttpRequest.Builder requestBuilder = HttpRequest.newBuilder().uri(describeLayerUri); + setHttpBasicAuthenticationHeader(requestBuilder, username, password); + + HttpResponse response = + httpClient.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofInputStream()); + Document document = parseXmlAllowingDocType(response.body()); + Map descriptions = parseDescribeLayerResponse(document, url); return Collections.unmodifiableMap(descriptions); } catch (Exception e) { String msg = @@ -226,17 +197,81 @@ public static Map describeWMSLayers( return Map.of(); } - private static String getWfsUrl(LayerDescription ld, WebMapServer wms) { - String wfsUrl = (ld.getWfs() != null) ? ld.getWfs().toString() : null; - if (wfsUrl == null && "WFS".equalsIgnoreCase(ld.getOwsType())) { - wfsUrl = ld.getOwsURL().toString(); + private static URI getDescribeLayerRequestUrl(String url, List layers) { + MultiValueMap parameters = new LinkedMultiValueMap<>(); + parameters.add("LAYERS", String.join(",", layers)); + return getOGCRequestURL(url, "WMS", "1.1.1", "DescribeLayer", parameters); + } + + private static Document parseSecureXml(InputStream inputStream) throws Exception { + return getDocumentBuilder().parse(inputStream); + } + + private static DocumentBuilder getDocumentBuilder() throws ParserConfigurationException { + DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance(); + documentBuilderFactory.setNamespaceAware(true); + documentBuilderFactory.setExpandEntityReferences(false); + documentBuilderFactory.setValidating(false); + documentBuilderFactory.setXIncludeAware(false); + documentBuilderFactory.setCoalescing(true); + documentBuilderFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + return documentBuilderFactory.newDocumentBuilder(); + } + + private static Document parseXmlAllowingDocType(InputStream inputStream) throws Exception { + DocumentBuilder documentBuilder = getDocumentBuilder(); + EntityResolver entityResolver = (publicId, systemId) -> new InputSource(Reader.of("")); + documentBuilder.setEntityResolver(entityResolver); + + return documentBuilder.parse(inputStream); + } + + private static Map parseDescribeLayerResponse( + Document document, String fallbackUrl) { + Element root = document.getDocumentElement(); + if (root == null) { + return Map.of(); } - // OGC 02-070 Annex B says the wfs/owsURL attributed are not required but implied. Some - // Deegree instance encountered has all attributes empty, and apparently the meaning is that - // the WFS URL is the same as the WMS URL (not explicitly defined in the spec). - if (wfsUrl == null) { - wfsUrl = wms.getInfo().getSource().toString(); + + NodeList layerDescriptions = root.getElementsByTagNameNS("*", "LayerDescription"); + Map descriptions = new HashMap<>(); + + for (int i = 0; i < layerDescriptions.getLength(); i++) { + Element layerDescription = (Element) layerDescriptions.item(i); + + String layerName = layerDescription.getAttribute("name"); + String wfsUrl = getDescribeLayerWfsUrl(layerDescription, fallbackUrl); + + List typeNames = new ArrayList<>(); + NodeList queryNodes = layerDescription.getElementsByTagNameNS("*", "Query"); + for (int j = 0; j < queryNodes.getLength(); j++) { + Element query = (Element) queryNodes.item(j); + String typeName = query.getAttribute("typeName"); + if (!typeName.isBlank()) { + typeNames.add(typeName); + } + } + + if (!layerName.isBlank() && wfsUrl != null && !typeNames.isEmpty()) { + descriptions.put(layerName, new SimpleWFSLayerDescription(wfsUrl, List.copyOf(typeNames))); + } + } + + return descriptions; + } + + private static String getDescribeLayerWfsUrl(Element layerDescription, String fallbackUrl) { + String wfsUrl = layerDescription.getAttribute("wfs"); + if (!wfsUrl.isBlank()) { + return wfsUrl; } - return wfsUrl; + + String owsType = layerDescription.getAttribute("owsType"); + String owsUrl = layerDescription.getAttribute("owsURL"); + if ("WFS".equalsIgnoreCase(owsType) && !owsUrl.isBlank()) { + return owsUrl; + } + + return fallbackUrl; } } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 3b87452213..c514bc94c7 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -28,7 +28,7 @@ tailormap-api.features.wfs_count_exact=false tailormap-api.feature.info.maxitems=30 # see org.tailormap.api.controller.LayerExtractController.ExtractOutputFormat for valid values -tailormap-api.extract.allowed-outputformats=csv,xlsx,shape,geopackage +tailormap-api.extract.allowed-outputformats=csv,xlsx,shape,geopackage,geojson # any files older than this (in minutes) in the extract output directory will be deleted by a scheduled job, to prevent filling up the disk # tailormap-api.extract.cleanup-minutes=120 # the (base) directory where the extract output files are stored, should be writable by the application diff --git a/src/test/java/org/tailormap/api/controller/LayerExtractControllerIntegrationTest.java b/src/test/java/org/tailormap/api/controller/LayerExtractControllerIntegrationTest.java index b6b4c95112..5dce90fe6d 100644 --- a/src/test/java/org/tailormap/api/controller/LayerExtractControllerIntegrationTest.java +++ b/src/test/java/org/tailormap/api/controller/LayerExtractControllerIntegrationTest.java @@ -54,7 +54,6 @@ import org.junit.jupiter.api.TestMethodOrder; import org.junit.jupiter.api.parallel.Execution; import org.junit.jupiter.api.parallel.ExecutionMode; -import org.junitpioneer.jupiter.DisabledUntil; import org.junitpioneer.jupiter.Stopwatch; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -407,10 +406,6 @@ void should_export_large_filter_to_excel() throws Exception { } } - @DisabledUntil( - date = "2026-07-01", - reason = - "This test relies on GeoTools 35.0 (or 34.4), see https://osgeo-org.atlassian.net/browse/GEOT-7894") @Test void should_export_large_filter_to_geojson() throws Exception { final String extractUrl = apiBasePath + layerBegroeidTerreindeelPostgis + extractPath + sseClientId; diff --git a/src/test/resources/application.properties b/src/test/resources/application.properties index 31e331d7aa..9e64ec8111 100644 --- a/src/test/resources/application.properties +++ b/src/test/resources/application.properties @@ -3,7 +3,7 @@ tailormap-api.admin.base-path=/api/admin management.endpoints.web.base-path=/api/actuator tailormap-api.new-admin-username=tm-admin # see org.tailormap.api.controller.LayerExtractController.ExtractOutputFormat for valid values -tailormap-api.extract.allowed-outputformats=csv,xlsx,shape,geopackage +tailormap-api.extract.allowed-outputformats=csv,xlsx,shape,geopackage,geojson # the number of features after which a progress report is sent back to the viewer, to update the progress bar tailormap-api.extract.progress-report-interval=10 # any files older than this (in minutes) in the extract output directory will be deleted by a scheduled job, to prevent filling up the disk