From 4337a03d9ef25db40643fa8255a7fc214b29b18c Mon Sep 17 00:00:00 2001 From: Sean Arms <67096+lesserwhirls@users.noreply.github.com> Date: Tue, 5 Aug 2025 13:37:11 -0600 Subject: [PATCH 1/2] Have HTTPRandomAccessFile use https by default When using "nodods:" or "httpserver:" for a dataset url, try connecting with https first, and if that fails, try to fall back to http. --- .../unidata/io/http/HTTPRandomAccessFile.java | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/cdm/core/src/main/java/ucar/unidata/io/http/HTTPRandomAccessFile.java b/cdm/core/src/main/java/ucar/unidata/io/http/HTTPRandomAccessFile.java index 739061aa48..b1b4a16be7 100644 --- a/cdm/core/src/main/java/ucar/unidata/io/http/HTTPRandomAccessFile.java +++ b/cdm/core/src/main/java/ucar/unidata/io/http/HTTPRandomAccessFile.java @@ -1,7 +1,8 @@ /* - * Copyright (c) 1998-2019 University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2025 University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ + package ucar.unidata.io.http; import java.io.FileNotFoundException; @@ -40,6 +41,9 @@ public final class HTTPRandomAccessFile extends RemoteRandomAccessFile { private static final long httpMaxCacheSize = Long .parseLong(System.getProperty("ucar.unidata.io.http.maxReadCacheSize", String.valueOf(defaultMaxReadCacheSize))); + private static final String HTTPS = "https"; + private static final String HTTP = "http"; + private static final boolean debug = false, debugDetails = false; /////////////////////////////////////////////////////////////////////////////////// @@ -234,7 +238,7 @@ public long getLastModified() { */ public static class Provider implements RandomAccessFileProvider { - private static final List possibleSchemes = Arrays.asList("http", "https", "nodods", "httpserver"); + private static final List possibleSchemes = Arrays.asList(HTTP, HTTPS, "nodods", "httpserver"); @Override public boolean isOwnerOf(String location) { @@ -250,10 +254,26 @@ public RandomAccessFile open(String location) throws IOException { @Override public RandomAccessFile open(String location, int bufferSize) throws IOException { String scheme = location.split(":")[0]; - if (!scheme.equalsIgnoreCase("https") && !scheme.equalsIgnoreCase("http")) { - location = location.replace(scheme, "http"); + boolean fallback = false; + if (!scheme.equalsIgnoreCase(HTTPS) && !scheme.equalsIgnoreCase(HTTP)) { + location = location.replaceFirst(scheme, HTTPS); + fallback = true; + } + HTTPRandomAccessFile raf; + try { + // try with https first + raf = new HTTPRandomAccessFile(location, bufferSize, httpMaxCacheSize); + } catch (IOException e) { + if (fallback) { + // if we had to guess the scheme, fallback to http, and if it does not work, let the IOError be thrown + raf = new HTTPRandomAccessFile(location.replaceFirst(HTTPS, HTTP), bufferSize, httpMaxCacheSize); + } else { + // didn't try to fallback to http, just throw the original error + throw e; + } } - return new HTTPRandomAccessFile(location, bufferSize, httpMaxCacheSize); + + return raf; } } } From 780aa1c58574c3b80e1bc64895fcfa6263b257a2 Mon Sep 17 00:00:00 2001 From: Sean Arms <67096+lesserwhirls@users.noreply.github.com> Date: Tue, 5 Aug 2025 13:43:38 -0600 Subject: [PATCH 2/2] Error handling in disambiguateHttp Add some consistency to the error handling in the checkIfXXX methods used in disambiguateHttp. Return IOErrors if for 401 responses. Log 403 responses and move on to the next check. Fixes Unidata/netcdf-java#1435 --- .../java/ucar/nc2/dataset/DatasetUrl.java | 49 +++++++++++++------ 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/DatasetUrl.java b/cdm/core/src/main/java/ucar/nc2/dataset/DatasetUrl.java index 4048f5c104..c40048adc5 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/DatasetUrl.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/DatasetUrl.java @@ -1,27 +1,34 @@ /* - * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2025 John Caron and University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ + package ucar.nc2.dataset; import static java.net.HttpURLConnection.HTTP_FORBIDDEN; import static java.net.HttpURLConnection.HTTP_NOT_ACCEPTABLE; import static java.net.HttpURLConnection.HTTP_OK; import static java.net.HttpURLConnection.HTTP_UNAUTHORIZED; + import com.google.common.annotations.VisibleForTesting; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; import javax.annotation.Nullable; -import com.google.common.collect.Multimap; import thredds.client.catalog.ServiceType; -import thredds.inventory.MFile; -import thredds.inventory.MFiles; import ucar.httpservices.HTTPFactory; import ucar.httpservices.HTTPMethod; import ucar.nc2.util.EscapeStrings; import ucar.unidata.util.StringUtil2; import ucar.unidata.util.Urlencoded; -import java.io.*; -import java.util.*; /** * Detection of the protocol from a location string. @@ -31,6 +38,7 @@ * @since 10/20/2015. */ public class DatasetUrl { + private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DatasetUrl.class); private static final String alpha = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; private static final String slashalpha = "\\/" + alpha; @@ -398,15 +406,23 @@ private static ServiceType disambiguateHttp(String location) throws IOException return null; } + private static void handleCheckIfResponse(HTTPMethod method, int statusCode, ServiceType serviceChecked) + throws IOException { + if (statusCode == HTTP_UNAUTHORIZED) { + throw new IOException("Unauthorized to open dataset " + method.getURI()); + } else if (statusCode == HTTP_FORBIDDEN) { + logger.warn(String.format( + "Forbidden Request to %s - assuming remote server is not a %s server, but could also indicate incorrect credentials.", + method.getURI(), serviceChecked)); + } + } + // cdmremote private static ServiceType checkIfCdmr(String location) throws IOException { try (HTTPMethod method = HTTPFactory.Head(location + "?req=header")) { int statusCode = method.execute(); if (statusCode >= 300) { - if (statusCode == HTTP_UNAUTHORIZED || statusCode == HTTP_FORBIDDEN) - throw new IOException("Unauthorized to open dataset " + location); - else - throw new IOException(location + " is not a valid URL, return status=" + statusCode); + handleCheckIfResponse(method, statusCode, ServiceType.CdmRemote); } Optional value = method.getResponseHeaderValue("Content-Description"); @@ -442,8 +458,7 @@ private static ServiceType checkIfDods(String location) throws IOException { throw new IOException("OPeNDAP Server Error= " + method.getResponseAsString()); } } - if (status == HTTP_UNAUTHORIZED || status == HTTP_FORBIDDEN) - throw new IOException("Unauthorized to open dataset " + location); + handleCheckIfResponse(method, status, ServiceType.OPENDAP); // not dods return null; @@ -471,6 +486,7 @@ else if (location.endsWith(".html")) return ServiceType.DAP4; } } + handleCheckIfResponse(method, status, ServiceType.DAP4); // not dap4 return null; } @@ -497,14 +513,15 @@ private static boolean checkIfRemoteNcml(String location) throws IOException { method.setRequestHeader("accept-encoding", "identity"); int statusCode = method.execute(); if (statusCode >= 300) { - if (statusCode == HTTP_UNAUTHORIZED) { - throw new IOException("Unauthorized to open dataset " + location); - } else if (statusCode == HTTP_NOT_ACCEPTABLE) { + handleCheckIfResponse(method, statusCode, ServiceType.NCML); + // additional checks + if (statusCode == HTTP_NOT_ACCEPTABLE) { String msg = location + " - this server does not support returning content without any encoding."; msg = msg + " Please download the file locally. Return status=" + statusCode; throw new IOException(msg); } else { - throw new IOException(location + " is not a valid URL, return status=" + statusCode); + throw new IOException(String.format("Error opening %s: %s, (HTTP Status Code %d)", location, + method.getResponseAsString(), statusCode)); } }