Load API specifications from a remote URL with authentication (Apicurio Registry / HTTP)

README.md

@@ -834,3 +834,69 @@ from a local JAR containing `contracts/event-api.yml` in its resources:
   </dependencies>
 </plugin>
 ```
+
+## Loading specifications from a remote URL (Apicurio Registry, HTTP)
+
+`filePath` also accepts a remote URL (`http`, `https` or `file` scheme). When
+it does, the spec is downloaded at generation time instead of being read from
+the filesystem or the classpath. This works for both OpenAPI and AsyncAPI
+specs. Remote fetches use bounded connect/read timeouts so an unresponsive
+server cannot hang the build.
+
+This is the mechanism to consume a spec published in an **Apicurio Registry**,
+whose artifacts are served over HTTP. Point `filePath` at the artifact's
+content endpoint, for example:
+
+```xml
+<specFile>
+    <filePath>https://my-apicurio-host/apis/registry/v2/groups/default/artifacts/my-api</filePath>
+    <apiPackage>com.sngular.apigenerator.openapi.api</apiPackage>
+    <modelPackage>com.sngular.apigenerator.openapi.model</modelPackage>
+</specFile>
+```
+
+Gradle:
+
+```groovy
+filePath = 'https://my-apicurio-host/apis/registry/v2/groups/default/artifacts/my-api'
+```
+
+### Authenticated registries
+
+For a protected registry (for example an Apicurio Registry with security
+enabled), credentials are read from **system properties** (preferred) or
+**environment variables** — never from the build files — and sent as request
+headers on `http`/`https` fetches. Each mechanism has a system property and an
+equivalent environment variable:
+
+- **Bearer token**: `scs.multiapi.remote.token` /
+  `SCS_MULTIAPI_REMOTE_TOKEN`.
+- **Basic auth**: `scs.multiapi.remote.user` + `scs.multiapi.remote.password`
+  (env `SCS_MULTIAPI_REMOTE_USER` / `SCS_MULTIAPI_REMOTE_PASSWORD`).
+- **Custom header**: `scs.multiapi.remote.header.name` +
+  `scs.multiapi.remote.header.value` (env `SCS_MULTIAPI_REMOTE_HEADER_NAME` /
+  `SCS_MULTIAPI_REMOTE_HEADER_VALUE`), e.g. an `X-Registry-ApiKey`.
+- **Restrict credentials to a host**: `scs.multiapi.remote.host`
+  (env `SCS_MULTIAPI_REMOTE_HOST`).
+
+A bearer token takes precedence over basic auth; the custom header is
+additive. Example (bearer token from the CI environment):
+
+```bash
+export SCS_MULTIAPI_REMOTE_TOKEN="$APICURIO_TOKEN"
+export SCS_MULTIAPI_REMOTE_HOST="my-apicurio-host"   # optional but recommended
+mvn generate-sources
+```
+
+Notes:
+
+- Set `scs.multiapi.remote.host` to the registry host so the token is sent
+  **only** to that host and never leaked to a different host reached through
+  an external `$ref` or a cross-host redirect.
+- Provide credentials via CI secrets / environment variables; they are never
+  logged.
+- External `$ref`s are resolved relative to the spec's URL when the spec is
+  remote.
+- For an `https` registry using an internally-issued or self-signed
+  certificate, the certificate must be trusted by the JVM running the build
+  (for example imported into its truststore); validation is not disabled.

multiapi-engine/pom.xml

@@ -4,7 +4,7 @@
 
   <groupId>com.sngular</groupId>
   <artifactId>multiapi-engine</artifactId>
-  <version>6.5.3</version>
+  <version>6.6.0</version>
   <packaging>jar</packaging>
 
   <properties>

multiapi-engine/src/main/java/com/sngular/api/generator/plugin/asyncapi/AsyncApiGenerator.java

@@ -26,6 +26,7 @@
 import com.sngular.api.generator.plugin.common.files.ClasspathFileLocation;
 import com.sngular.api.generator.plugin.common.files.DirectoryFileLocation;
 import com.sngular.api.generator.plugin.common.files.FileLocation;
+import com.sngular.api.generator.plugin.common.files.RemoteFileLocation;
 import com.sngular.api.generator.plugin.common.tools.PathUtil;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.tuple.ImmutablePair;
@@ -85,6 +86,16 @@ public final void processFileSpec(final List<SpecFile> specsListFile) {
 
   private static Pair<InputStream, FileLocation> resolveYmlLocation(final String ymlFilePath) throws FileNotFoundException {
     log.debug("Resolving YAML file location:{}", ymlFilePath);
+    if (PathUtil.isRemoteUri(ymlFilePath)) {
+      log.debug("Loading spec from remote URL");
+      try {
+        final URI uri = URI.create(ymlFilePath);
+        final InputStream remoteStream = PathUtil.openUrlStream(uri.toURL());
+        return new ImmutablePair<>(remoteStream, new RemoteFileLocation(uri.resolve(".")));
+      } catch (final IOException e) {
+        throw new FileNotFoundException("Could not open remote YAML file: " + ymlFilePath);
+      }
+    }
     final InputStream classPathInput = AsyncApiGenerator.class.getClassLoader().getResourceAsStream(ymlFilePath);
     final InputStream ymlFile;
     final FileLocation ymlParentPath;

multiapi-engine/src/main/java/com/sngular/api/generator/plugin/asyncapi/handler/BaseAsyncApiHandler.java

@@ -21,6 +21,8 @@
 import com.sngular.api.generator.plugin.asyncapi.parameter.SpecFile;
 import com.sngular.api.generator.plugin.asyncapi.template.TemplateFactory;
 import com.sngular.api.generator.plugin.common.files.ClasspathFileLocation;
+import com.sngular.api.generator.plugin.common.files.RemoteFileLocation;
+import com.sngular.api.generator.plugin.common.tools.PathUtil;
 import com.sngular.api.generator.plugin.common.files.DirectoryFileLocation;
 import com.sngular.api.generator.plugin.common.files.FileLocation;
 import com.sngular.api.generator.plugin.common.model.CommonSpecFile;
@@ -108,6 +110,9 @@ protected BaseAsyncApiHandler(
   }
 
   protected static FileLocation resolveYmlLocation(final String ymlFilePath) throws IOException, URISyntaxException {
+    if (PathUtil.isRemoteUri(ymlFilePath)) {
+      return new RemoteFileLocation(URI.create(ymlFilePath).resolve("."));
+    }
     final var classPathInput = BaseAsyncApiHandler.class.getClassLoader().getResource(ymlFilePath);
     if (Objects.nonNull(classPathInput)) {
       return new ClasspathFileLocation(getParentUri(classPathInput.toURI()));

multiapi-engine/src/main/java/com/sngular/api/generator/plugin/common/files/RemoteFileLocation.java

@@ -0,0 +1,39 @@
+/*
+ *  This Source Code Form is subject to the terms of the Mozilla Public
+ *  * License, v. 2.0. If a copy of the MPL was not distributed with this
+ *  * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+package com.sngular.api.generator.plugin.common.files;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.URL;
+
+import com.sngular.api.generator.plugin.common.tools.PathUtil;
+
+/**
+ * {@link FileLocation} for specifications loaded from a remote URL (e.g. an Apicurio Registry
+ * artifact served over HTTP). Relative sibling files (external {@code $ref}s) are resolved
+ * against the spec's base URI and fetched over the same protocol.
+ */
+public class RemoteFileLocation implements FileLocation {
+
+  private final URI baseUri;
+
+  public RemoteFileLocation(final URI baseUri) {
+    this.baseUri = baseUri;
+  }
+
+  @Override
+  public InputStream getFileAtLocation(final String filename) throws IOException {
+    final URL target = baseUri.resolve(filename).toURL();
+    return PathUtil.openUrlStream(target);
+  }
+
+  @Override
+  public URI path() {
+    return baseUri;
+  }
+}

multiapi-engine/src/main/java/com/sngular/api/generator/plugin/common/tools/ApiTool.java

@@ -5,6 +5,7 @@
 import java.io.InputStream;
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import java.net.URI;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -453,8 +454,10 @@ public static boolean hasComponents(final JsonNode node) {
 
   public static JsonNode nodeFromFile(final FileLocation ymlParent, final String filePath, final FactoryTypeEnum factoryTypeEnum) throws IOException {
     final InputStream file;
-    // Check if path is absolute first
-    if (PathUtil.isAbsolutePath(filePath)) {
+    // Remote references (http/https/ftp/file URLs) are fetched directly from their URL.
+    if (PathUtil.isRemoteUri(filePath)) {
+      file = PathUtil.openUrlStream(URI.create(filePath).toURL());
+    } else if (PathUtil.isAbsolutePath(filePath)) {
       // For absolute paths, open directly
       file = new FileInputStream(filePath);
     } else if (filePath.startsWith(PACKAGE_SEPARATOR_STR) || filePath.matches("^\\w.*$")) {

multiapi-engine/src/main/java/com/sngular/api/generator/plugin/common/tools/PathUtil.java

@@ -6,8 +6,16 @@
 
 package com.sngular.api.generator.plugin.common.tools;
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.net.URLConnection;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.InvalidPathException;
 import java.nio.file.Paths;
+import java.util.Base64;
+import java.util.LinkedHashMap;
+import java.util.Map;
 
 import org.apache.commons.lang3.StringUtils;
 
@@ -16,6 +24,12 @@
  */
 public final class PathUtil {
 
+  /** Connection timeout (ms) applied when fetching a remote specification. */
+  public static final int REMOTE_CONNECT_TIMEOUT_MS = 15_000;
+
+  /** Read timeout (ms) applied when fetching a remote specification. */
+  public static final int REMOTE_READ_TIMEOUT_MS = 30_000;
+
   private PathUtil() {
     // Utility class
   }
@@ -37,4 +51,92 @@ public static boolean isAbsolutePath(final String filePath) {
       return false;
     }
   }
+
+  /**
+   * Checks whether a spec path points to a remote/URL location that must be fetched via a
+   * {@link java.net.URL} stream rather than the filesystem or classpath. Enables loading
+   * specifications from HTTP(S) endpoints such as an Apicurio Registry artifact.
+   *
+   * @param filePath the spec path to check
+   * @return true if the path uses an http, https or file scheme
+   */
+  public static boolean isRemoteUri(final String filePath) {
+    return StringUtils.isNotEmpty(filePath)
+           && filePath.matches("^(?i)(https?|file)://.*");
+  }
+
+  /**
+   * Opens a stream to a URL with bounded connect/read timeouts, so an unresponsive remote
+   * spec server (e.g. an Apicurio Registry) cannot hang the build indefinitely. Timeouts are
+   * harmless for non-network schemes such as {@code file:} and {@code jar:}. For {@code http}/
+   * {@code https} URLs any configured authentication headers (see {@link #remoteAuthHeaders})
+   * are applied.
+   *
+   * @param url the URL to open
+   * @return the input stream
+   * @throws IOException if the connection cannot be established or times out
+   */
+  public static InputStream openUrlStream(final URL url) throws IOException {
+    final URLConnection connection = url.openConnection();
+    connection.setConnectTimeout(REMOTE_CONNECT_TIMEOUT_MS);
+    connection.setReadTimeout(REMOTE_READ_TIMEOUT_MS);
+    final String protocol = url.getProtocol();
+    if ("http".equalsIgnoreCase(protocol) || "https".equalsIgnoreCase(protocol)) {
+      remoteAuthHeaders(url.getHost()).forEach(connection::setRequestProperty);
+    }
+    return connection.getInputStream();
+  }
+
+  /**
+   * Resolves the authentication headers to attach when fetching a remote spec, from system
+   * properties (preferred) or environment variables. Credentials are never read from the build
+   * files. Supported schemes (first match wins for {@code Authorization}):
+   * <ul>
+   *   <li>Bearer token: {@code scs.multiapi.remote.token} / {@code SCS_MULTIAPI_REMOTE_TOKEN}</li>
+   *   <li>Basic auth: {@code scs.multiapi.remote.user}(+{@code .password}) /
+   *       {@code SCS_MULTIAPI_REMOTE_USER}(+{@code _PASSWORD})</li>
+   * </ul>
+   * plus an optional arbitrary header ({@code scs.multiapi.remote.header.name}/{@code .value} or
+   * {@code SCS_MULTIAPI_REMOTE_HEADER_NAME}/{@code _VALUE}), e.g. an {@code X-Registry-ApiKey}.
+   *
+   * <p>When {@code scs.multiapi.remote.host} / {@code SCS_MULTIAPI_REMOTE_HOST} is set, credentials
+   * are only sent to that host, so a token is never leaked to a different host reached through an
+   * external {@code $ref} or a cross-host redirect.
+   *
+   * @param host the host of the URL being fetched
+   * @return the headers to apply (empty if none configured, or scoped out by host)
+   */
+  public static Map<String, String> remoteAuthHeaders(final String host) {
+    final Map<String, String> headers = new LinkedHashMap<>();
+
+    final String scopedHost = config("scs.multiapi.remote.host", "SCS_MULTIAPI_REMOTE_HOST");
+    if (StringUtils.isNotBlank(scopedHost) && !scopedHost.equalsIgnoreCase(host)) {
+      return headers;
+    }
+
+    final String token = config("scs.multiapi.remote.token", "SCS_MULTIAPI_REMOTE_TOKEN");
+    final String user = config("scs.multiapi.remote.user", "SCS_MULTIAPI_REMOTE_USER");
+    if (StringUtils.isNotBlank(token)) {
+      headers.put("Authorization", "Bearer " + token);
+    } else if (StringUtils.isNotBlank(user)) {
+      final String password = StringUtils.defaultString(
+          config("scs.multiapi.remote.password", "SCS_MULTIAPI_REMOTE_PASSWORD"));
+      final String credentials = user + ":" + password;
+      final String basic = Base64.getEncoder().encodeToString(credentials.getBytes(StandardCharsets.UTF_8));
+      headers.put("Authorization", "Basic " + basic);
+    }
+
+    final String headerName = config("scs.multiapi.remote.header.name", "SCS_MULTIAPI_REMOTE_HEADER_NAME");
+    final String headerValue = config("scs.multiapi.remote.header.value", "SCS_MULTIAPI_REMOTE_HEADER_VALUE");
+    if (StringUtils.isNotBlank(headerName) && StringUtils.isNotBlank(headerValue)) {
+      headers.put(headerName, headerValue);
+    }
+
+    return headers;
+  }
+
+  private static String config(final String systemProperty, final String environmentVariable) {
+    final String fromProperty = System.getProperty(systemProperty);
+    return StringUtils.isNotBlank(fromProperty) ? fromProperty : System.getenv(environmentVariable);
+  }
 }

multiapi-engine/src/main/java/com/sngular/api/generator/plugin/common/tools/SchemaUtil.java

@@ -137,6 +137,12 @@ private static String readFile(final URI rootFilePath, final String filePath) th
       throw new IllegalArgumentException("File Path cannot be empty");
     }
 
+    // Remote specifications (http/https/ftp/file URLs, e.g. an Apicurio Registry artifact) are
+    // fetched directly from their URL, bypassing classpath and filesystem resolution.
+    if (PathUtil.isRemoteUri(filePath)) {
+      return readFromUrl(URI.create(filePath).toURL());
+    }
+
     // Normalize the incoming filePath: remove leading './' and replace backslashes with forward slashes
     final String cleaned = cleanUpPath(filePath).replace('\\', '/');
 
@@ -161,8 +167,12 @@ private static String readFile(final URI rootFilePath, final String filePath) th
         }
       }
     }
+    return readFromUrl(fileURL);
+  }
+
+  private static String readFromUrl(final URL fileURL) {
     final var sb = new StringBuilder();
-    try (BufferedReader reader = new BufferedReader(new InputStreamReader(fileURL.openStream()))) {
+    try (BufferedReader reader = new BufferedReader(new InputStreamReader(PathUtil.openUrlStream(fileURL)))) {
       String inputLine;
       while ((inputLine = reader.readLine()) != null) {
         sb.append(inputLine).append(System.lineSeparator());

multiapi-engine/src/main/java/com/sngular/api/generator/plugin/openapi/OpenApiGenerator.java

@@ -8,6 +8,7 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.net.URI;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.HashSet;
@@ -23,6 +24,7 @@
 import com.sngular.api.generator.plugin.common.tools.ApiTool;
 import com.sngular.api.generator.plugin.common.tools.MapperContentUtil;
 import com.sngular.api.generator.plugin.common.tools.MapperUtil;
+import com.sngular.api.generator.plugin.common.tools.PathUtil;
 import com.sngular.api.generator.plugin.exception.GeneratorTemplateException;
 import com.sngular.api.generator.plugin.openapi.exception.DuplicateModelClassException;
 import com.sngular.api.generator.plugin.openapi.model.AuthObject;
@@ -99,7 +101,10 @@ private void processFile(final SpecFile specFile) {
 
     final JsonNode openAPI = OpenApiUtil.getPojoFromSpecFile(baseDir, specFile);
     OpenApiUtil.mergeWebhooksIntoPaths(openAPI);
-    OpenApiUtil.solvePathRefs(openAPI, baseDir.resolve(specFile.getFilePath()).getParent().toUri());
+    final URI specBaseUri = PathUtil.isRemoteUri(specFile.getFilePath())
+                                ? URI.create(specFile.getFilePath()).resolve(".")
+                                : baseDir.resolve(specFile.getFilePath()).getParent().toUri();
+    OpenApiUtil.solvePathRefs(openAPI, specBaseUri);
     final String clientPackage = specFile.getClientPackage();
 
     if (specFile.isCallMode()) {