Introduce extensible configuration fetcher SPI for JCasC

plugin/src/main/java/io/jenkins/plugins/casc/ConfigurationAsCode.java

@@ -1,7 +1,7 @@
 package io.jenkins.plugins.casc;
 
 import static io.jenkins.plugins.casc.SchemaGeneration.writeJSONSchema;
-import static java.lang.String.format;
+import static io.jenkins.plugins.casc.fetcher.FetchCredentials.resolveAll;
 import static java.util.stream.Collectors.toList;
 import static org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK;
 import static org.yaml.snakeyaml.DumperOptions.ScalarStyle.DOUBLE_QUOTED;
@@ -22,6 +22,9 @@
 import hudson.security.ACLContext;
 import hudson.security.Permission;
 import hudson.util.FormValidation;
+import io.jenkins.plugins.casc.fetcher.CasCConfigFetcher;
+import io.jenkins.plugins.casc.fetcher.FetchContext;
+import io.jenkins.plugins.casc.fetcher.FetchCredentials;
 import io.jenkins.plugins.casc.impl.DefaultConfiguratorRegistry;
 import io.jenkins.plugins.casc.model.CNode;
 import io.jenkins.plugins.casc.model.Mapping;
@@ -43,8 +46,6 @@
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.io.Writer;
-import java.net.URI;
-import java.net.URISyntaxException;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.FileSystems;
@@ -232,20 +233,23 @@ public void doReplace(StaplerRequest2 request, StaplerResponse2 response) throws
                 }
             }
             if (!candidateSources.isEmpty()) {
-                List<YamlSource> candidates = getConfigFromSources(candidateSources);
-                if (canApplyFrom(candidates)) {
-                    sources = candidateSources;
-                    configureWith(getConfigFromSources(getSources()));
-                    CasCGlobalConfig config = GlobalConfiguration.all().get(CasCGlobalConfig.class);
-                    if (config != null) {
-                        config.setConfigurationPath(normalizedSource);
-                        config.save();
+                try (FetchContext context = getConfigFromSources(candidateSources)) {
+                    if (canApplyFrom(context.getSources())) {
+                        sources = candidateSources;
+                        try (FetchContext applyContext = getConfigFromSources(getSources())) {
+                            configureWith(applyContext.getSources());
+                        }
+                        CasCGlobalConfig config = GlobalConfiguration.all().get(CasCGlobalConfig.class);
+                        if (config != null) {
+                            config.setConfigurationPath(normalizedSource);
+                            config.save();
+                        }
+                        LOGGER.log(Level.FINE, "Replace configuration with: " + normalizedSource);
+                    } else {
+                        LOGGER.log(Level.WARNING, "Provided sources could not be applied");
+                        throw new ConfiguratorException(
+                                "Provided sources could not be applied. Please check the syntax and validity of the provided configuration.");
                     }
-                    LOGGER.log(Level.FINE, "Replace configuration with: " + normalizedSource);
-                } else {
-                    LOGGER.log(Level.WARNING, "Provided sources could not be applied");
-                    throw new ConfiguratorException(
-                            "Provided sources could not be applied. Please check the syntax and validity of the provided configuration.");
                 }
             } else {
                 LOGGER.log(Level.FINE, "No such source exists, applying default");
@@ -294,9 +298,8 @@ public FormValidation doCheckNewSource(@QueryParameter String newSource) {
             }
             candidateSources.add(candidateSource);
         }
-        try {
-            List<YamlSource> candidates = getConfigFromSources(candidateSources);
-            final Map<Source, String> issues = checkWith(candidates);
+        try (FetchContext context = getConfigFromSources(candidateSources)) {
+            final Map<Source, String> issues = checkWith(context.getSources());
             final JSONArray errors = collectProblems(issues, "error");
             if (!errors.isEmpty()) {
                 return FormValidation.error(errors.toString());
@@ -305,10 +308,16 @@ public FormValidation doCheckNewSource(@QueryParameter String newSource) {
             if (!warnings.isEmpty()) {
                 return FormValidation.warning(warnings.toString());
             }
-            return FormValidation.okWithMarkup("The configuration can be applied");
-        } catch (ConfiguratorException | IllegalArgumentException e) {
-            return FormValidation.error(
-                    e, e.getCause() == null ? e.getMessage() : e.getCause().getMessage());
+            return FormValidation.ok("The configuration can be applied");
+        } catch (Exception e) {
+            String errorMessage = e.getMessage();
+            if (errorMessage == null && e.getCause() != null) {
+                errorMessage = e.getCause().getMessage();
+            }
+            if (errorMessage == null) {
+                errorMessage = "An unexpected " + e.getClass().getSimpleName() + " occurred.";
+            }
+            return FormValidation.error("Invalid configuration: " + errorMessage);
         }
     }
 
@@ -326,21 +335,31 @@ private JSONArray collectProblems(Map<Source, String> issues, String severity) {
         return problems;
     }
 
-    private void appendSources(List<YamlSource> sources, String source) throws ConfiguratorException {
-        if (isSupportedURI(source)) {
-            sources.add(YamlSource.of(source));
-        } else {
-            sources.addAll(configs(source).stream().map(YamlSource::of).collect(toList()));
-        }
-    }
-
-    private List<YamlSource> getConfigFromSources(List<String> newSources) throws ConfiguratorException {
-        List<YamlSource> sources = new ArrayList<>();
+    private FetchContext getConfigFromSources(List<String> newSources) throws ConfiguratorException {
+        FetchContext context = new FetchContext();
+        FetchCredentials credentials = resolveAll();
 
         for (String p : newSources) {
-            appendSources(sources, p);
+            boolean fetched = false;
+
+            for (CasCConfigFetcher fetcher : Jenkins.get().getExtensionList(CasCConfigFetcher.class)) {
+                if (fetcher.supports(p)) {
+                    try {
+                        context.add(fetcher.fetch(p, credentials));
+                        fetched = true;
+                        break;
+                    } catch (IOException e) {
+                        throw new ConfiguratorException("Failed to fetch configuration from " + p, e);
+                    }
+                }
+            }
+
+            if (!fetched) {
+                throw new ConfiguratorException("Source '" + p
+                        + "' is not supported by any registered configuration fetcher or does not exist.");
+            }
         }
-        return sources;
+        return context;
     }
 
     /**
@@ -367,18 +386,16 @@ public static void init() throws Exception {
      * @throws ConfiguratorException Configuration error
      */
     public void configure() throws ConfiguratorException {
-        configureWith(getStandardConfigSources());
+        try (FetchContext context = getStandardConfigSources()) {
+            configureWith(context.getSources());
+        }
     }
 
-    private List<YamlSource> getStandardConfigSources() throws ConfiguratorException {
-        List<YamlSource> configs = new ArrayList<>();
-
+    private FetchContext getStandardConfigSources() throws ConfiguratorException {
         List<String> standardConfig = getStandardConfig();
-        for (String p : standardConfig) {
-            appendSources(configs, p);
-        }
+        FetchContext context = getConfigFromSources(standardConfig);
         sources = Collections.unmodifiableList(standardConfig);
-        return configs;
+        return context;
     }
 
     private List<String> getStandardConfig() {
@@ -710,38 +727,35 @@ public void configure(String... configParameters) throws ConfiguratorException {
 
     public void configure(Collection<String> configParameters) throws ConfiguratorException {
 
-        List<YamlSource> configs = new ArrayList<>();
+        List<String> newSources = new ArrayList<>(configParameters);
 
-        for (String p : configParameters) {
-            appendSources(configs, p);
+        try (FetchContext context = getConfigFromSources(newSources)) {
+            sources = Collections.unmodifiableList(newSources);
+            configureWith(context.getSources());
+            lastTimeLoaded = System.currentTimeMillis();
         }
-        sources = Collections.unmodifiableList(new ArrayList<>(configParameters));
-        configureWith(configs);
-        lastTimeLoaded = System.currentTimeMillis();
     }
 
     public static boolean isSupportedURI(String configurationParameter) {
         if (configurationParameter == null) {
             return false;
         }
-        final List<String> supportedProtocols = Arrays.asList("https", "http", "file");
-        URI uri;
-        try {
-            uri = new URI(configurationParameter);
-        } catch (URISyntaxException ex) {
-            return false;
-        }
-        if (uri.getScheme() == null) {
-            return false;
+        for (CasCConfigFetcher fetcher : Jenkins.get().getExtensionList(CasCConfigFetcher.class)) {
+            if (fetcher.supports(configurationParameter)) {
+                return true;
+            }
         }
-        return supportedProtocols.contains(uri.getScheme());
+        return false;
     }
 
     @Restricted(NoExternalUse.class)
+    @SuppressWarnings("rawtypes")
     public void configureWith(YamlSource source) throws ConfiguratorException {
-        final List<YamlSource> sources = getStandardConfigSources();
-        sources.add(source);
-        configureWith(sources);
+        try (FetchContext context = getStandardConfigSources()) {
+            List<YamlSource> allSources = new ArrayList<>(context.getSources());
+            allSources.add(source);
+            configureWith(allSources);
+        }
     }
 
     private void configureWith(List<YamlSource> sources) throws ConfiguratorException {

plugin/src/main/java/io/jenkins/plugins/casc/fetcher/BootstrapEnvVarCredentialResolver.java

@@ -0,0 +1,46 @@
+package io.jenkins.plugins.casc.fetcher;
+
+public final class BootstrapEnvVarCredentialResolver {
+
+    public static final BootstrapEnvVarCredentialResolver INSTANCE = new BootstrapEnvVarCredentialResolver();
+
+    private BootstrapEnvVarCredentialResolver() {}
+
+    @SuppressWarnings("unchecked")
+    public <T extends FetchAuthData> T resolve(String credentialId, Class<T> type) {
+        if (credentialId == null || credentialId.isEmpty() || type == null) {
+            return null;
+        }
+        if (type == FetchAuthData.Token.class) {
+            String token = System.getenv(credentialId);
+            if (token != null && !token.isEmpty()) {
+                return (T) (FetchAuthData.Token) () -> token;
+            }
+        }
+
+        if (type == FetchAuthData.SshKey.class) {
+            String privateKey = System.getenv(credentialId + "_PRIVATE_KEY");
+            if (privateKey != null && !privateKey.isEmpty()) {
+                String username = System.getenv(credentialId + "_USERNAME");
+                String passphrase = System.getenv(credentialId + "_PASSPHRASE");
+                return (T) new FetchAuthData.SshKey() {
+                    @Override
+                    public String getUsername() {
+                        return username != null ? username : "git";
+                    }
+
+                    @Override
+                    public String getPrivateKey() {
+                        return privateKey;
+                    }
+
+                    @Override
+                    public String getPassphrase() {
+                        return passphrase;
+                    }
+                };
+            }
+        }
+        return null;
+    }
+}

plugin/src/main/java/io/jenkins/plugins/casc/fetcher/CasCConfigFetcher.java

@@ -0,0 +1,11 @@
+package io.jenkins.plugins.casc.fetcher;
+
+import hudson.ExtensionPoint;
+import java.io.IOException;
+
+public interface CasCConfigFetcher extends ExtensionPoint {
+
+    boolean supports(String location);
+
+    FetchResult fetch(String location, FetchCredentials credentials) throws IOException;
+}

plugin/src/main/java/io/jenkins/plugins/casc/fetcher/DefaultHttpFetcher.java

@@ -0,0 +1,67 @@
+package io.jenkins.plugins.casc.fetcher;
+
+import static java.lang.Thread.currentThread;
+
+import hudson.Extension;
+import hudson.ProxyConfiguration;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.time.Duration;
+import java.util.Collections;
+
+@Extension(ordinal = -100)
+public class DefaultHttpFetcher implements CasCConfigFetcher {
+
+    @Override
+    public boolean supports(String location) {
+        return location != null && (location.startsWith("http://") || location.startsWith("https://"));
+    }
+
+    @Override
+    public FetchResult fetch(String location, FetchCredentials credentials) throws IOException {
+        URI uri;
+        try {
+            uri = new URI(location);
+        } catch (URISyntaxException e) {
+            throw new IOException("Invalid URL: " + location, e);
+        }
+
+        String path = uri.getPath();
+        String fileName =
+                (path != null && path.contains("/")) ? path.substring(path.lastIndexOf('/') + 1) : "casc.yaml";
+
+        if (fileName.isEmpty()) {
+            fileName = "casc.yaml";
+        }
+
+        HttpClient client = ProxyConfiguration.newHttpClient();
+
+        HttpRequest request = ProxyConfiguration.newHttpRequestBuilder(uri)
+                .GET()
+                .timeout(Duration.ofSeconds(30))
+                .build();
+
+        byte[] yamlBytes;
+        try {
+            HttpResponse<byte[]> response = client.send(request, HttpResponse.BodyHandlers.ofByteArray());
+
+            if (response.statusCode() < 200 || response.statusCode() >= 300) {
+                throw new IOException("Failed to fetch configuration from " + location + ". HTTP status code: "
+                        + response.statusCode());
+            }
+
+            yamlBytes = response.body();
+        } catch (InterruptedException e) {
+            currentThread().interrupt();
+            throw new IOException("Interrupted while fetching configuration from: " + location, e);
+        }
+
+        ResolvedYaml resolved = new ResolvedYaml(fileName, () -> new java.io.ByteArrayInputStream(yamlBytes));
+
+        return new FetchResult(Collections.singletonList(resolved), (AutoCloseable) null);
+    }
+}

plugin/src/main/java/io/jenkins/plugins/casc/fetcher/EnvVarFetchCredentialsProvider.java

@@ -0,0 +1,12 @@
+package io.jenkins.plugins.casc.fetcher;
+
+import hudson.Extension;
+
+@Extension(ordinal = -100)
+public class EnvVarFetchCredentialsProvider implements FetchCredentialsProvider {
+
+    @Override
+    public <T extends FetchAuthData> T getCredentials(String credentialId, Class<T> type) {
+        return BootstrapEnvVarCredentialResolver.INSTANCE.resolve(credentialId, type);
+    }
+}

plugin/src/main/java/io/jenkins/plugins/casc/fetcher/FetchAuthData.java

@@ -0,0 +1,25 @@
+package io.jenkins.plugins.casc.fetcher;
+
+import edu.umd.cs.findbugs.annotations.CheckForNull;
+
+public interface FetchAuthData {
+
+    interface Token extends FetchAuthData {
+        String getToken();
+    }
+
+    interface UsernamePassword extends FetchAuthData {
+        String getUsername();
+
+        String getPassword();
+    }
+
+    interface SshKey extends FetchAuthData {
+        String getUsername();
+
+        String getPrivateKey();
+
+        @CheckForNull
+        String getPassphrase();
+    }
+}