DefaultPackageRepository: simplify HTTP & JSON (#3253)

The use of a SolrClient didn't seem to make sense if the web server isn't (necessarily) Solr. Seemed to be getting more in the way. There was some double or even triple(?) JSON back & forth parsing / serializing that I couldn't let be on principle.

* Don't comment out bats tests (annoying to uncomment); use the "skip" feature

Author: dsmiley

solr/core/src/java/org/apache/solr/packagemanager/DefaultPackageRepository.java

@@ -18,26 +18,21 @@
 package org.apache.solr.packagemanager;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.core.type.TypeReference;
 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
 import java.net.URI;
 import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.Collection;
+import java.util.List;
 import java.util.Map;
-import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.FilenameUtils;
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.Http2SolrClient;
-import org.apache.solr.client.solrj.impl.JsonMapResponseParser;
-import org.apache.solr.client.solrj.request.GenericSolrRequest;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrException.ErrorCode;
-import org.apache.solr.common.util.CollectionUtil;
-import org.apache.solr.common.util.NamedList;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -86,10 +81,7 @@ public boolean hasPackage(String packageName) {
   public Path download(String artifactName) throws SolrException, IOException {
     Path tmpDirectory = Files.createTempDirectory("solr-packages");
     tmpDirectory.toFile().deleteOnExit();
-    URL url =
-        URI.create(repositoryURL.endsWith("/") ? repositoryURL : repositoryURL + "/")
-            .resolve(artifactName)
-            .toURL();
+    URL url = getRepoUri().resolve(artifactName).toURL();
     String fileName = FilenameUtils.getName(url.getPath());
     Path destination = tmpDirectory.resolve(fileName);
 
@@ -107,42 +99,24 @@ public Path download(String artifactName) throws SolrException, IOException {
     return destination;
   }
 
+  private URI getRepoUri() {
+    return URI.create(repositoryURL.endsWith("/") ? repositoryURL : repositoryURL + "/");
+  }
+
   private void initPackages() {
-    // We need http 1.1 protocol here because we are talking to the repository server and not to
-    // an actual Solr server.
-    // We use an Http2SolrClient so that we do not need a raw jetty http client for this GET.
-    // We may get a text/plain mimetype for the repository.json (for instance when looking up a repo
-    // on Github), so use custom ResponseParser.
-    try (Http2SolrClient client =
-        new Http2SolrClient.Builder(repositoryURL).useHttp1_1(true).build()) {
-      GenericSolrRequest request =
-          new GenericSolrRequest(SolrRequest.METHOD.GET, "/repository.json");
-      request.setResponseParser(new TalkToRepoResponseParser());
-      NamedList<Object> resp = client.request(request);
-      SolrPackage[] items =
-          PackageUtils.getMapper().readValue("[" + resp.jsonStr() + "]", SolrPackage[].class);
-      packages = CollectionUtil.newHashMap(items.length);
-      for (SolrPackage pkg : items) {
-        pkg.setRepository(name);
-        packages.put(pkg.name, pkg);
-      }
-    } catch (SolrServerException | IOException ex) {
+    try {
+      final var url = getRepoUri().resolve("repository.json").toURL();
+      packages =
+          PackageUtils.getMapper()
+              .readValue(url, new TypeReference<List<SolrPackage>>() {})
+              .stream()
+              .peek(pkg -> pkg.setRepository(name))
+              .collect(Collectors.toMap(pkg -> pkg.name, Function.identity()));
+    } catch (IOException ex) {
       throw new SolrException(ErrorCode.INVALID_STATE, ex);
     }
     if (log.isDebugEnabled()) {
-      log.debug("Found {} packages in repository '{}'", packages.size(), name);
-    }
-  }
-
-  /**
-   * Github links for repository.json are returned in JSON format but with text/plain mimetype, so
-   * this works around that issue.
-   */
-  private static class TalkToRepoResponseParser extends JsonMapResponseParser {
-
-    @Override
-    public Collection<String> getContentTypes() {
-      return Set.of("application/json", "text/plain");
+      log.debug("Found {} packages in repository '{}'", this.packages.size(), name);
     }
   }
 }

solr/packaging/test/test_packages.bats

@@ -58,22 +58,24 @@ teardown() {
 }
 
 # This test is useful if you are debugging/working with packages.
-# We have commented it out for now since it depends on a live internet
+# We have disabled it for now since it depends on a live internet
 # connection to run.  This could be updated with a local Repo server if we had
 # a package that is part of the Solr project to use.
-# @test "deploying and undeploying a cluster level package" {
-#  run solr start -Denable.packages=true
+@test "deploying and undeploying a cluster level package" {
+  skip "For developing package infra; requires a connection to github"
+
+  run solr start -Denable.packages=true
 
-#  run solr package add-repo splainer "https://raw.githubusercontent.com/o19s/splainer/main/solr-splainer-package/repo"
-#  assert_output --partial "Added repository: splainer"
+  run solr package add-repo splainer "https://raw.githubusercontent.com/o19s/splainer/refs/heads/main/solr-splainer-package/repo/"
+  assert_output --partial "Added repository: splainer"
 
-#  run solr package list-available
-#  assert_output --partial "solr-splainer 		Splainer for Solr"
-#  run solr package install solr-splainer
-#  assert_output --partial "solr-splainer installed."
+  run solr package list-available
+  assert_output --partial "solr-splainer 		Splainer for Solr"
+  run solr package install solr-splainer
+  assert_output --partial "solr-splainer installed."
 
-#  run solr package deploy solr-splainer -y --cluster
-#  assert_output --partial "Deployment successful"
+  run solr package deploy solr-splainer -y --cluster
+  assert_output --partial "Deployment successful"
 
-#  run -0 curl --fail http://localhost:${SOLR_PORT}/v2/splainer/index.html
-# }
+  run -0 curl --fail http://localhost:${SOLR_PORT}/v2/splainer/index.html
+}