SOLR-17949: Fix Azure Blob path/listing bugs and wire tests to shared backup suites

Align the Azure Blob backup repository with S3/GCS conventions:
- resolve()/getBlobPath() now mirror S3 exactly (fold URI host into the path),
  fixing exists()/getPathType() on virtual directories
- listDir() strips trailing delimiters so it returns bare child names per the
  BackupRepository contract
- delete() tolerates already-absent files (lenient, like local/S3)
- commitBlockList(..., true) for overwrite semantics on retried uploads
- handleBlobException() logs HTTP 404 at debug, other failures at error
- fail fast with a clear message when the container name is missing
- drop the unused CHUNK_SIZE constant and unused throws on sanitizedPath()
- document the ~195 GiB single-file block-count ceiling

Rewrite the tests to extend the shared abstract suites (AbstractBackupRepositoryTest,
AbstractIncrementalBackupTest, AbstractInstallShardTest) against Azurite via a new
AzuriteTestContainer helper. Recommend supplying secrets via sysprops/env vars in the
reference guide.

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: prateeksinghalgit

solr/modules/azure-blob-repository/build.gradle

@@ -65,7 +65,7 @@ dependencies {
   testImplementation libs.commonsio.commonsio
 
   testImplementation libs.azure.core.http.okhttp
-  testImplementation libs.squareup.okhttp3.okhttp
+  testImplementation libs.squareup.okhttp3.okhttp.jvm
 
   // Testcontainers for Azurite integration testing
   testImplementation libs.testcontainers

solr/modules/azure-blob-repository/gradle.lockfile

@@ -53,7 +53,6 @@ public class AzureBlobBackupRepository extends AbstractBackupRepository {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
   static final String BLOB_SCHEME = "blob";
-  private static final int CHUNK_SIZE = 16 * 1024 * 1024;
   private static final int COPY_BUFFER_SIZE = 8192;
 
   private AzureBlobStorageClient client;
@@ -121,6 +120,7 @@ public URI resolve(URI baseUri, String... pathComponents) {
       throw new IllegalArgumentException("URI must begin with 'blob:' scheme");
     }
 
+    // If paths contain unnecessary '/' separators, they'll be removed by URI.normalize()
     String path = baseUri + "/" + String.join("/", pathComponents);
     return URI.create(path).normalize();
   }
@@ -170,30 +170,26 @@ public void delete(URI path, Collection<String> files) throws IOException {
     Objects.requireNonNull(path, "cannot delete with a null URI");
     Objects.requireNonNull(files, "cannot delete with a null files collection");
 
-    String basePath = getBlobPath(path);
-
-    if (!client.isDirectory(basePath)) {
-      int lastSlash = basePath.lastIndexOf('/');
-      basePath = lastSlash >= 0 ? basePath.substring(0, lastSlash) : "";
-    }
-
-    final String prefix;
-    if (basePath.isEmpty() || basePath.endsWith("/")) {
-      prefix = basePath;
-    } else {
-      prefix = basePath + "/";
-    }
-
     Set<String> fullPaths =
         files.stream()
-            .map(file -> prefix + file.replaceFirst("^/+", ""))
+            .map(file -> resolve(path, file))
+            .map(this::getBlobPath)
             .collect(Collectors.toSet());
 
     if (log.isDebugEnabled()) {
       log.debug("Delete files '{}'", fullPaths);
     }
 
-    client.delete(fullPaths);
+    try {
+      client.delete(fullPaths);
+    } catch (AzureBlobNotFoundException e) {
+      // Deleting files that are already absent is a no-op at the repository level, matching the
+      // lenient behavior of the local-filesystem and S3 repositories. Any present files in the
+      // batch were still removed before this was thrown.
+      if (log.isDebugEnabled()) {
+        log.debug("Some files requested for deletion were already absent", e);
+      }
+    }
   }
 
   @Override
@@ -352,7 +348,7 @@ public void copyIndexFileTo(
 
     try (InputStream inputStream = client.pullStream(blobPath);
         IndexOutput indexOutput = dest.createOutput(destFileName, IOContext.DEFAULT)) {
-      byte[] buffer = new byte[CHUNK_SIZE];
+      byte[] buffer = new byte[COPY_BUFFER_SIZE];
       int len;
       while ((len = inputStream.read(buffer)) != -1) {
         indexOutput.writeBytes(buffer, 0, len);
@@ -377,7 +373,10 @@ private String getBlobPath(URI uri) {
     if (!BLOB_SCHEME.equalsIgnoreCase(uri.getScheme())) {
       throw new IllegalArgumentException("URI must begin with 'blob:' scheme");
     }
-    return uri.getPath();
+    // Depending on the scheme, the first path element may be parsed as the URI host (e.g.
+    // "blob://dir/file" -> host="dir"). Fold it back into the path, mirroring S3BackupRepository.
+    String host = uri.getHost();
+    return host == null ? uri.getPath() : host + uri.getPath();
   }
 
   private void writeFooter(long checksum, OutputStream outputStream) throws IOException {

solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobBackupRepository.java

@@ -18,6 +18,7 @@
 
 import org.apache.solr.common.util.EnvUtils;
 import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.StrUtils;
 
 public class AzureBlobBackupRepositoryConfig {
 
@@ -54,6 +55,12 @@ public AzureBlobBackupRepositoryConfig(NamedList<?> config) {
   }
 
   public AzureBlobStorageClient buildClient() {
+    if (StrUtils.isNullOrEmpty(containerName)) {
+      throw new IllegalArgumentException(
+          "Missing required configuration '"
+              + CONTAINER_NAME
+              + "' for the Azure Blob backup repository");
+    }
     return new AzureBlobStorageClient(
         containerName,
         connectionString,

solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobBackupRepositoryConfig.java

@@ -38,8 +38,7 @@
  */
 class AzureBlobOutputStream extends OutputStream {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
-  static final int BLOCK_SIZE = 4 * 1024 * 1024;
+  private static final int BLOCK_SIZE = 4 * 1024 * 1024;
 
   private final BlobClient blobClient;
   private final String blobPath;
@@ -220,7 +219,7 @@ void complete() throws IOException {
 
       try {
         BlockBlobClient blockBlobClient = blobClient.getBlockBlobClient();
-        blockBlobClient.commitBlockList(blockIds);
+        blockBlobClient.commitBlockList(blockIds, true);
       } catch (BlobStorageException e) {
         throw new IOException(
             "Failed to commit block list", AzureBlobStorageClient.handleBlobException(e));

solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobOutputStream.java

@@ -222,6 +222,8 @@ String[] listDir(String path) throws AzureBlobException {
                 int slashIndex = s.indexOf(BLOB_FILE_PATH_DELIMITER);
                 return slashIndex == -1 || slashIndex == s.length() - 1;
               })
+          .map(s -> s.endsWith(BLOB_FILE_PATH_DELIMITER) ? s.substring(0, s.length() - 1) : s)
+          .distinct()
           .toArray(String[]::new);
     } catch (BlobStorageException e) {
       throw handleBlobException(e);
@@ -579,7 +581,7 @@ private String getParentDirectory(String path) {
         : "";
   }
 
-  String sanitizedPath(String path) throws AzureBlobException {
+  String sanitizedPath(String path) {
     String sanitizedPath = path.trim();
     while (sanitizedPath.startsWith(BLOB_FILE_PATH_DELIMITER)) {
       sanitizedPath = sanitizedPath.substring(1).trim();
@@ -621,12 +623,14 @@ static AzureBlobException handleBlobException(BlobStorageException e) {
             e.getErrorCode(),
             e.getMessage());
 
-    log.error(errMessage);
-
     if (e.getStatusCode() == HTTP_NOT_FOUND) {
+      if (log.isDebugEnabled()) {
+        log.debug(errMessage);
+      }
       return new AzureBlobNotFoundException(errMessage, e);
-    } else {
-      return new AzureBlobException(errMessage, e);
     }
+
+    log.error(errMessage);
+    return new AzureBlobException(errMessage, e);
   }
 }

solr/modules/azure-blob-repository/src/java/org/apache/solr/azureblob/AzureBlobStorageClient.java

@@ -37,8 +37,6 @@
 import org.junit.Assume;
 import org.junit.Before;
 import org.junit.BeforeClass;
-import org.testcontainers.containers.GenericContainer;
-import org.testcontainers.utility.DockerImageName;
 
 /** Abstract class for tests with Azure Blob Storage emulator. */
 @ThreadLeakFilters(
@@ -50,10 +48,7 @@
     })
 public class AbstractAzureBlobClientTest extends SolrTestCase {
 
-  private static final String AZURITE_IMAGE = "mcr.microsoft.com/azure-storage/azurite:3.35.0";
-  private static final int BLOB_SERVICE_PORT = 10000;
-
-  private static GenericContainer<?> azuriteContainer;
+  private static AzuriteTestContainer azurite;
   private static OkHttpClient sharedOkHttpClient;
   private static String connectionString;
 
@@ -62,15 +57,10 @@ public class AbstractAzureBlobClientTest extends SolrTestCase {
 
   protected AzureBlobStorageClient client;
 
-  @SuppressWarnings("resource")
   @BeforeClass
   public static void setUpClass() {
     try {
-      azuriteContainer =
-          new GenericContainer<>(DockerImageName.parse(AZURITE_IMAGE))
-              .withExposedPorts(BLOB_SERVICE_PORT)
-              .withCommand("azurite-blob", "--blobHost", "0.0.0.0", "--skipApiVersionCheck");
-      azuriteContainer.start();
+      azurite = AzuriteTestContainer.start();
       sharedOkHttpClient = new OkHttpClient.Builder().build();
     } catch (Throwable t) {
       Assume.assumeNoException("Docker/Testcontainers not available; skipping Azure tests", t);
@@ -81,20 +71,17 @@ public static void setUpClass() {
   public void setUpClient() throws Exception {
     setAzureTestCredentials();
 
-    String blobServiceUrl = getBlobServiceUrl();
-    connectionString =
-        "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint="
-            + blobServiceUrl
-            + "/devstoreaccount1;";
+    URI blobServiceUri = new URI(getBlobServiceUrl());
+    connectionString = azurite.connectionString();
 
     proxy = new SocketProxy();
-    proxy.open(new URI(blobServiceUrl));
+    proxy.open(blobServiceUri);
 
     HttpClient httpClient = new OkHttpAsyncHttpClientBuilder(sharedOkHttpClient).build();
 
+    // Route the client through the proxy so tests can simulate connection loss.
     String proxiedConn =
-        connectionString.replace(
-            ":" + azuriteContainer.getMappedPort(BLOB_SERVICE_PORT), ":" + proxy.getListenPort());
+        connectionString.replace(":" + blobServiceUri.getPort(), ":" + proxy.getListenPort());
 
     BlobServiceClient blobServiceClient =
         new BlobServiceClientBuilder()
@@ -137,13 +124,9 @@ void initiateBlobConnectionLoss() {
 
   @AfterClass
   public static void afterAll() {
-    if (azuriteContainer != null) {
-      try {
-        azuriteContainer.stop();
-        azuriteContainer.close();
-      } catch (Throwable ignored) {
-      }
-      azuriteContainer = null;
+    if (azurite != null) {
+      azurite.stop();
+      azurite = null;
     }
     sharedOkHttpClient = null;
   }
@@ -165,10 +148,7 @@ static String getConnectionString() {
   }
 
   String getBlobServiceUrl() {
-    return "http://"
-        + azuriteContainer.getHost()
-        + ":"
-        + azuriteContainer.getMappedPort(BLOB_SERVICE_PORT);
+    return azurite.blobEndpoint();
   }
 
   public static class OkHttpThreadLeakFilterTest implements ThreadFilter {