diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/jclouds/JCloudsStore.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/jclouds/JCloudsStore.java index 6f33b7ca9ca7..8f2bd42187a4 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/jclouds/JCloudsStore.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/jclouds/JCloudsStore.java @@ -38,6 +38,7 @@ import java.net.URI; import java.util.Properties; import java.util.Set; +import java.util.stream.StreamSupport; import javax.annotation.CheckForNull; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -56,6 +57,7 @@ import org.jclouds.blobstore.BlobRequestSigner; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.BlobStoreContext; +import org.jclouds.blobstore.BlobStores; import org.jclouds.blobstore.LocalBlobRequestSigner; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.StorageMetadata; @@ -189,18 +191,22 @@ public void deleteDirectory(BlobKeyPrefix prefix) { public Iterable listFolders(BlobKeyPrefix prefix) { // JClouds directory listing requires a trailing "/" on the prefix String jcloudsPrefix = prefix.value() + "/"; - return getBlobStore() - .list(fileStoreConfig.container, prefix(jcloudsPrefix).delimiter("/")) - .stream() + return StreamSupport.stream( + BlobStores.listAll( + getBlobStore(), fileStoreConfig.container, prefix(jcloudsPrefix).delimiter("/")) + .spliterator(), + false) .map(m -> BlobKeyPrefix.of(m.getName())) .toList(); } @Override public Iterable listKeys(BlobKeyPrefix prefix) { - return getBlobStore() - .list(fileStoreConfig.container, prefix(prefix.value()).recursive()) - .stream() + return StreamSupport.stream( + BlobStores.listAll( + getBlobStore(), fileStoreConfig.container, prefix(prefix.value()).recursive()) + .spliterator(), + false) .map(StorageMetadata::getName) .map(BlobKey::new) .toList(); diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/jclouds/BlobStoreServiceContractTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/jclouds/BlobStoreServiceContractTest.java index 2dd2221a18cf..f46c128e8602 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/jclouds/BlobStoreServiceContractTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/jclouds/BlobStoreServiceContractTest.java @@ -314,6 +314,27 @@ void listKeys_filtersByPrefix() { assertEquals(Set.of(key("p1/x").value()), blobKeys); } + /** + * Regression test for the bundled-app cleanup bug where {@code listKeys} returned only the + * provider's first page (1000 keys on S3) and {@code JCloudsAppStorageService#deleteApp} silently + * stopped after deleting that page, leaving the rest of the app's files behind as orphans. + */ + @Test + void listKeys_paginatesPastDefaultPageSize() { + int total = 1100; // above the 1000-key default page size of S3-compatible stores + for (int i = 0; i < total; i++) { + putString(key("page/" + String.format("%05d", i)), "x"); + } + + Set keys = new HashSet<>(); + service().listKeys(BlobKeyPrefix.of(key("page").value())).forEach(k -> keys.add(k.value())); + + assertEquals( + total, + withoutDirectoryMarkers(keys).size(), + "listKeys must return every blob across all pages, not just the first page"); + } + // -------------------------------------------------------------------- presigning @Test