Tags pagination (#628)

Signed-off-by: Valentin Delaye <jonesbusy@users.noreply.github.com>

Author: jonesbusy

src/main/java/land/oras/ContainerRef.java

@@ -271,12 +271,46 @@ public String getTagsPath(@Nullable Registry target) {
         return "%s/tags/list".formatted(getApiPrefix(target));
     }
 
+    /**
+     * Return the tag URL
+     * @param n The optional number of tags to return, for pagination
+     * @param last The optional last tag index, for pagination
+     * @return The tag URL
+     */
+    public String getTagsPath(@Nullable Integer n, @Nullable String last) {
+        return getTagsPath(null, n, last);
+    }
+
+    /**
+     * Return the tag URL
+     * @param n The optional number of tags to return, for pagination
+     * @param last The optional last tag index, for pagination
+     * @param target The target registry
+     * @return The tag URL
+     */
+    public String getTagsPath(@Nullable Registry target, @Nullable Integer n, @Nullable String last) {
+        if (n == null && last == null) {
+            return getTagsPath(target);
+        }
+        StringBuilder url = new StringBuilder(getTagsPath(target)).append("?");
+        if (n != null) {
+            url.append("n=").append(n);
+        }
+        if (last != null) {
+            if (n != null) {
+                url.append("&");
+            }
+            url.append("last=").append(URLEncoder.encode(last, StandardCharsets.UTF_8));
+        }
+        return url.toString();
+    }
+
     /**
      * Return the tag URL
      * @return The tag URL
      */
     public String getTagsPath() {
-        return getTagsPath(null);
+        return getTagsPath(null, null);
     }
 
     /**

src/main/java/land/oras/OCI.java

@@ -307,6 +307,15 @@ public final Manifest attachArtifact(T ref, ArtifactType artifactType, LocalPath
      */
     public abstract Tags getTags(T ref);
 
+    /**
+     * Get the tags for a ref
+     * @param ref The ref
+     * @param n The number of tags to return. If n is less than or equal to 0, return all tags
+     * @param last The last tag index, to iterate. If null, start from the beginning
+     * @return The tags
+     */
+    public abstract Tags getTags(T ref, int n, @Nullable String last);
+
     /**
      * Get the tags for a ref
      * @return The repositories

src/main/java/land/oras/OCILayout.java

@@ -329,10 +329,26 @@ public Tags getTags(LayoutRef ref) {
         List<String> tags = index.getManifests().stream()
                 .filter(m -> m.getAnnotations() != null && m.getAnnotations().containsKey(Const.ANNOTATION_REF))
                 .map(m -> m.getAnnotations().get(Const.ANNOTATION_REF))
+                .sorted()
                 .toList();
         return new Tags(name, tags);
     }
 
+    @Override
+    public Tags getTags(LayoutRef ref, int n, @Nullable String last) {
+        Tags allTags = getTags(ref);
+        String name = allTags.name();
+        List<String> tags = allTags.tags();
+        int startIndex = 0;
+        if (last != null) {
+            int lastIndex = tags.indexOf(last);
+            if (lastIndex == -1) {
+                throw new OrasException("Last tag not found: %s".formatted(last));
+            }
+        }
+        return new Tags(name, tags.stream().skip(startIndex).limit(n).toList());
+    }
+
     @Override
     public Repositories getRepositories() {
         return new Repositories(List.of(path.getFileName().toString()));

src/main/java/land/oras/Registry.java

@@ -32,6 +32,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.function.Supplier;
 import land.oras.auth.AuthProvider;
 import land.oras.auth.AuthStoreAuthenticationProvider;
@@ -200,8 +201,25 @@ public Tags getTags(ContainerRef containerRef) {
         URI uri = URI.create("%s://%s".formatted(getScheme(), ref.getTagsPath(this)));
         HttpClient.ResponseWrapper<String> response = client.get(
                 uri, Map.of(Const.ACCEPT_HEADER, Const.DEFAULT_JSON_MEDIA_TYPE), Scopes.of(this, ref), authProvider);
+        logResponse(response);
         handleError(response);
-        return JsonUtils.fromJson(response.response(), Tags.class);
+        return JsonUtils.fromJson(response.response(), Tags.class)
+                .withLast(getLastFromLink(response).orElse(null));
+    }
+
+    @Override
+    public Tags getTags(ContainerRef containerRef, int n, @Nullable String last) {
+        ContainerRef ref = containerRef.forRegistry(this).checkBlocked(this);
+        if (ref.isInsecure(this) && !this.isInsecure()) {
+            return asInsecure().getTags(containerRef);
+        }
+        URI uri = URI.create("%s://%s".formatted(getScheme(), ref.getTagsPath(this, n, last)));
+        HttpClient.ResponseWrapper<String> response = client.get(
+                uri, Map.of(Const.ACCEPT_HEADER, Const.DEFAULT_JSON_MEDIA_TYPE), Scopes.of(this, ref), authProvider);
+        logResponse(response);
+        handleError(response);
+        return JsonUtils.fromJson(response.response(), Tags.class)
+                .withLast(getLastFromLink(response).orElse(null));
     }
 
     @Override
@@ -215,6 +233,7 @@ && getRegistriesConf().isInsecure(ContainerRef.parse(registry).forRegistry(regis
         URI uri = URI.create("%s://%s".formatted(getScheme(), ref.getRepositoriesPath(this)));
         HttpClient.ResponseWrapper<String> response = client.get(
                 uri, Map.of(Const.ACCEPT_HEADER, Const.DEFAULT_JSON_MEDIA_TYPE), Scopes.of(this, ref), authProvider);
+        logResponse(response);
         handleError(response);
         return JsonUtils.fromJson(response.response(), Repositories.class);
     }
@@ -231,6 +250,7 @@ public Referrers getReferrers(ContainerRef containerRef, @Nullable ArtifactType
         URI uri = URI.create("%s://%s".formatted(getScheme(), ref.getReferrersPath(this, artifactType)));
         HttpClient.ResponseWrapper<String> response = client.get(
                 uri, Map.of(Const.ACCEPT_HEADER, Const.DEFAULT_INDEX_MEDIA_TYPE), Scopes.of(this, ref), authProvider);
+        logResponse(response);
         handleError(response);
         return JsonUtils.fromJson(response.response(), Referrers.class);
     }
@@ -968,6 +988,35 @@ ResolvedRegistry getResolvedHeaders(ContainerRef containerRef) {
         return new ResolvedRegistry(ref.getRegistry(), response.headers());
     }
 
+    private Optional<String> getLastFromLink(HttpClient.ResponseWrapper<String> response) {
+        String linkHeader = response.headers().get(Const.LINK_HEADER.toLowerCase());
+        if (linkHeader == null) {
+            return Optional.empty();
+        }
+
+        int start = linkHeader.indexOf('<');
+        int end = linkHeader.indexOf('>', start + 1);
+        if (start == -1 || end == -1) {
+            return Optional.empty();
+        }
+
+        String uri = linkHeader.substring(start + 1, end);
+        int q = uri.indexOf('?');
+        if (q == -1) {
+            return Optional.empty();
+        }
+
+        String query = uri.substring(q + 1);
+        for (String param : query.split("&")) {
+            int eq = param.indexOf('=');
+            if (eq > 0 && "last".equals(param.substring(0, eq))) {
+                return Optional.of(param.substring(eq + 1));
+            }
+        }
+
+        return Optional.empty();
+    }
+
     /**
      * Holds a resolved registry to avoid resolution on every request (specially like blob)
      * @param registry The registry URL

src/main/java/land/oras/Tags.java

@@ -22,12 +22,33 @@
 
 import java.util.List;
 import org.jspecify.annotations.NullMarked;
+import org.jspecify.annotations.Nullable;
 
 /**
  * The tags response object
  * @param name The name
  * @param tags The tags
+ * @param last The last tag index, to iterate
  */
 @NullMarked
 @OrasModel
-public record Tags(String name, List<String> tags) {}
+public record Tags(String name, List<String> tags, @Nullable String last) {
+
+    /**
+     * Constructor without last
+     * @param name The name
+     * @param tags The tags
+     */
+    public Tags(String name, List<String> tags) {
+        this(name, tags, null);
+    }
+
+    /**
+     * With last
+     * @param last The last tag index, to iterate
+     * @return A new Tags object with the last index
+     */
+    public Tags withLast(@Nullable String last) {
+        return new Tags(this.name, this.tags, last);
+    }
+}

src/main/java/land/oras/utils/Const.java

@@ -376,6 +376,11 @@ public static String currentTimestamp() {
      */
     public static final String AUTHORIZATION_HEADER = "Authorization";
 
+    /**
+     * Link header, which is used for pagination in the registry API
+     */
+    public static final String LINK_HEADER = "Link";
+
     /**
      * User agent header
      */

src/test/java/land/oras/ContainerRefTest.java

@@ -370,6 +370,10 @@ void shouldParseImageWithNoTagAndNoRegistry() {
     void shouldGetTagsPathDockerIo() {
         ContainerRef containerRef = ContainerRef.parse("docker.io/library/foo/alpine:latest@sha256:1234567890abcdef");
         assertEquals("registry-1.docker.io/v2/library/foo/alpine/tags/list", containerRef.getTagsPath());
+        assertEquals("registry-1.docker.io/v2/library/foo/alpine/tags/list?n=1", containerRef.getTagsPath(1, null));
+        assertEquals(
+                "registry-1.docker.io/v2/library/foo/alpine/tags/list?n=1&last=latest",
+                containerRef.getTagsPath(1, "latest"));
     }
 
     @Test

src/test/java/land/oras/DockerIoITCase.java

@@ -41,6 +41,23 @@ class DockerIoITCase {
     @Container
     private final ZotUnsecureContainer unsecureRegistry = new ZotUnsecureContainer().withStartupAttempts(3);
 
+    @Test
+    void shouldGetTags() {
+        Registry registry = Registry.builder().build();
+        ContainerRef containerRef = ContainerRef.parse("docker.io/library/alpine");
+        Tags tags = registry.getTags(containerRef);
+        assertNotNull(tags);
+        assertTrue(tags.tags().size() > 100);
+        tags = registry.getTags(containerRef, 2, null);
+        assertNotNull(tags);
+        assertEquals(2, tags.tags().size());
+        assertEquals("2.7", tags.last());
+        tags = registry.getTags(containerRef, 2, tags.last());
+        assertNotNull(tags);
+        assertEquals(2, tags.tags().size());
+        assertEquals("20190408", tags.last());
+    }
+
     @Test
     void shouldPullAnonymousIndexFQDN() {
 

src/test/java/land/oras/OCILayoutTest.java

@@ -141,6 +141,33 @@ void shouldListTags() throws Exception {
         assertEquals("latest", tags.tags().get(0));
     }
 
+    @Test
+    void shouldListTagsWithLimit() throws Exception {
+        Path extractDir1 = extractDir.resolve("shouldListTags");
+        Files.createDirectory(extractDir1);
+
+        LayoutRef layoutRef = LayoutRef.parse("src/test/resources/oci/subject:latest");
+        OCILayout ociLayout =
+                OCILayout.Builder.builder().defaults(layoutRef.getFolder()).build();
+        Tags tags = ociLayout.getTags(layoutRef, 1, null);
+        assertEquals("subject", tags.name());
+        assertEquals(1, tags.tags().size());
+        assertEquals("latest", tags.tags().get(0));
+    }
+
+    @Test
+    void shouldThrowIfLastTagInvalid() throws Exception {
+        Path extractDir1 = extractDir.resolve("shouldListTags");
+        Files.createDirectory(extractDir1);
+
+        LayoutRef layoutRef = LayoutRef.parse("src/test/resources/oci/subject:latest");
+        OCILayout ociLayout =
+                OCILayout.Builder.builder().defaults(layoutRef.getFolder()).build();
+        assertThrows(OrasException.class, () -> {
+            ociLayout.getTags(layoutRef, 1, "unknown");
+        });
+    }
+
     @Test
     void shouldListRepositories() throws Exception {
         Path extractDir1 = extractDir.resolve("shouldListRepositories");

src/test/java/land/oras/RegistryWireMockTest.java

@@ -237,6 +237,33 @@ void shouldListTags(WireMockRuntimeInfo wmRuntimeInfo) {
         assertEquals("0.1.1", tags.get(1));
     }
 
+    @Test
+    void shouldListTagsWithLimit(WireMockRuntimeInfo wmRuntimeInfo) {
+
+        // Return data from wiremock
+        WireMock wireMock = wmRuntimeInfo.getWireMock();
+        wireMock.register(WireMock.get(WireMock.urlEqualTo("/v2/library/artifact-text/tags/list?n=1"))
+                .willReturn(WireMock.okJson(JsonUtils.toJson(new Tags("artifact-text", List.of("latest"))))));
+
+        // Insecure registry
+        Registry registry = Registry.Builder.builder()
+                .withAuthProvider(authProvider)
+                .withInsecure(true)
+                .build();
+
+        // Test
+        List<String> tags = registry.getTags(
+                        ContainerRef.parse("%s/library/artifact-text"
+                                .formatted(wmRuntimeInfo.getHttpBaseUrl().replace("http://", ""))),
+                        1,
+                        null)
+                .tags();
+
+        // Assert
+        assertEquals(1, tags.size());
+        assertEquals("latest", tags.get(0));
+    }
+
     @Test
     void shouldListTagsWithConfig(WireMockRuntimeInfo wmRuntimeInfo) throws Exception {
 
@@ -267,6 +294,16 @@ void shouldListTagsWithConfig(WireMockRuntimeInfo wmRuntimeInfo) throws Exceptio
             assertEquals(2, tags.size());
             assertEquals("latest", tags.get(0));
             assertEquals("0.1.1", tags.get(1));
+
+            // With limit
+            tags = registry.getTags(
+                            ContainerRef.parse("%s/library/artifact-text-with-confg".formatted(registryAsString)),
+                            1,
+                            null)
+                    .tags();
+            assertEquals(2, tags.size());
+            assertEquals("latest", tags.get(0));
+            assertEquals("0.1.1", tags.get(1));
         });
     }