Skip to content

Commit be30c05

Browse files
jonesbusyjonesbusy
authored
Add CopyOption with includeReferrers option and ensure to copy index on index (#610)
Signed-off-by: Valentin Delaye <jonesbusy@users.noreply.github.com>
1 parent a64d114 commit be30c05

9 files changed

Lines changed: 236 additions & 50 deletions

File tree

‎src/main/java/land/oras/ContainerRef.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,15 @@ public ContainerRef withDigest(String digest) {
213213
return new ContainerRef(registry, unqualified, namespace, repository, tag, digest);
214214
}
215215

216+
/**
217+
* Return a copy of reference with the given tag
218+
* @param tag The tag
219+
* @return The container reference with the given tag
220+
*/
221+
public ContainerRef withTag(String tag) {
222+
return new ContainerRef(registry, unqualified, namespace, repository, tag, digest);
223+
}
224+
216225
@Override
217226
public SupportedAlgorithm getAlgorithm() {
218227
// Default if not set

‎src/main/java/land/oras/CopyUtils.java‎

Lines changed: 122 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -43,23 +43,102 @@ private CopyUtils() {
4343
// Utils class
4444
}
4545

46+
/**
47+
* Options for copy.
48+
* @param includeReferrers Whether to include referrers in the copy
49+
*/
50+
public record CopyOptions(boolean includeReferrers) {
51+
52+
/**
53+
* The default copy options with includeReferrers to false
54+
* @return The default copy options
55+
*/
56+
public static CopyOptions shallow() {
57+
return new CopyOptions(false);
58+
}
59+
60+
/**
61+
* The copy options with includeReferrers and recursive set to true.
62+
* @return The copy options with includeReferrers and recursive set to true
63+
*/
64+
public static CopyOptions deep() {
65+
return new CopyOptions(true);
66+
}
67+
}
68+
4669
/**
4770
* Copy a container from source to target.
71+
* @deprecated Use {@link #copy(OCI, Ref, OCI, Ref, CopyOptions)} instead. This method will be removed in a future release.
4872
* @param source The source OCI
4973
* @param sourceRef The source reference
5074
* @param target The target OCI
5175
* @param targetRef The target reference
52-
* @param recursive Whether to copy referrers recursively
76+
* @param recursive Copy refferers
5377
* @param <SourceRefType> The source reference type
5478
* @param <TargetRefType> The target reference type
5579
*/
80+
@Deprecated(forRemoval = true)
5681
public static <SourceRefType extends Ref<@NonNull SourceRefType>, TargetRefType extends Ref<@NonNull TargetRefType>>
5782
void copy(
5883
OCI<SourceRefType> source,
5984
SourceRefType sourceRef,
6085
OCI<TargetRefType> target,
6186
TargetRefType targetRef,
6287
boolean recursive) {
88+
copy(source, sourceRef, target, targetRef, recursive ? CopyOptions.deep() : CopyOptions.shallow());
89+
}
90+
91+
/**
92+
* Copy all layers for a given reference and content type from source to target.
93+
* @param source The source OCI
94+
* @param sourceRef The source reference
95+
* @param target The target OCI
96+
* @param targetRef The target reference
97+
* @param contentType The content type (manifest or index media type)
98+
* @param <SourceRefType> The source reference type
99+
* @param <TargetRefType> The target reference type
100+
*/
101+
private static <
102+
SourceRefType extends Ref<@NonNull SourceRefType>,
103+
TargetRefType extends Ref<@NonNull TargetRefType>>
104+
void copyLayers(
105+
OCI<SourceRefType> source,
106+
SourceRefType sourceRef,
107+
OCI<TargetRefType> target,
108+
TargetRefType targetRef,
109+
String contentType) {
110+
for (Layer layer : source.collectLayers(sourceRef, contentType, true)) {
111+
Objects.requireNonNull(layer.getDigest(), "Layer digest is required for streaming copy");
112+
Objects.requireNonNull(layer.getSize(), "Layer size is required for streaming copy");
113+
LOG.debug("Copying layer {}", layer.getDigest());
114+
target.pushBlob(
115+
targetRef.withDigest(layer.getDigest()),
116+
layer.getSize(),
117+
() -> source.fetchBlob(sourceRef.withDigest(layer.getDigest())),
118+
layer.getAnnotations());
119+
LOG.debug("Copied layer {}", layer.getDigest());
120+
}
121+
}
122+
123+
/**
124+
* Copy a container from source to target.
125+
* @param source The source OCI
126+
* @param sourceRef The source reference
127+
* @param target The target OCI
128+
* @param targetRef The target reference
129+
* @param options The copy option
130+
* @param <SourceRefType> The source reference type
131+
* @param <TargetRefType> The target reference type
132+
*/
133+
public static <SourceRefType extends Ref<@NonNull SourceRefType>, TargetRefType extends Ref<@NonNull TargetRefType>>
134+
void copy(
135+
OCI<SourceRefType> source,
136+
SourceRefType sourceRef,
137+
OCI<TargetRefType> target,
138+
TargetRefType targetRef,
139+
CopyOptions options) {
140+
141+
boolean includeReferrers = options.includeReferrers();
63142

64143
Descriptor descriptor = source.probeDescriptor(sourceRef);
65144

@@ -79,22 +158,12 @@ void copy(
79158
SourceRefType effectiveSourceRef = sourceRef.forTarget(source).forTarget(resolveSourceRegistry);
80159
TargetRefType effectiveTargetRef = targetRef.forTarget(target).forTarget(effectiveTargetRegistry);
81160

82-
// Write all layer
83-
for (Layer layer : source.collectLayers(effectiveSourceRef, contentType, true)) {
84-
Objects.requireNonNull(layer.getDigest(), "Layer digest is required for streaming copy");
85-
Objects.requireNonNull(layer.getSize(), "Layer size is required for streaming copy");
86-
LOG.debug("Copying layer {}", layer.getDigest());
87-
target.pushBlob(
88-
effectiveTargetRef.withDigest(layer.getDigest()),
89-
layer.getSize(),
90-
() -> source.fetchBlob(effectiveSourceRef.withDigest(layer.getDigest())),
91-
layer.getAnnotations());
92-
LOG.debug("Copied layer {}", layer.getDigest());
93-
}
94-
95161
// Single manifest
96162
if (source.isManifestMediaType(contentType)) {
97163

164+
// Write all layers
165+
copyLayers(source, effectiveSourceRef, target, effectiveTargetRef, contentType);
166+
98167
// Write manifest as any blob
99168
Manifest manifest = source.getManifest(effectiveSourceRef);
100169
String tag = effectiveSourceRef.getTag();
@@ -109,18 +178,20 @@ void copy(
109178
target.pushManifest(effectiveTargetRef.withDigest(tag), manifest);
110179
LOG.debug("Copied manifest {}", manifestDigest);
111180

112-
if (recursive) {
113-
LOG.debug("Recursively copy referrers");
181+
if (includeReferrers) {
182+
LOG.debug("Including referrers on copy of manifest {}", manifestDigest);
114183
Referrers referrers = source.getReferrers(effectiveSourceRef.withDigest(manifestDigest), null);
115184
for (ManifestDescriptor referer : referrers.getManifests()) {
116-
LOG.info("Copy reference {}", referer.getDigest());
185+
LOG.debug("Copy reference from referrers {}", referer.getDigest());
117186
copy(
118187
source,
119188
effectiveSourceRef.withDigest(referer.getDigest()),
120189
target,
121190
effectiveTargetRef,
122-
recursive);
191+
options);
123192
}
193+
} else {
194+
LOG.debug("Not including referrers on copy of manifest {}", manifestDigest);
124195
}
125196

126197
}
@@ -132,17 +203,41 @@ else if (source.isIndexMediaType(contentType)) {
132203

133204
// Write all manifests and their config
134205
for (ManifestDescriptor manifestDescriptor : index.getManifests()) {
135-
Manifest manifest = source.getManifest(effectiveSourceRef.withDigest(manifestDescriptor.getDigest()));
136206

137-
// Push config
138-
copyConfig(manifest, source, effectiveSourceRef, target, effectiveTargetRef);
207+
// Copy manifest
208+
if (source.isManifestMediaType(manifestDescriptor.getMediaType())) {
209+
Manifest manifest =
210+
source.getManifest(effectiveSourceRef.withDigest(manifestDescriptor.getDigest()));
139211

140-
// Push the manifest
141-
LOG.debug("Copying manifest {}", manifestDigest);
142-
target.pushManifest(
143-
effectiveTargetRef.withDigest(manifest.getDigest()),
144-
manifest.withDescriptor(manifestDescriptor));
145-
LOG.debug("Copied manifest {}", manifestDigest);
212+
// Copy all layers for this manifest
213+
copyLayers(
214+
source,
215+
effectiveSourceRef.withDigest(manifestDescriptor.getDigest()),
216+
target,
217+
effectiveTargetRef,
218+
manifestDescriptor.getMediaType());
219+
220+
// Push config
221+
copyConfig(manifest, source, effectiveSourceRef, target, effectiveTargetRef);
222+
223+
// Push the manifest
224+
LOG.debug("Copying nested manifest {}", manifestDescriptor.getDigest());
225+
Manifest pushedManifest = target.pushManifest(
226+
effectiveTargetRef.withDigest(manifest.getDigest()),
227+
manifest.withDescriptor(manifestDescriptor));
228+
LOG.debug("Copied nested manifest {}", manifestDescriptor.getDigest());
229+
230+
} else if (source.isIndexMediaType(manifestDescriptor.getMediaType())) {
231+
// Copy index of index
232+
LOG.debug("Copying nested index {}", manifestDescriptor.getDigest());
233+
copy(
234+
source,
235+
effectiveSourceRef.withDigest(manifestDescriptor.getDigest()),
236+
target,
237+
effectiveTargetRef.withDigest(manifestDescriptor.getDigest()),
238+
options);
239+
LOG.debug("Copied nested index {}", manifestDescriptor.getDigest());
240+
}
146241
}
147242

148243
LOG.debug("Copying index {}", manifestDigest);

‎src/main/java/land/oras/Index.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,16 @@ public ManifestDescriptor getDescriptor() {
291291
return null;
292292
}
293293

294+
/**
295+
* Return a new index with the given annotations
296+
* @param annotations The annotations
297+
* @return The index
298+
*/
299+
public Index withAnnotations(Map<String, String> annotations) {
300+
return new Index(
301+
schemaVersion, mediaType, artifactType, manifests, annotations, subject, descriptor, registry, json);
302+
}
303+
294304
/**
295305
* Return a new index with the given artifact type
296306
* @param artifactType The artifact type

‎src/main/java/land/oras/OCI.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,16 @@ protected List<Layer> collectLayers(T ref, String contentType, boolean includeAl
133133
}
134134
Index index = getIndex(ref);
135135
for (ManifestDescriptor manifestDescriptor : index.getManifests()) {
136+
String manifestContentType = manifestDescriptor.getMediaType();
137+
// We just skip unknown media type descriptor
138+
if (!isManifestMediaType(manifestContentType) && !isIndexMediaType(manifestContentType)) {
139+
LOG.info(
140+
"Unrecognized content type {}, skipping descriptor {}",
141+
manifestContentType,
142+
manifestDescriptor.getDigest());
143+
continue;
144+
}
145+
// Collect layer for each manifest
136146
List<Layer> manifestLayers =
137147
getManifest(ref.withDigest(manifestDescriptor.getDigest())).getLayers();
138148
for (Layer manifestLayer : manifestLayers) {

‎src/main/java/land/oras/Registry.java‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -255,7 +255,11 @@ public void deleteManifest(ContainerRef containerRef) {
255255
public Manifest pushManifest(ContainerRef containerRef, Manifest manifest) {
256256

257257
Map<String, String> annotations = manifest.getAnnotations();
258-
if (!annotations.containsKey(Const.ANNOTATION_CREATED) && containerRef.getDigest() == null) {
258+
259+
// Only add created annotation if not already present or from original JSON
260+
if (!annotations.containsKey(Const.ANNOTATION_CREATED)
261+
&& containerRef.getDigest() == null
262+
&& manifest.getJson() == null) {
259263
Map<String, String> manifestAnnotations = new HashMap<>(annotations);
260264
manifestAnnotations.put(Const.ANNOTATION_CREATED, Const.currentTimestamp());
261265
manifest = manifest.withAnnotations(manifestAnnotations);
@@ -289,6 +293,18 @@ public Manifest pushManifest(ContainerRef containerRef, Manifest manifest) {
289293

290294
@Override
291295
public Index pushIndex(ContainerRef containerRef, Index index) {
296+
297+
Map<String, String> annotations = index.getAnnotations();
298+
299+
// Only add created annotation if not already present or from original JSON
300+
if ((annotations == null || !annotations.containsKey(Const.ANNOTATION_CREATED))
301+
&& containerRef.getDigest() == null
302+
&& index.getJson() == null) {
303+
Map<String, String> indexAnnotations = new HashMap<>(annotations != null ? annotations : new HashMap<>());
304+
indexAnnotations.put(Const.ANNOTATION_CREATED, Const.currentTimestamp());
305+
index = index.withAnnotations(indexAnnotations);
306+
}
307+
292308
ContainerRef ref = containerRef.forRegistry(this).checkBlocked(this);
293309
if (ref.isInsecure(this) && !this.isInsecure()) {
294310
return asInsecure().pushIndex(containerRef, index);

‎src/test/java/land/oras/DockerIoITCase.java‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -99,13 +99,8 @@ void shouldCopyTagToInternalRegistry() {
9999
ContainerRef containerTarget =
100100
ContainerRef.parse("%s/docker/library/alpine:latest".formatted(unsecureRegistry.getRegistry()));
101101

102-
// CopyUtils.copy(sourceRegistry, containerSource, targetRegistry, containerTarget, true);
103-
// assertTrue(targetRegistry.exists(containerTarget));
104-
105-
Index index = targetRegistry.getIndex(containerSource);
106-
107-
// Ensure standard platform matching
108-
assertTrue(index.getManifests().stream().anyMatch(m -> m.getPlatform().equals(Platform.linux386())));
102+
CopyUtils.copy(sourceRegistry, containerSource, targetRegistry, containerTarget, CopyUtils.CopyOptions.deep());
103+
assertTrue(targetRegistry.exists(containerTarget));
109104
}
110105

111106
@Test
@@ -156,7 +151,8 @@ void shouldCopyTagToInternalRegistryViaAlias(@TempDir Path homeDir) throws Excep
156151
ContainerRef containerTarget =
157152
ContainerRef.parse("%s/docker/library/alpine:latest".formatted(unsecureRegistry.getRegistry()));
158153

159-
CopyUtils.copy(sourceRegistry, containerSource, targetRegistry, containerTarget, true);
154+
CopyUtils.copy(
155+
sourceRegistry, containerSource, targetRegistry, containerTarget, CopyUtils.CopyOptions.deep());
160156
assertTrue(targetRegistry.exists(containerTarget));
161157
});
162158
}

‎src/test/java/land/oras/GitHubContainerRegistryITCase.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ void shouldCopyTagToInternalRegistry() {
120120
ContainerRef containerTarget =
121121
ContainerRef.parse("%s/docker/library/oras:main".formatted(unsecureRegistry.getRegistry()));
122122

123-
CopyUtils.copy(sourceRegistry, containerSource, targetRegistry, containerTarget, true);
123+
CopyUtils.copy(sourceRegistry, containerSource, targetRegistry, containerTarget, CopyUtils.CopyOptions.deep());
124124
assertTrue(targetRegistry.exists(containerTarget));
125125
}
126126
}

0 commit comments

Comments
 (0)