Skip to content

Commit 29ed324

Browse files
ducminh02hohwille
ducminh02
and
hohwille
authored
#1447: Add deterministic archive generation and checksum verification tests (#1826)
Co-authored-by: Jörg Hohwiller <hohwille@users.noreply.github.com>
1 parent 9847d70 commit 29ed324

6 files changed

Lines changed: 316 additions & 5 deletions

File tree

‎cli/src/main/java/com/devonfw/tools/ide/io/FileAccessImpl.java‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,9 +47,11 @@
4747
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
4848
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
4949
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
50+
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
5051
import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
5152
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
5253
import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
54+
import org.apache.commons.compress.compressors.gzip.GzipParameters;
5355
import org.apache.commons.io.IOUtils;
5456
import org.slf4j.Logger;
5557
import org.slf4j.LoggerFactory;
@@ -980,7 +982,9 @@ public void compressTar(Path dir, OutputStream out, TarCompression tarCompressio
980982
@Override
981983
public void compressTarGz(Path dir, OutputStream out) {
982984

983-
try (GzipCompressorOutputStream gzOut = new GzipCompressorOutputStream(out)) {
985+
GzipParameters parameters = new GzipParameters();
986+
parameters.setModificationTime(0);
987+
try (GzipCompressorOutputStream gzOut = new GzipCompressorOutputStream(out, parameters)) {
984988
compressTarOrThrow(dir, gzOut);
985989
} catch (IOException e) {
986990
throw new IllegalStateException("Failed to compress directory " + dir + " to tar.gz file.", e);
@@ -1028,15 +1032,15 @@ public void compressZip(Path dir, OutputStream out) {
10281032

10291033
private <E extends ArchiveEntry> void compressRecursive(Path path, ArchiveOutputStream<E> out, String relativePath) {
10301034

1031-
try (Stream<Path> childStream = Files.list(path)) {
1035+
try (Stream<Path> childStream = Files.list(path).sorted()) {
10321036
Iterator<Path> iterator = childStream.iterator();
10331037
while (iterator.hasNext()) {
10341038
Path child = iterator.next();
10351039
String relativeChildPath = relativePath + "/" + child.getFileName().toString();
10361040
boolean isDirectory = Files.isDirectory(child);
10371041
E archiveEntry = out.createArchiveEntry(child, relativeChildPath);
1042+
FileTime none = FileTime.fromMillis(0);
10381043
if (archiveEntry instanceof TarArchiveEntry tarEntry) {
1039-
FileTime none = FileTime.fromMillis(0);
10401044
tarEntry.setCreationTime(none);
10411045
tarEntry.setModTime(none);
10421046
tarEntry.setLastAccessTime(none);
@@ -1047,6 +1051,11 @@ private <E extends ArchiveEntry> void compressRecursive(Path path, ArchiveOutput
10471051
tarEntry.setGroupName("group");
10481052
PathPermissions filePermissions = getFilePermissions(child);
10491053
tarEntry.setMode(filePermissions.toMode());
1054+
} else if (archiveEntry instanceof ZipArchiveEntry zipEntry) {
1055+
zipEntry.setCreationTime(none);
1056+
zipEntry.setLastAccessTime(none);
1057+
zipEntry.setLastModifiedTime(none);
1058+
zipEntry.setTime(none);
10501059
}
10511060
out.putArchiveEntry(archiveEntry);
10521061
if (!isDirectory) {

‎cli/src/test/java/com/devonfw/tools/ide/context/MvnRepositoryMock.java‎

Lines changed: 57 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,20 @@
1010
import java.io.IOException;
1111
import java.nio.file.Files;
1212
import java.nio.file.Path;
13+
import java.security.MessageDigest;
14+
import java.security.NoSuchAlgorithmException;
15+
import java.util.Collections;
16+
import java.util.HashMap;
17+
import java.util.Iterator;
18+
import java.util.Map;
1319
import java.util.stream.Stream;
1420

1521
import com.devonfw.tools.ide.tool.mvn.MvnArtifact;
1622
import com.devonfw.tools.ide.tool.mvn.MvnArtifactMetadata;
1723
import com.devonfw.tools.ide.tool.mvn.MvnRepository;
1824
import com.devonfw.tools.ide.url.model.file.UrlChecksums;
25+
import com.devonfw.tools.ide.url.model.file.UrlGenericChecksum;
26+
import com.devonfw.tools.ide.util.HexUtil;
1927
import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo;
2028

2129
/**
@@ -25,6 +33,22 @@ public class MvnRepositoryMock extends MvnRepository {
2533

2634
private final WireMockRuntimeInfo wmRuntimeInfo;
2735

36+
/**
37+
* Maps artifact download path to its pre-computed SHA-256 checksum.
38+
* Populated when the mock archive is compressed, queried during verification.
39+
*/
40+
private final Map<String, String> checksumByPath = new HashMap<>();
41+
42+
/**
43+
* Registers an expected SHA-256 checksum for a given artifact path.
44+
*
45+
* @param path the artifact path (relative to repo root, e.g. "/org/apache/maven/apache-maven/3.8.1/apache-maven-3.8.1-bin.zip").
46+
* @param sha256 the expected SHA-256 hash.
47+
*/
48+
public void putExpectedChecksum(String path, String sha256) {
49+
this.checksumByPath.put(path, sha256);
50+
}
51+
2852
/**
2953
* The constructor.
3054
*
@@ -47,6 +71,15 @@ public Path download(MvnArtifactMetadata metadata) {
4771
try (ByteArrayOutputStream baos = new ByteArrayOutputStream(1024)) {
4872
this.context.getFileAccess().compress(archiveFolder, baos, artifact.getFilename());
4973
byte[] body = baos.toByteArray();
74+
// Pre-compute and store the SHA-256 of the archive bytes so verifyChecksum() can check them.
75+
// We use putIfAbsent so that a hardcoded 'expected' checksum from a test takes precedence.
76+
String sha256;
77+
try {
78+
sha256 = HexUtil.toHexString(MessageDigest.getInstance("SHA-256").digest(body));
79+
} catch (NoSuchAlgorithmException e) {
80+
throw new IllegalStateException("SHA-256 algorithm not found", e);
81+
}
82+
this.checksumByPath.putIfAbsent(path, sha256);
5083
stubFor(get(urlPathEqualTo(path)).willReturn(
5184
aResponse().withStatus(200).withBody(body)));
5285
} catch (IOException e) {
@@ -56,8 +89,30 @@ public Path download(MvnArtifactMetadata metadata) {
5689
}
5790

5891
@Override
59-
protected UrlChecksums getChecksums(MvnArtifact artifact) {
60-
return null;
92+
public UrlChecksums getChecksums(MvnArtifact artifact) {
93+
String path = artifact.getDownloadUrl().replace(MvnRepositoryMock.MAVEN_CENTRAL, "");
94+
String sha256 = this.checksumByPath.get(path);
95+
if (sha256 == null) {
96+
// checksum not yet computed (e.g. metadata requests) – skip verification
97+
return null;
98+
}
99+
return new SingleChecksumWrapper(sha256);
100+
}
101+
102+
103+
/**
104+
* Simple {@link UrlChecksums} wrapper for a single pre-computed checksum.
105+
*/
106+
private record SingleChecksumWrapper(String sha256) implements UrlChecksums {
107+
108+
@Override
109+
public Iterator<UrlGenericChecksum> iterator() {
110+
UrlGenericChecksum entry = new UrlGenericChecksum() {
111+
@Override public String getChecksum() { return sha256; }
112+
@Override public String getHashAlgorithm() { return "SHA-256"; }
113+
};
114+
return Collections.singletonList(entry).iterator();
115+
}
61116
}
62117

63118
@Override

‎cli/src/test/java/com/devonfw/tools/ide/io/ArchiveDeterminismTest.java‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
package com.devonfw.tools.ide.io;
2+
3+
import java.io.IOException;
4+
import java.io.OutputStream;
5+
import java.nio.file.Files;
6+
import java.nio.file.Path;
7+
import java.nio.file.attribute.FileTime;
8+
9+
import org.junit.jupiter.api.Test;
10+
import org.junit.jupiter.api.io.TempDir;
11+
12+
import com.devonfw.tools.ide.context.AbstractIdeContextTest;
13+
import com.devonfw.tools.ide.context.IdeTestContext;
14+
15+
/**
16+
* Test of archive determinism in {@link FileAccessImpl}.
17+
*/
18+
public class ArchiveDeterminismTest extends AbstractIdeContextTest {
19+
20+
@TempDir
21+
Path tempDir;
22+
23+
/**
24+
* Test that {@link FileAccessImpl#compressTarGz(Path, OutputStream)} is deterministic.
25+
*
26+
* @throws IOException if an I/O error occurs.
27+
*/
28+
@Test
29+
public void testTarGzDeterminism() throws IOException, InterruptedException {
30+
31+
// arrange
32+
IdeTestContext context = new IdeTestContext();
33+
FileAccessImpl fileAccess = new FileAccessImpl(context);
34+
Path contentDir = this.tempDir.resolve("content");
35+
Files.createDirectories(contentDir);
36+
Files.writeString(contentDir.resolve("file1.txt"), "Content 1");
37+
Path binDir = contentDir.resolve("bin");
38+
Files.createDirectories(binDir);
39+
Files.writeString(binDir.resolve("script.sh"), "#!/bin/bash\necho hello");
40+
41+
Path archive1 = this.tempDir.resolve("archive1.tar.gz");
42+
Path archive2 = this.tempDir.resolve("archive2.tar.gz");
43+
44+
// act
45+
try (OutputStream out1 = Files.newOutputStream(archive1)) {
46+
fileAccess.compressTarGz(contentDir, out1);
47+
}
48+
// Modify modification time of a file to ensure it would affect the hash if not normalized
49+
Path file1 = contentDir.resolve("file1.txt");
50+
FileTime newTime = FileTime.fromMillis(System.currentTimeMillis() + 10000);
51+
Files.setLastModifiedTime(file1, newTime);
52+
try (OutputStream out2 = Files.newOutputStream(archive2)) {
53+
fileAccess.compressTarGz(contentDir, out2);
54+
}
55+
56+
// assert
57+
assertThat(archive1).hasSameBinaryContentAs(archive2);
58+
}
59+
60+
/**
61+
* Test that {@link FileAccessImpl#compressZip(Path, OutputStream)} is deterministic.
62+
*
63+
* @throws IOException if an I/O error occurs.
64+
*/
65+
@Test
66+
public void testZipDeterminism() throws IOException, InterruptedException {
67+
68+
// arrange
69+
IdeTestContext context = new IdeTestContext();
70+
FileAccessImpl fileAccess = new FileAccessImpl(context);
71+
Path contentDir = this.tempDir.resolve("content-zip");
72+
Files.createDirectories(contentDir);
73+
Files.writeString(contentDir.resolve("file1.txt"), "Content 1");
74+
75+
Path archive1 = this.tempDir.resolve("archive1.zip");
76+
Path archive2 = this.tempDir.resolve("archive2.zip");
77+
78+
// act
79+
try (OutputStream out1 = Files.newOutputStream(archive1)) {
80+
fileAccess.compressZip(contentDir, out1);
81+
}
82+
// Modify modification time of a file to ensure it would affect the hash if not normalized
83+
Path file1 = contentDir.resolve("file1.txt");
84+
FileTime newTime = FileTime.fromMillis(System.currentTimeMillis() + 10000);
85+
Files.setLastModifiedTime(file1, newTime);
86+
try (OutputStream out2 = Files.newOutputStream(archive2)) {
87+
fileAccess.compressZip(contentDir, out2);
88+
}
89+
90+
// assert
91+
assertThat(archive1).hasSameBinaryContentAs(archive2);
92+
}
93+
}

‎cli/src/test/java/com/devonfw/tools/ide/tool/mvn/MvnFinalChecksumTest.java‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
package com.devonfw.tools.ide.tool.mvn;
2+
3+
import java.io.IOException;
4+
import java.nio.file.Files;
5+
import java.nio.file.Path;
6+
7+
import org.junit.jupiter.api.Test;
8+
9+
import com.devonfw.tools.ide.context.AbstractIdeContextTest;
10+
import com.devonfw.tools.ide.context.IdeTestContext;
11+
import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo;
12+
import com.github.tomakehurst.wiremock.junit5.WireMockTest;
13+
14+
/**
15+
* Integration test verifying that the deterministic archives produced by our mock repositories
16+
* match our hardcoded "Gold Standard" values.
17+
*/
18+
@WireMockTest
19+
class MvnFinalChecksumTest extends AbstractIdeContextTest {
20+
21+
private static final String PROJECT_MVN = "mvn";
22+
23+
24+
25+
/**
26+
* Integration test verifying that the tool installation correctly performs and validates
27+
* the checksum from the URL repository (urls.sha256).
28+
*/
29+
@Test
30+
void testVerifyUrlChecksum(WireMockRuntimeInfo wmRuntimeInfo) throws IOException {
31+
32+
// 1. Arrange: Use the "mvn" project context with WireMock
33+
IdeTestContext context = newContext(PROJECT_MVN, wmRuntimeInfo);
34+
Mvn mvn = context.getCommandletManager().getCommandlet(Mvn.class);
35+
36+
// Read the expected hash from the URL repository file in the test resources
37+
Path urlsSha256Path = context.getIdePath().resolve("urls/mvn/mvn/3.9.7/urls.sha256");
38+
String expectedSha256 = Files.readString(urlsSha256Path).trim();
39+
40+
// 2. Act: Trigger the installation of Maven 3.9.7
41+
// This will use ToolRepositoryMock which triggers deterministic compression
42+
// and verifies it against urls.sha256 in the test resources.
43+
mvn.install();
44+
45+
// 3. Assert: Verify the installation and checksum success logs
46+
assertThat(context).logAtSuccess().hasMessageContaining("Successfully installed mvn in version 3.9.7");
47+
assertThat(context).logAtSuccess().hasMessageContaining("SHA-256 checksum " + expectedSha256 + " is correct.");
48+
}
49+
}

‎cli/src/test/java/com/devonfw/tools/ide/tool/repository/ChecksumVerificationTest.java‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
package com.devonfw.tools.ide.tool.repository;
2+
3+
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
4+
import static org.junit.jupiter.api.Assertions.assertThrows;
5+
6+
import java.io.IOException;
7+
import java.nio.file.Files;
8+
import java.nio.file.Path;
9+
10+
import org.junit.jupiter.api.Test;
11+
import org.junit.jupiter.api.io.TempDir;
12+
13+
import com.devonfw.tools.ide.cli.CliException;
14+
import com.devonfw.tools.ide.context.AbstractIdeContextTest;
15+
import com.devonfw.tools.ide.context.IdeTestContext;
16+
import com.devonfw.tools.ide.url.model.file.UrlGenericChecksum;
17+
18+
/**
19+
* Test of checksum verification in {@link AbstractToolRepository}.
20+
*/
21+
public class ChecksumVerificationTest extends AbstractIdeContextTest {
22+
23+
@TempDir
24+
Path tempDir;
25+
26+
/**
27+
* Test {@link AbstractToolRepository#verifyChecksum(Path, UrlGenericChecksum)} with matching checksum.
28+
*
29+
* @throws IOException if an I/O error occurs.
30+
*/
31+
@Test
32+
public void testVerifyChecksumMatching() throws IOException {
33+
34+
// arrange
35+
IdeTestContext context = newContext(PROJECT_BASIC);
36+
AbstractToolRepository repo = new DefaultToolRepository(context);
37+
Path file = this.tempDir.resolve("testfile.txt");
38+
String content = "Hello World";
39+
Files.writeString(file, content);
40+
String checksum = context.getFileAccess().checksum(file, "SHA-256");
41+
UrlGenericChecksum expectedChecksum = new TestUrlGenericChecksum(checksum, "SHA-256");
42+
43+
// act & assert
44+
assertDoesNotThrow(() -> {
45+
repo.verifyChecksum(file, expectedChecksum);
46+
});
47+
}
48+
49+
/**
50+
* Test {@link AbstractToolRepository#verifyChecksum(Path, UrlGenericChecksum)} with mismatching checksum.
51+
*
52+
* @throws IOException if an I/O error occurs.
53+
*/
54+
@Test
55+
public void testVerifyChecksumMismatch() throws IOException {
56+
57+
// arrange
58+
IdeTestContext context = newContext(PROJECT_BASIC);
59+
AbstractToolRepository repo = new DefaultToolRepository(context);
60+
Path file = this.tempDir.resolve("testfile.txt");
61+
String content = "Hello World";
62+
Files.writeString(file, content);
63+
String wrongChecksum = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; // SHA-256 of empty string
64+
UrlGenericChecksum expectedChecksum = new TestUrlGenericChecksum(wrongChecksum, "SHA-256");
65+
66+
// act & assert
67+
CliException e = assertThrows(CliException.class, () -> {
68+
repo.verifyChecksum(file, expectedChecksum);
69+
});
70+
assertThat(e).hasMessageContaining("has the wrong SHA-256 checksum");
71+
assertThat(e).hasMessageContaining("Expected " + wrongChecksum);
72+
}
73+
74+
private static class TestUrlGenericChecksum implements UrlGenericChecksum {
75+
76+
private final String checksum;
77+
78+
private final String algorithm;
79+
80+
public TestUrlGenericChecksum(String checksum, String algorithm) {
81+
82+
this.checksum = checksum;
83+
this.algorithm = algorithm;
84+
}
85+
86+
@Override
87+
public String getChecksum() {
88+
89+
return this.checksum;
90+
}
91+
92+
@Override
93+
public String getHashAlgorithm() {
94+
95+
return this.algorithm;
96+
}
97+
98+
@Override
99+
public String toString() {
100+
101+
return this.checksum;
102+
}
103+
}
104+
}

‎cli/src/test/resources/ide-projects/mvn/_ide/urls/mvn/mvn/3.9.7/urls.sha256‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
e3aac9e671f88f0e14dcf8240a5301778fd335e14b8752d15fa2f28b1a51ab32

0 commit comments

Comments
 (0)