Skip to content

Commit b316697

Browse files
LexManosLexManos
committed
Add 'fast' timestamp based File entries to HashStore
1 parent 9a0dce6 commit b316697

2 files changed

Lines changed: 241 additions & 2 deletions

File tree

‎hash-utils/src/main/java/net/minecraftforge/util/hash/HashStore.java‎

Lines changed: 49 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import java.io.IOException;
1212
import java.nio.charset.StandardCharsets;
1313
import java.nio.file.Files;
14+
import java.nio.file.attribute.FileTime;
1415
import java.util.ArrayList;
1516
import java.util.HashMap;
1617
import java.util.Objects;
@@ -25,6 +26,7 @@ public class HashStore {
2526
private final HashMap<String, String> newHashes = new HashMap<>();
2627
private @Nullable File target;
2728
private boolean saved;
29+
private boolean timestamps = false;
2830

2931
public static HashStore fromFile(File path) {
3032
File parent = path.getAbsoluteFile().getParentFile();
@@ -49,6 +51,25 @@ private HashStore(String root) {
4951
this.root = root;
5052
}
5153

54+
/**
55+
* Use file size and last-modified timestamps instead of hashing the contents of files.
56+
* Note: This stores the file size and last modified time, which has a resolution of 1ms on most operating systems.
57+
* So it IS possible for this to not properly detect changes if the file is modified, but the timestamp is kept the same.
58+
*/
59+
public HashStore timestamps() {
60+
return timestamps(true);
61+
}
62+
63+
/**
64+
* Sets wither to use file size and last-modified timestamps or hashing the contents of files.
65+
* Note: This stores the file size and last modified time, which has a resolution of 1ms on most operating systems.
66+
* So it IS possible for this to not properly detect changes if the file is modified, but the timestamp is kept the same.
67+
*/
68+
public HashStore timestamps(boolean value) {
69+
this.timestamps = value;
70+
return this;
71+
}
72+
5273
public boolean areSame(File... files) {
5374
for (File file : files) {
5475
if (!isSame(file))
@@ -141,17 +162,28 @@ public HashStore add(@Nullable String key, File file) {
141162
String prefix = getPath(file);
142163
for (File f : listFiles(file)) {
143164
String suffix = getPath(f).substring(prefix.length());
144-
this.newHashes.put(key + " - " + suffix, HASH.hash(f));
165+
addFile(key + " - " + suffix, f);
145166
}
146167
} else {
147-
this.newHashes.put(key, HASH.hash(file));
168+
addFile(key, file);
148169
}
149170
} catch (IOException e) {
150171
throw new RuntimeException(e);
151172
}
152173
return this;
153174
}
154175

176+
private void addFile(String key, File file) throws IOException {
177+
String value;
178+
if (this.timestamps) {
179+
// Use this over file.lastModified() so we get IOExceptions if the file doesn't exist
180+
FileTime lastModified = Files.getLastModifiedTime(file.toPath());
181+
value = file.length() + "-" + lastModified.toMillis();
182+
} else
183+
value = HASH.hash(file);
184+
this.newHashes.put(key, value);
185+
}
186+
155187
public HashStore add(File... files) {
156188
for (File file : files)
157189
add(null, file);
@@ -179,6 +211,21 @@ public HashStore clear() {
179211
return this;
180212
}
181213

214+
/** Clears the old hashes loaded from {@link #load(File) load}, {@link #fromFile(File) fromFile}, or {@link #fromDir(File) fromDir} */
215+
public HashStore invalidate() {
216+
return invalidate(true);
217+
}
218+
219+
/**
220+
* Conditionally Clears the old hashes loaded from {@link #load(File) load}, {@link #fromFile(File) fromFile}, or {@link #fromDir(File) fromDir}.
221+
* This is meant as a builder type helper
222+
*/
223+
public HashStore invalidate(boolean value) {
224+
if (value)
225+
this.oldHashes.clear();
226+
return this;
227+
}
228+
182229
public void save() {
183230
if (this.target == null)
184231
throw new RuntimeException("HashStore.save() called without load(File) so we dont know where to save it! Use load(File) or save(File)");

‎hash-utils/src/test/java/net/minecraftforge/util/hash/HashStoreTest.java‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
/*
2+
* Copyright (c) Forge Development LLC
3+
* SPDX-License-Identifier: LGPL-2.1-only
4+
*/
5+
package net.minecraftforge.util.hash;
6+
7+
import java.io.File;
8+
import java.io.IOException;
9+
import java.nio.file.Files;
10+
import org.junit.jupiter.api.Assertions;
11+
import org.junit.jupiter.api.Test;
12+
13+
public class HashStoreTest {
14+
private static final byte[] SHORT_EMPTY = new byte[0x10];
15+
private static final byte[] BULK_EMPTY = new byte[0x1000];
16+
private static final byte[] BULK_DATA = new byte[0x1000];
17+
static {
18+
for (int x = 0; x < BULK_DATA.length; x++)
19+
BULK_DATA[x] = (byte)(x & 0xFF);
20+
}
21+
22+
private static void isSame(HashStore cache, File file) throws IOException {
23+
if (cache.isSame())
24+
return;
25+
System.out.println("Original: " + read(file));
26+
cache.save();
27+
System.out.println("Modified: " + read(file));
28+
Assertions.assertTrue(cache.isSame());
29+
}
30+
31+
private static void isNotSame(HashStore cache, File file) throws IOException {
32+
if (!cache.isSame())
33+
return;
34+
System.out.println("Original: " + read(file));
35+
cache.save();
36+
System.out.println("Modified: " + read(file));
37+
Assertions.assertFalse(cache.isSame());
38+
}
39+
40+
private static String read(File file) throws IOException {
41+
if (file.isDirectory())
42+
file = new File(file, ".cache");
43+
else
44+
file = new File(file.getAbsolutePath() + ".cache");
45+
46+
return String.join("\n", Files.readAllLines(file.toPath()));
47+
}
48+
49+
@Test
50+
public void invalidate() throws IOException {
51+
File file = File.createTempFile("hash-store-test", ".dat");
52+
// Save a cache file
53+
HashStore.fromFile(file)
54+
.add("key", "value")
55+
.save();
56+
57+
HashStore cache = HashStore.fromFile(file)
58+
.invalidate()
59+
.add("key", "value");
60+
61+
isNotSame(cache, file);
62+
}
63+
64+
@Test
65+
public void loadsFile() throws IOException {
66+
File file = File.createTempFile("hash-store-test", ".dat");
67+
// Save a cache file
68+
HashStore.fromFile(file)
69+
.add("key", "value")
70+
.save();
71+
72+
HashStore cache = HashStore.fromFile(file)
73+
.add("key", "value");
74+
75+
isSame(cache, file);
76+
}
77+
78+
@Test
79+
public void fileEntry() throws IOException {
80+
File file = File.createTempFile("hash-store-test", ".dat");
81+
Files.write(file.toPath(), BULK_DATA);
82+
83+
// Save a cache file
84+
HashStore.fromFile(file)
85+
.add(file)
86+
.save();
87+
88+
HashStore cache = HashStore.fromFile(file)
89+
.add(file);
90+
91+
isSame(cache, file);
92+
}
93+
94+
@Test
95+
public void fileEntryTimestamp() throws IOException {
96+
File file = File.createTempFile("hash-store-test", ".dat");
97+
Files.write(file.toPath(), BULK_DATA);
98+
99+
// Save a cache file
100+
HashStore.fromFile(file)
101+
.timestamps()
102+
.add(file)
103+
.save();
104+
105+
HashStore cache = HashStore.fromFile(file)
106+
.timestamps()
107+
.add(file);
108+
109+
isSame(cache, file);
110+
}
111+
112+
@Test
113+
public void detectsChange() throws IOException {
114+
File file = File.createTempFile("hash-store-test", ".dat");
115+
Files.write(file.toPath(), BULK_DATA);
116+
117+
// Save a cache file
118+
HashStore.fromFile(file)
119+
.add(file)
120+
.save();
121+
122+
Files.write(file.toPath(), BULK_EMPTY);
123+
124+
HashStore cache = HashStore.fromFile(file)
125+
.add(file);
126+
127+
isNotSame(cache, file);
128+
}
129+
130+
@Test
131+
public void detectsChangeTimestamp() throws IOException, InterruptedException {
132+
File file = File.createTempFile("hash-store-test", ".dat");
133+
Files.write(file.toPath(), BULK_DATA);
134+
135+
// Save a cache file
136+
HashStore.fromFile(file)
137+
.timestamps()
138+
.add(file)
139+
.save();
140+
141+
// Race condition here, we can write both files in less then 1ms which means the modified time wont change
142+
// There is no way to actually detect this using the 'timestamp' only case. This will have to be a 'known issue'
143+
// It shouldn't actually show up in the real world as I can't think of a case where there isnt *some* other processing going on tha would take more then 1ms
144+
// So only real option is to sleep for at least 1ms
145+
Thread.sleep(5);
146+
Files.write(file.toPath(), BULK_EMPTY);
147+
148+
HashStore cache = HashStore.fromFile(file)
149+
.timestamps()
150+
.add(file);
151+
152+
isNotSame(cache, file);
153+
}
154+
155+
@Test
156+
public void detectsSizeChange() throws IOException {
157+
File file = File.createTempFile("hash-store-test", ".dat");
158+
Files.write(file.toPath(), BULK_DATA);
159+
160+
// Save a cache file
161+
HashStore.fromFile(file)
162+
.add(file)
163+
.save();
164+
165+
Files.write(file.toPath(), SHORT_EMPTY);
166+
167+
HashStore cache = HashStore.fromFile(file)
168+
.add(file);
169+
170+
isNotSame(cache, file);
171+
}
172+
173+
@Test
174+
public void detectsSizeChangeTimestamp() throws IOException, InterruptedException {
175+
File file = File.createTempFile("hash-store-test", ".dat");
176+
Files.write(file.toPath(), BULK_DATA);
177+
178+
// Save a cache file
179+
HashStore.fromFile(file)
180+
.timestamps()
181+
.add(file)
182+
.save();
183+
184+
Files.write(file.toPath(), SHORT_EMPTY);
185+
186+
HashStore cache = HashStore.fromFile(file)
187+
.timestamps()
188+
.add(file);
189+
190+
isNotSame(cache, file);
191+
}
192+
}

0 commit comments

Comments
 (0)