Skip to content

Commit 9d70b37

Browse files
LatvianModderLatvianModder
committed
Added FileMD5, better Instant codecs, Hex64, FileInfo
1 parent bebb833 commit 9d70b37

11 files changed

Lines changed: 334 additions & 45 deletions

File tree

‎src/main/java/dev/latvian/mods/klib/codec/KLibCodecs.java‎

Lines changed: 3 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
import net.minecraft.util.StringRepresentable;
1414

1515
import java.time.Instant;
16-
import java.time.format.DateTimeFormatter;
1716
import java.util.ArrayList;
1817
import java.util.Arrays;
1918
import java.util.HashMap;
@@ -96,7 +95,7 @@ static <T> Codec<List<T>> listOrSelf(Codec<T> elementCodec) {
9695
}
9796
});
9897

99-
Codec<Instant> INSTANT = Codec.STRING.flatXmap(string -> {
98+
Codec<Instant> ISO_INSTANT = Codec.STRING.flatXmap(string -> {
10099
try {
101100
return DataResult.success(Instant.parse(string));
102101
} catch (Exception ex) {
@@ -110,20 +109,6 @@ static <T> Codec<List<T>> listOrSelf(Codec<T> elementCodec) {
110109
}
111110
});
112111

113-
Codec<Instant> ISO_INSTANT = Codec.STRING.flatXmap(string -> {
114-
try {
115-
return DataResult.success(Instant.from(DateTimeFormatter.ISO_INSTANT.parse(string)));
116-
} catch (Exception ex) {
117-
return DataResult.error(() -> "Invalid ISO date: " + string);
118-
}
119-
}, instant -> {
120-
try {
121-
return DataResult.success(DateTimeFormatter.ISO_INSTANT.format(instant));
122-
} catch (Exception ex) {
123-
return DataResult.error(() -> "Invalid ISO date: " + instant);
124-
}
125-
});
126-
127112
Codec<Instant> UINT64_INSTANT = Codec.LONG.flatXmap(number -> {
128113
try {
129114
return DataResult.success(Instant.ofEpochMilli(number));
@@ -138,6 +123,8 @@ static <T> Codec<List<T>> listOrSelf(Codec<T> elementCodec) {
138123
}
139124
});
140125

126+
Codec<Instant> INSTANT = KLibCodecs.or(ISO_INSTANT, UINT64_INSTANT);
127+
141128
static <E> Codec<E> anyEnumCodec(E[] enumValues, Function<E, String> nameGetter) {
142129
var map = new HashMap<String, E>(enumValues.length);
143130

‎src/main/java/dev/latvian/mods/klib/codec/KLibStreamCodecs.java‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
import io.netty.buffer.ByteBuf;
66
import net.minecraft.core.Registry;
77
import net.minecraft.network.RegistryFriendlyByteBuf;
8+
import net.minecraft.network.VarInt;
9+
import net.minecraft.network.VarLong;
810
import net.minecraft.network.codec.ByteBufCodecs;
911
import net.minecraft.network.codec.StreamCodec;
1012
import net.minecraft.resources.ResourceKey;
@@ -143,7 +145,20 @@ public void encode(RegistryFriendlyByteBuf buf, String value) {
143145
}
144146
};
145147

146-
StreamCodec<ByteBuf, Instant> INSTANT = ByteBufCodecs.LONG.map(Instant::ofEpochMilli, Instant::toEpochMilli);
148+
StreamCodec<ByteBuf, Instant> INSTANT = new StreamCodec<>() {
149+
@Override
150+
public Instant decode(ByteBuf buf) {
151+
var second = VarLong.read(buf);
152+
var nano = VarInt.read(buf);
153+
return Instant.ofEpochSecond(second, nano);
154+
}
155+
156+
@Override
157+
public void encode(ByteBuf buf, Instant value) {
158+
VarLong.write(buf, value.getEpochSecond());
159+
VarInt.write(buf, value.getNano());
160+
}
161+
};
147162

148163
static <T> StreamCodec<ByteBuf, ResourceKey<T>> resourceKey(ResourceKey<? extends Registry<T>> registry) {
149164
return ResourceLocation.STREAM_CODEC.map(id -> ResourceKey.create(registry, id), ResourceKey::location);

‎src/main/java/dev/latvian/mods/klib/data/DataTypes.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import dev.latvian.mods.klib.shape.Shape;
2727
import dev.latvian.mods.klib.util.Cast;
2828
import dev.latvian.mods.klib.util.Hex32;
29+
import dev.latvian.mods.klib.util.Hex64;
2930
import dev.latvian.mods.klib.util.ID;
3031
import dev.latvian.mods.klib.util.IntOrUUID;
3132
import dev.latvian.mods.klib.util.MD5;
@@ -84,9 +85,9 @@ public interface DataTypes {
8485
DataType<String> STRING = DataType.of(Codec.STRING, ByteBufCodecs.STRING_UTF8, String.class);
8586
DataType<UUID> UUID = DataType.of(KLibCodecs.UUID, KLibStreamCodecs.UUID, UUID.class);
8687
DataType<byte[]> B64_BYTE_ARRAY = DataType.of(KLibCodecs.B64_BYTE_ARRAY, ByteBufCodecs.BYTE_ARRAY, byte[].class);
87-
DataType<Instant> INSTANT = DataType.of(KLibCodecs.INSTANT, KLibStreamCodecs.INSTANT, Instant.class);
8888
DataType<Instant> ISO_INSTANT = DataType.of(KLibCodecs.ISO_INSTANT, KLibStreamCodecs.INSTANT, Instant.class);
8989
DataType<Instant> UINT64_INSTANT = DataType.of(KLibCodecs.UINT64_INSTANT, KLibStreamCodecs.INSTANT, Instant.class);
90+
DataType<Instant> INSTANT = DataType.of(KLibCodecs.INSTANT, KLibStreamCodecs.INSTANT, Instant.class);
9091

9192
DataType<Component> TEXT_COMPONENT = DataType.of(ComponentSerialization.CODEC, ComponentSerialization.STREAM_CODEC, Component.class);
9293
DataType<Mirror> MIRROR = DataType.of(Mirror.values());
@@ -118,9 +119,9 @@ static void register() {
118119
DataType.register(ID.java("string"), STRING, StringArgumentType::string, StringArgumentType::getString);
119120
DataType.register(ID.java("uuid"), UUID, UuidArgument::uuid, UuidArgument::getUuid);
120121
DataType.register(ID.java("b64_byte_array"), B64_BYTE_ARRAY);
121-
DataType.register(ID.java("instant"), INSTANT);
122122
DataType.register(ID.java("iso_instant"), ISO_INSTANT);
123123
DataType.register(ID.java("uint64_instant"), UINT64_INSTANT);
124+
DataType.register(ID.java("instant"), INSTANT);
124125

125126
DataType.register(ID.mc("id"), ID.DATA_TYPE, ResourceLocationArgument::id, ResourceLocationArgument::getId);
126127
DataType.register(ID.mc("text_component"), TEXT_COMPONENT, ComponentArgument::textComponent, ComponentArgument::getResolvedComponent);
@@ -168,6 +169,7 @@ static void register() {
168169
DataType.register(KLibMod.id("timestamp"), Timestamp.DATA_TYPE);
169170
DataType.register(KLibMod.id("md5"), MD5.DATA_TYPE);
170171
DataType.register(KLibMod.id("hex32"), Hex32.DATA_TYPE);
172+
DataType.register(KLibMod.id("hex64"), Hex64.DATA_TYPE);
171173
DataType.register(KLibMod.id("uint64"), UInt64.DATA_TYPE);
172174
}
173175
}

‎src/main/java/dev/latvian/mods/klib/io/FileInfo.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package dev.latvian.mods.klib.io;
2+
3+
import java.nio.file.Path;
4+
5+
public record FileInfo(Path path, String name, long size) {
6+
public FileInfo(Path file) {
7+
this(file, file.getFileName().toString(), IOUtils.getSize(file));
8+
}
9+
10+
public FileInfo(Path root, Path file) {
11+
this(file, root.relativize(file).toString().replace('\\', '/'), IOUtils.getSize(file));
12+
}
13+
}

‎src/main/java/dev/latvian/mods/klib/io/FileInfoFilter.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
package dev.latvian.mods.klib.io;
2+
3+
@FunctionalInterface
4+
public interface FileInfoFilter {
5+
boolean test(FileInfo fileInfo) throws Exception;
6+
}

‎src/main/java/dev/latvian/mods/klib/io/FileMD5.java‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
package dev.latvian.mods.klib.io;
2+
3+
import com.mojang.serialization.Codec;
4+
import com.mojang.serialization.MapCodec;
5+
import com.mojang.serialization.codecs.RecordCodecBuilder;
6+
import dev.latvian.mods.klib.codec.KLibCodecs;
7+
import dev.latvian.mods.klib.util.MD5;
8+
import org.jetbrains.annotations.Nullable;
9+
10+
import java.io.ByteArrayInputStream;
11+
import java.io.ByteArrayOutputStream;
12+
import java.io.DataInput;
13+
import java.io.DataInputStream;
14+
import java.io.DataOutput;
15+
import java.io.DataOutputStream;
16+
import java.io.IOException;
17+
import java.nio.file.Files;
18+
import java.nio.file.Path;
19+
import java.nio.file.attribute.FileTime;
20+
import java.time.Instant;
21+
import java.util.function.LongConsumer;
22+
23+
public record FileMD5(MD5 checksum, long size, Instant lastModified, boolean changed) {
24+
public static final Codec<FileMD5> CODEC = RecordCodecBuilder.create(instance -> instance.group(
25+
MD5.CODEC.fieldOf("md5").forGetter(FileMD5::checksum),
26+
Codec.LONG.fieldOf("size").forGetter(FileMD5::size),
27+
KLibCodecs.INSTANT.fieldOf("last_modified").forGetter(FileMD5::lastModified),
28+
MapCodec.unit(false).forGetter(FileMD5::changed)
29+
).apply(instance, FileMD5::new));
30+
31+
public static FileMD5 read(DataInput data) throws IOException {
32+
var checksum = MD5.read(data);
33+
var size = IOUtils.readVarLong(data);
34+
var lastModified = IOUtils.readExactTime(data);
35+
return new FileMD5(checksum, size, lastModified, false);
36+
}
37+
38+
@Nullable
39+
public static FileMD5 loadExisting(FileInfo fileInfo) {
40+
try {
41+
var attribute = IOUtils.getAttributeBytes(fileInfo.path(), "latviandev-file-md5");
42+
43+
if (attribute != null) {
44+
try (var data = new DataInputStream(new ByteArrayInputStream(attribute))) {
45+
data.readUnsignedByte(); // Binary marker
46+
return read(data);
47+
}
48+
}
49+
} catch (Exception ignored) {
50+
}
51+
52+
return null;
53+
}
54+
55+
public static FileMD5 load(FileInfo fileInfo, @Nullable LongConsumer progress) throws IOException {
56+
var existing = loadExisting(fileInfo);
57+
var lastModified = IOUtils.getLastModifiedTime(fileInfo.path());
58+
59+
if (existing == null || fileInfo.size() != existing.size || lastModified == null || lastModified.isAfter(existing.lastModified)) {
60+
var md5 = MD5.of(fileInfo, progress);
61+
62+
return new FileMD5(
63+
md5,
64+
fileInfo.size(),
65+
lastModified,
66+
true
67+
);
68+
} else if (progress != null) {
69+
progress.accept(fileInfo.size());
70+
}
71+
72+
return existing;
73+
}
74+
75+
public static void save(Path file, FileMD5 metadata) throws IOException {
76+
try (var bytes = new ByteArrayOutputStream();
77+
var data = new DataOutputStream(bytes)
78+
) {
79+
data.writeByte(0);
80+
metadata.write(data);
81+
IOUtils.setAttributeBytes(file, "latviandev-file-md5", bytes.toByteArray());
82+
}
83+
}
84+
85+
@Nullable
86+
public static FileMD5 loadChanged(FileInfo fileInfo, @Nullable LongConsumer progress) throws IOException {
87+
var meta = load(fileInfo, progress);
88+
89+
if (meta.changed()) {
90+
save(fileInfo.path(), meta);
91+
Files.setLastModifiedTime(fileInfo.path(), FileTime.from(meta.lastModified()));
92+
return meta;
93+
}
94+
95+
return null;
96+
}
97+
98+
public void write(DataOutput data) throws IOException {
99+
checksum.write(data);
100+
IOUtils.writeVarLong(data, size);
101+
IOUtils.writeExactTime(data, lastModified);
102+
}
103+
}

‎src/main/java/dev/latvian/mods/klib/io/IOUtils.java‎

Lines changed: 66 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
package dev.latvian.mods.klib.io;
22

3-
import dev.latvian.mods.klib.util.StringUtils;
43
import io.netty.buffer.ByteBuf;
54
import org.jetbrains.annotations.Nullable;
65

@@ -22,6 +21,7 @@
2221
import java.util.Comparator;
2322
import java.util.EnumSet;
2423
import java.util.Set;
24+
import java.util.function.LongConsumer;
2525
import java.util.function.Predicate;
2626

2727
public interface IOUtils {
@@ -130,6 +130,12 @@ static String readUTF(DataInput in) throws IOException {
130130
return new String(readBytes(in), StandardCharsets.UTF_8);
131131
}
132132

133+
static Instant readExactTime(DataInput in) throws IOException {
134+
var second = readVarLong(in);
135+
var nano = readVarInt(in);
136+
return Instant.ofEpochSecond(second, nano);
137+
}
138+
133139
static void writeVarInt(DataOutput out, int value) throws IOException {
134140
while ((value & -128) != 0) {
135141
out.writeByte(value & 127 | 128);
@@ -157,6 +163,11 @@ static void writeUTF(DataOutput out, String value) throws IOException {
157163
writeBytes(out, value.getBytes(StandardCharsets.UTF_8));
158164
}
159165

166+
static void writeExactTime(DataOutput out, Instant value) throws IOException {
167+
writeVarLong(out, value.getEpochSecond());
168+
writeVarInt(out, value.getNano());
169+
}
170+
160171
static byte[] toByteArray(ByteBuf buf, boolean release) {
161172
var bytes = new byte[buf.readableBytes()];
162173
buf.getBytes(buf.readerIndex(), bytes);
@@ -204,53 +215,90 @@ static void appendBytes(Path path, ByteBuffer buf, long remainingBytes) throws I
204215
}
205216
}
206217

207-
static byte[] digest(String algorithm, Path file) throws NoSuchAlgorithmException, IOException {
208-
var md = MessageDigest.getInstance(algorithm);
218+
static ByteBuffer allocateTempBuffer(int maxBufferSize, long fileSize) {
219+
return ByteBuffer.allocate(Math.min(maxBufferSize, (int) Math.min(Integer.MAX_VALUE, fileSize)));
220+
}
221+
222+
static ByteBuffer allocateTempBuffer(Path file) throws IOException {
223+
return allocateTempBuffer(4096, Files.size(file));
224+
}
225+
226+
static MessageDigest md(String algorithm) {
227+
try {
228+
return MessageDigest.getInstance(algorithm);
229+
} catch (NoSuchAlgorithmException ex) {
230+
throw new RuntimeException(ex);
231+
}
232+
}
233+
234+
static MessageDigest md5() {
235+
return md("MD5");
236+
}
237+
238+
static byte[] digest(String algorithm, Path file, long size, @Nullable LongConsumer callback) throws IOException {
239+
var md = md(algorithm);
209240

210241
try (var channel = Files.newByteChannel(file)) {
211-
var buf = ByteBuffer.allocate(2048);
242+
var buf = allocateTempBuffer(4096, size);
243+
int len;
212244

213-
while (channel.read(buf) != -1) {
245+
while ((len = channel.read(buf)) != -1) {
214246
buf.flip();
215247
md.update(buf);
216248
buf.clear();
249+
250+
if (callback != null) {
251+
callback.accept(len);
252+
}
217253
}
218254

219255
return md.digest();
220256
}
221257
}
222258

223-
static String md5bytes(Path file) throws NoSuchAlgorithmException, IOException {
224-
return StringUtils.toHex(digest("MD5", file));
225-
}
226-
227-
static String md5(Path file) throws NoSuchAlgorithmException, IOException {
228-
return StringUtils.toHex(digest("MD5", file));
229-
}
230-
231-
static String getAttribute(Path file, String attribute) throws IOException {
259+
@Nullable
260+
static ByteBuffer getAttributeBuffer(Path file, String attribute) throws IOException {
232261
var attributes = Files.getFileAttributeView(file, UserDefinedFileAttributeView.class);
233262

234263
if (attributes != null && attributes.list().contains(attribute)) {
235264
var attributeBuffer = ByteBuffer.allocate(attributes.size(attribute));
236265
attributes.read(attribute, attributeBuffer);
237266
attributeBuffer.flip();
238-
return Charset.defaultCharset().decode(attributeBuffer).toString();
267+
return attributeBuffer;
239268
}
240269

241-
return "";
270+
return null;
242271
}
243272

244-
static boolean setAttribute(Path file, String attribute, String value) throws IOException {
273+
static String getAttribute(Path file, String attribute) throws IOException {
274+
var buffer = getAttributeBuffer(file, attribute);
275+
return buffer == null ? "" : Charset.defaultCharset().decode(buffer).toString();
276+
}
277+
278+
@Nullable
279+
static byte[] getAttributeBytes(Path file, String attribute) throws IOException {
280+
var buffer = getAttributeBuffer(file, attribute);
281+
return buffer == null ? null : buffer.array();
282+
}
283+
284+
static boolean setAttributeBuffer(Path file, String attribute, ByteBuffer value) throws IOException {
245285
var attributes = Files.getFileAttributeView(file, UserDefinedFileAttributeView.class);
246286

247287
if (attributes != null) {
248-
return attributes.write(attribute, Charset.defaultCharset().encode(value)) > 0;
288+
return attributes.write(attribute, value) > 0;
249289
}
250290

251291
return false;
252292
}
253293

294+
static boolean setAttribute(Path file, String attribute, String value) throws IOException {
295+
return setAttributeBuffer(file, attribute, Charset.defaultCharset().encode(value));
296+
}
297+
298+
static boolean setAttributeBytes(Path file, String attribute, byte[] value) throws IOException {
299+
return setAttributeBuffer(file, attribute, ByteBuffer.wrap(value));
300+
}
301+
254302
static Predicate<Path> pathEndsWith(String suffix) {
255303
return path -> path.toString().endsWith(suffix);
256304
}

0 commit comments

Comments
 (0)