Skip to content

Commit 317e19f

Browse files
RogerstalkerBrutus5000
Rogerstalker
authored and
Brutus5000
committed
Improved qUncompress performance
- Eliminated unnecessary array copying by reusing the inputBuffer for inflater - Using the uncompressed length stored in the binary file to allocate appropriately sized inflater output array. Improved readReplayData performance - Eliminated unnecessary array copying - Reusing ByteBuffer for UTF8 decode Improved decompress performance by Reusing ByteBuffer for Base64 decode
1 parent 59eafa1 commit 317e19f

5 files changed

Lines changed: 67 additions & 62 deletions

File tree

‎data/src/main/java/com/faforever/commons/replay/QtCompress.java‎

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,8 @@
33
import java.io.ByteArrayOutputStream;
44
import java.io.DataOutputStream;
55
import java.io.IOException;
6-
import java.util.Arrays;
7-
import java.util.zip.Deflater;
8-
import java.util.zip.DeflaterOutputStream;
9-
import java.util.zip.Inflater;
10-
import java.util.zip.InflaterOutputStream;
6+
import java.nio.ByteBuffer;
7+
import java.util.zip.*;
118

129
/**
1310
* Utility class that compresses and uncompresses bytes like QT's <a href="http://doc.qt.io/qt-5/qbytearray.html">QByteArray</a>.
@@ -22,17 +19,19 @@ private QtCompress() {
2219
* Compresses the specified bytes like <a href="http://doc.qt.io/qt-5/qbytearray.html#qCompress">QByteArray.qCompress()</a>
2320
* does.
2421
*/
25-
public static byte[] qUncompress(byte[] bytes) throws IOException {
22+
public static ByteBuffer qUncompress(ByteBuffer inputBuffer) {
2623
Inflater inflater = new Inflater();
27-
inflater.setInput(Arrays.copyOfRange(bytes, 4, bytes.length));
28-
29-
ByteArrayOutputStream byteArray = new ByteArrayOutputStream();
30-
31-
try (InflaterOutputStream inflaterOutputStream = new InflaterOutputStream(byteArray, inflater)) {
32-
inflaterOutputStream.flush();
24+
final int uncompressedLength = inputBuffer.getInt();
25+
inflater.setInput(inputBuffer);
26+
final var outputBytes = new byte[uncompressedLength];
27+
28+
try {
29+
inflater.inflate(outputBytes);
30+
} catch (DataFormatException e) {
31+
throw new RuntimeException(e);
3332
}
3433

35-
return byteArray.toByteArray();
34+
return ByteBuffer.wrap(outputBytes);
3635
}
3736

3837
/**

‎data/src/main/java/com/faforever/commons/replay/ReplayDataParser.java‎

Lines changed: 24 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
import com.faforever.commons.replay.shared.LuaData;
99
import com.fasterxml.jackson.databind.ObjectMapper;
1010
import com.google.common.annotations.VisibleForTesting;
11-
import com.google.common.io.BaseEncoding;
1211
import lombok.Getter;
1312
import lombok.extern.slf4j.Slf4j;
1413
import org.apache.commons.compress.compressors.CompressorException;
@@ -46,7 +45,7 @@ public class ReplayDataParser {
4645
@Getter
4746
private String replayPatchFieldId;
4847
@Getter
49-
private byte[] data;
48+
private ByteBuffer data;
5049
@Getter
5150
private String map;
5251
@Getter
@@ -129,34 +128,42 @@ private int peek(ByteBuffer buffer) {
129128
private void readReplayData(Path replayFile) throws IOException, CompressorException {
130129
byte[] allReplayData = Files.readAllBytes(replayFile);
131130
int headerEnd = findReplayHeaderEnd(allReplayData);
132-
metadata = objectMapper.readValue(new String(Arrays.copyOf(allReplayData, headerEnd), StandardCharsets.UTF_8), ReplayMetadata.class);
133-
data = decompress(Arrays.copyOfRange(allReplayData, headerEnd + 1, allReplayData.length), metadata);
131+
ByteBuffer buffer = ByteBuffer.wrap(allReplayData);
132+
133+
buffer.limit(headerEnd);
134+
final String decodedMetadata = StandardCharsets.UTF_8.newDecoder().decode(buffer).toString();
135+
metadata = objectMapper.readValue(decodedMetadata, ReplayMetadata.class);
136+
buffer.limit(buffer.capacity());
137+
138+
buffer.position(headerEnd + 1);
139+
data = decompress(buffer, metadata);
134140
}
135141

136142
private int findReplayHeaderEnd(byte[] replayData) {
137-
int headerEnd;
138-
for (headerEnd = 0; headerEnd < replayData.length; headerEnd++) {
143+
for (int headerEnd = 0; headerEnd < replayData.length; headerEnd++) {
139144
if (replayData[headerEnd] == '\n') {
140145
return headerEnd;
141146
}
142147
}
143148
throw new IllegalArgumentException("Missing separator between replay header and body");
144149
}
145150

146-
private byte[] decompress(byte[] data, @NotNull ReplayMetadata metadata) throws IOException, CompressorException {
151+
private ByteBuffer decompress(ByteBuffer inputBuffer, @NotNull ReplayMetadata metadata) throws IOException, CompressorException {
147152
CompressionType compressionType = Objects.requireNonNullElse(metadata.getCompression(), CompressionType.QTCOMPRESS);
148153

149154
switch (compressionType) {
150155
case QTCOMPRESS: {
151-
return QtCompress.qUncompress(BaseEncoding.base64().decode(new String(data)));
156+
return QtCompress.qUncompress(Base64.getDecoder().decode(inputBuffer));
152157
}
153158
case ZSTD: {
154-
ByteArrayInputStream arrayInputStream = new ByteArrayInputStream(data);
159+
byte[] inputArray = new byte[inputBuffer.remaining()];
160+
inputBuffer.get(inputArray);
161+
ByteArrayInputStream arrayInputStream = new ByteArrayInputStream(inputArray);
155162
CompressorInputStream compressorInputStream = new CompressorStreamFactory().createCompressorInputStream(arrayInputStream);
156163

157164
ByteArrayOutputStream out = new ByteArrayOutputStream();
158165
IOUtils.copy(compressorInputStream, out);
159-
return out.toByteArray();
166+
return ByteBuffer.wrap(out.toByteArray());
160167
}
161168
case UNKNOWN:
162169
default:
@@ -249,7 +256,7 @@ private void interpretEvents(List<Event> events) {
249256
previousTick = ticks;
250257

251258
if (desync) {
252-
log.warn("Replay desynced");
259+
// log.warn("Replay desynced");
253260
return;
254261
}
255262
}
@@ -435,17 +442,15 @@ private Duration tickToTime(int tick) {
435442

436443
private void parse() throws IOException, CompressorException {
437444
readReplayData(path);
445+
data.order(ByteOrder.LITTLE_ENDIAN);
438446

439-
final ByteBuffer buffer = ByteBuffer.wrap(data);
440-
buffer.order(ByteOrder.LITTLE_ENDIAN);
441-
442-
parseHeader(buffer);
447+
parseHeader(data);
443448

444-
var rewindPosition = buffer.position();
445-
tokens = ReplayBodyTokenizer.tokenize(buffer);
446-
buffer.position(rewindPosition);
449+
var rewindPosition = data.position();
450+
tokens = ReplayBodyTokenizer.tokenize(data);
451+
data.position(rewindPosition);
447452

448-
events = ReplayBodyParser.parseTokens(tokens, buffer);
453+
events = ReplayBodyParser.parseTokens(tokens, data);
449454
interpretEvents(events);
450455
}
451456
}

‎data/src/main/java/com/faforever/commons/replay/ReplayLoader.java‎

Lines changed: 25 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,10 @@
99
import com.faforever.commons.replay.header.Source;
1010
import com.fasterxml.jackson.databind.DeserializationFeature;
1111
import com.fasterxml.jackson.databind.ObjectMapper;
12-
import com.google.common.io.BaseEncoding;
1312
import org.apache.commons.compress.compressors.CompressorException;
1413
import org.apache.commons.compress.compressors.CompressorInputStream;
1514
import org.apache.commons.compress.compressors.CompressorStreamFactory;
16-
import org.apache.commons.compress.utils.IOUtils;
15+
import org.apache.commons.io.IOUtils;
1716
import org.jetbrains.annotations.Contract;
1817
import org.jetbrains.annotations.NotNull;
1918

@@ -26,7 +25,7 @@
2625
import java.nio.charset.StandardCharsets;
2726
import java.nio.file.Files;
2827
import java.nio.file.Path;
29-
import java.util.Arrays;
28+
import java.util.Base64;
3029
import java.util.List;
3130
import java.util.Objects;
3231

@@ -48,14 +47,13 @@ private static ReplayHeader loadSCFAReplayHeader(ByteBuffer buffer) {
4847
}
4948

5049
@Contract(pure = true)
51-
private static ReplayContainer loadSCFAReplayFromMemory(ReplayMetadata metadata, byte[] scfaReplayBytes) throws IOException {
52-
final ByteBuffer buffer = ByteBuffer.wrap(scfaReplayBytes);
53-
buffer.order(ByteOrder.LITTLE_ENDIAN);
50+
private static ReplayContainer loadSCFAReplayFromMemory(ReplayMetadata metadata, ByteBuffer scfaReplayBuffer) throws IOException {
51+
scfaReplayBuffer.order(ByteOrder.LITTLE_ENDIAN);
5452

55-
ReplayHeader replayHeader = loadSCFAReplayHeader(buffer);
56-
List<RegisteredEvent> replayBody = loadSCFAReplayBody(replayHeader.sources(), buffer);
53+
ReplayHeader replayHeader = loadSCFAReplayHeader(scfaReplayBuffer);
54+
List<RegisteredEvent> replayBody = loadSCFAReplayBody(replayHeader.sources(), scfaReplayBuffer);
5755

58-
if (buffer.position() != buffer.limit()) {
56+
if (scfaReplayBuffer.position() != scfaReplayBuffer.limit()) {
5957
throw new EOFException();
6058
}
6159

@@ -68,22 +66,25 @@ public static ReplayContainer loadSCFAReplayFromDisk(Path scfaReplayFile) throws
6866
}
6967

7068
byte[] bytes = Files.readAllBytes(scfaReplayFile);
71-
return loadSCFAReplayFromMemory(null, bytes);
69+
return loadSCFAReplayFromMemory(null, ByteBuffer.wrap(bytes));
7270
}
7371

7472
@Contract(pure = true)
7573
private static ReplayContainer loadFAFReplayFromMemory(byte[] fafReplayBytes) throws IOException, CompressorException {
7674
int separator = findSeparatorIndex(fafReplayBytes);
77-
byte[] metadataBytes = Arrays.copyOfRange(fafReplayBytes, 0, separator);
78-
String metadataString = new String(metadataBytes, StandardCharsets.UTF_8);
75+
ByteBuffer buffer = ByteBuffer.wrap(fafReplayBytes);
76+
77+
buffer.limit(separator);
78+
final String decodedMetadata = StandardCharsets.UTF_8.newDecoder().decode(buffer).toString();
7979

8080
ObjectMapper parsedMetadata = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
81-
ReplayMetadata replayMetadata = parsedMetadata.readValue(metadataString, ReplayMetadata.class);
81+
ReplayMetadata replayMetadata = parsedMetadata.readValue(decodedMetadata, ReplayMetadata.class);
82+
buffer.limit(buffer.capacity());
8283

83-
byte[] compressedReplayBytes = Arrays.copyOfRange(fafReplayBytes, separator + 1, fafReplayBytes.length);
84-
byte[] scfaReplayBytes = decompress(compressedReplayBytes, replayMetadata);
84+
buffer.position(separator + 1);
85+
ByteBuffer scfaReplayBuffer= decompress(buffer, replayMetadata);
8586

86-
return loadSCFAReplayFromMemory(replayMetadata, scfaReplayBytes);
87+
return loadSCFAReplayFromMemory(replayMetadata, scfaReplayBuffer);
8788
}
8889

8990
public static ReplayContainer loadFAFReplayFromDisk(Path fafReplayFile) throws IOException, CompressorException, IllegalArgumentException {
@@ -96,30 +97,30 @@ public static ReplayContainer loadFAFReplayFromDisk(Path fafReplayFile) throws I
9697
}
9798

9899
private static int findSeparatorIndex(byte[] replayData) {
99-
int headerEnd;
100-
for (headerEnd = 0; headerEnd < replayData.length; headerEnd++) {
100+
for (int headerEnd = 0; headerEnd < replayData.length; headerEnd++) {
101101
if (replayData[headerEnd] == '\n') {
102102
return headerEnd;
103103
}
104104
}
105105
throw new IllegalArgumentException("Missing separator between replay header and body");
106106
}
107107

108-
private static byte[] decompress(byte[] data, @NotNull ReplayMetadata metadata) throws IOException, CompressorException {
108+
private static ByteBuffer decompress(ByteBuffer inputBuffer, @NotNull ReplayMetadata metadata) throws IOException, CompressorException {
109109
CompressionType compressionType = Objects.requireNonNullElse(metadata.getCompression(), CompressionType.QTCOMPRESS);
110110

111111
switch (compressionType) {
112112
case QTCOMPRESS: {
113-
return QtCompress.qUncompress(BaseEncoding.base64().decode(new String(data)));
113+
return QtCompress.qUncompress(Base64.getDecoder().decode(inputBuffer));
114114
}
115115
case ZSTD: {
116-
ByteArrayInputStream arrayInputStream = new ByteArrayInputStream(data);
117-
CompressorInputStream compressorInputStream = new CompressorStreamFactory()
118-
.createCompressorInputStream(arrayInputStream);
116+
byte[] inputArray = new byte[inputBuffer.remaining()];
117+
inputBuffer.get(inputArray);
118+
ByteArrayInputStream arrayInputStream = new ByteArrayInputStream(inputArray);
119+
CompressorInputStream compressorInputStream = new CompressorStreamFactory().createCompressorInputStream(arrayInputStream);
119120

120121
ByteArrayOutputStream out = new ByteArrayOutputStream();
121122
IOUtils.copy(compressorInputStream, out);
122-
return out.toByteArray();
123+
return ByteBuffer.wrap(out.toByteArray());
123124
}
124125
case UNKNOWN:
125126
default:

‎data/src/test/java/com/faforever/commons/replay/QtCompressTest.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import org.junit.jupiter.api.Test;
44

5+
import java.nio.ByteBuffer;
56
import java.nio.charset.StandardCharsets;
67

78
import static com.faforever.commons.test.IsUtilityClassMatcher.isUtilityClass;
@@ -30,8 +31,8 @@ void testQCompress() throws Exception {
3031
}
3132

3233
@Test
33-
void testQUncompress() throws Exception {
34-
byte[] uncompressedBytes = QtCompress.qUncompress(COMPRESSED_BYTES);
35-
assertArrayEquals(UNCOMPRESSED_BYTES, uncompressedBytes);
34+
void testQUncompress() {
35+
ByteBuffer uncompressedByteBuffer = QtCompress.qUncompress(ByteBuffer.wrap(COMPRESSED_BYTES));
36+
assertArrayEquals(UNCOMPRESSED_BYTES, uncompressedByteBuffer.array());
3637
}
3738
}

‎data/src/test/java/com/faforever/commons/replay/ReplayLoaderTableParserTest.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
import java.util.Arrays;
1919
import java.util.List;
2020
import java.util.Objects;
21-
2221
import static org.hamcrest.CoreMatchers.is;
2322
import static org.hamcrest.MatcherAssert.assertThat;
2423
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -109,7 +108,7 @@ public void testReferenceZstd() throws Exception {
109108
Files.copy(getClass().getResourceAsStream("/replay/zstd_reference.fafreplay"), replayFile);
110109
Files.copy(getClass().getResourceAsStream("/replay/zstd_reference.raw"), referenceFile);
111110

112-
byte[] data = new ReplayDataParser(replayFile, objectMapper).getData();
111+
byte[] data = new ReplayDataParser(replayFile, objectMapper).getData().array();
113112
byte[] reference = Files.readAllBytes(referenceFile);
114113
assertThat("Zstd compressed replay matches reference", Arrays.equals(data, reference));
115114
}
@@ -121,7 +120,7 @@ public void testLegacyFormat() throws Exception {
121120
Files.copy(getClass().getResourceAsStream("/replay/test.fafreplay"), replayFile);
122121
Files.copy(getClass().getResourceAsStream("/replay/test.raw"), referenceFile);
123122

124-
byte[] data = new ReplayDataParser(replayFile, objectMapper).getData();
123+
byte[] data = new ReplayDataParser(replayFile, objectMapper).getData().array();
125124
byte[] reference = Files.readAllBytes(referenceFile);
126125
assertThat("Legacy compressed file matches reference", Arrays.equals(data, reference));
127126
}

0 commit comments

Comments
 (0)