Skip to content

Commit 772dfcd

Browse files
oschwaldoschwald
committed
Optimize Decoder for mmap/byte-buffer
1 parent d294eb6 commit 772dfcd

4 files changed

Lines changed: 77 additions & 122 deletions

File tree

‎src/main/java/com/maxmind/maxminddb/Decoder.java‎

Lines changed: 63 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import java.io.IOException;
44
import java.math.BigInteger;
55
import java.nio.ByteBuffer;
6-
import java.nio.channels.FileChannel;
76
import java.nio.charset.Charset;
87

98
import com.fasterxml.jackson.databind.JsonNode;
@@ -19,7 +18,6 @@
1918
import com.fasterxml.jackson.databind.node.TextNode;
2019

2120
public class Decoder {
22-
private final FileChannel in;
2321

2422
private final boolean DEBUG;
2523
// XXX - This is only for unit testings. We should possibly make a
@@ -29,14 +27,15 @@ public class Decoder {
2927

3028
private final ObjectMapper objectMapper;
3129

30+
private final ByteBuffer mmap;
31+
3232
class Result {
3333
private final JsonNode node;
3434
private long offset;
3535

3636
Result(JsonNode node, long offset) {
3737
this.node = node;
3838
this.offset = offset;
39-
4039
}
4140

4241
JsonNode getNode() {
@@ -53,22 +52,22 @@ void setOffset(long offset) {
5352

5453
}
5554

56-
public Decoder(FileChannel in, long pointerBase) {
57-
this.in = in;
55+
public Decoder(ByteBuffer mmap, long pointerBase) {
5856
this.pointerBase = pointerBase;
57+
this.mmap = mmap;
5958
this.objectMapper = new ObjectMapper();
6059
this.DEBUG = System.getenv().get("MAXMIND_DB_DECODER_DEBUG") != null;
6160
}
6261

6362
public Result decode(long offset) throws MaxMindDbException, IOException {
64-
this.in.position(offset);
63+
this.mmap.position((int) offset);
64+
6565
if (this.DEBUG) {
6666
Log.debug("Offset", String.valueOf(offset));
6767
}
6868

69-
ByteBuffer buffer = ByteBuffer.wrap(new byte[1]);
70-
this.in.read(buffer);
71-
int ctrlByte = 0xFF & buffer.get(0);
69+
int ctrlByte = 0xFF & this.mmap.get();
70+
7271
offset++;
7372

7473
if (this.DEBUG) {
@@ -96,9 +95,7 @@ public Result decode(long offset) throws MaxMindDbException, IOException {
9695
}
9796

9897
if (type.equals(Type.EXTENDED)) {
99-
buffer = ByteBuffer.wrap(new byte[1]);
100-
this.in.read(buffer);
101-
int nextByte = buffer.get(0);
98+
int nextByte = this.mmap.get();
10299

103100
if (this.DEBUG) {
104101
Log.debug("Next byte", nextByte);
@@ -147,35 +144,31 @@ public Result decode(long offset) throws MaxMindDbException, IOException {
147144
Log.debug("Size", size);
148145
}
149146

150-
buffer = ByteBuffer.wrap(new byte[size]);
151-
this.in.read(buffer);
152-
byte[] bytes = buffer.array();
153-
154147
long new_offset = offset + size;
155148
switch (type) {
156149
case UTF8_STRING:
157-
TextNode s = new TextNode(Decoder.decodeString(bytes));
150+
TextNode s = new TextNode(this.decodeString(size));
158151
return new Result(s, new_offset);
159152
case DOUBLE:
160-
DoubleNode d = Decoder.decodeDouble(bytes);
153+
DoubleNode d = this.decodeDouble(size);
161154
return new Result(d, new_offset);
162155
case BYTES:
163-
BinaryNode b = new BinaryNode(Decoder.decodeBytes(bytes));
156+
BinaryNode b = new BinaryNode(this.getByteArray(size));
164157
return new Result(b, new_offset);
165158
case UINT16:
166-
IntNode i = Decoder.decodeUint16(bytes);
159+
IntNode i = this.decodeUint16(size);
167160
return new Result(i, new_offset);
168161
case UINT32:
169-
LongNode l = Decoder.decodeUint32(bytes);
162+
LongNode l = this.decodeUint32(size);
170163
return new Result(l, new_offset);
171164
case INT32:
172-
IntNode int32 = Decoder.decodeInt32(bytes);
165+
IntNode int32 = this.decodeInt32(size);
173166
return new Result(int32, new_offset);
174167
case UINT64:
175-
BigIntegerNode bi = Decoder.decodeUint64(bytes);
168+
BigIntegerNode bi = this.decodeBigInteger(size);
176169
return new Result(bi, new_offset);
177170
case UINT128:
178-
BigIntegerNode uint128 = Decoder.decodeUint128(bytes);
171+
BigIntegerNode uint128 = this.decodeBigInteger(size);
179172
return new Result(uint128, new_offset);
180173
default:
181174
throw new MaxMindDbException("Unknown or unexpected type: "
@@ -187,25 +180,17 @@ public Result decode(long offset) throws MaxMindDbException, IOException {
187180
private final long[] pointerValueOffset = { 0, 0, 1 << 11,
188181
(((long) 1) << 19) + ((1) << 11), 0 };
189182

190-
private Result decodePointer(int ctrlByte, long offset) throws IOException {
183+
private Result decodePointer(int ctrlByte, long offset) {
191184

192185
int pointerSize = ((ctrlByte >>> 3) & 0x3) + 1;
193186

194187
if (this.DEBUG) {
195188
Log.debug("Pointer size", String.valueOf(pointerSize));
196189
}
197190

198-
ByteBuffer buffer = ByteBuffer.wrap(new byte[pointerSize + 1]);
199-
200-
this.in.read(buffer, this.in.position() - 1);
201-
202-
if (this.DEBUG) {
203-
Log.debug("Buffer", buffer);
204-
}
205-
206-
buffer.put(0, pointerSize == 4 ? (byte) 0 : (byte) (ctrlByte & 0x7));
191+
int base = pointerSize == 4 ? (byte) 0 : (byte) (ctrlByte & 0x7);
207192

208-
long packed = Util.decodeLong(buffer.array());
193+
long packed = this.decodeLong(base, pointerSize);
209194

210195
if (this.DEBUG) {
211196
Log.debug("Packed pointer", String.valueOf(packed));
@@ -224,36 +209,55 @@ private Result decodePointer(int ctrlByte, long offset) throws IOException {
224209
return new Result(new LongNode(pointer), offset + pointerSize);
225210
}
226211

227-
private static String decodeString(byte[] bytes) {
228-
return new String(bytes, Charset.forName("UTF-8"));
212+
private String decodeString(int size) {
213+
ByteBuffer buffer = this.mmap.slice();
214+
buffer.limit(size);
215+
return Charset.forName("UTF-8").decode(buffer).toString();
229216
}
230217

231-
// XXX - nop
232-
private static byte[] decodeBytes(byte[] bytes) {
233-
return bytes;
218+
IntNode decodeUint16(int size) {
219+
return new IntNode(this.decodeInteger(size));
234220
}
235221

236-
static IntNode decodeUint16(byte[] bytes) {
237-
return new IntNode(Util.decodeInteger(bytes));
222+
IntNode decodeInt32(int size) {
223+
return new IntNode(this.decodeInteger(size));
238224
}
239225

240-
static IntNode decodeInt32(byte[] bytes) {
241-
return new IntNode(Util.decodeInteger(bytes));
226+
private int decodeInteger(int size) {
227+
return this.decodeInteger(0, size);
242228
}
243229

244-
static LongNode decodeUint32(byte[] bytes) {
245-
return new LongNode(Util.decodeLong(bytes));
230+
private int decodeInteger(int base, int size) {
231+
int integer = base;
232+
for (int i = 0; i < size; i++) {
233+
integer = (integer << 8) | (this.mmap.get() & 0xFF);
234+
}
235+
return integer;
246236
}
247237

248-
static BigIntegerNode decodeUint64(byte[] bytes) {
249-
return new BigIntegerNode(new BigInteger(1, bytes));
238+
LongNode decodeUint32(int size) {
239+
return new LongNode(this.decodeLong(size));
240+
}
241+
242+
long decodeLong(int size) {
243+
return this.decodeLong(0, size);
250244
}
251245

252-
static BigIntegerNode decodeUint128(byte[] bytes) {
246+
long decodeLong(long base, int size) {
247+
long longInt = base;
248+
for (int i = 0; i < size; i++) {
249+
longInt = (longInt << 8) | (this.mmap.get() & 0xFF);
250+
}
251+
return longInt;
252+
}
253+
254+
BigIntegerNode decodeBigInteger(int size) {
255+
byte[] bytes = this.getByteArray(size);
253256
return new BigIntegerNode(new BigInteger(1, bytes));
254257
}
255258

256-
private static DoubleNode decodeDouble(byte[] bytes) {
259+
private DoubleNode decodeDouble(int size) {
260+
byte[] bytes = this.getByteArray(size);
257261
return new DoubleNode(Double.parseDouble(new String(bytes, Charset
258262
.forName("US-ASCII"))));
259263
}
@@ -331,22 +335,24 @@ private long[] sizeFromCtrlByte(int ctrlByte, long offset)
331335

332336
int bytesToRead = size - 28;
333337

334-
ByteBuffer buffer = ByteBuffer.wrap(new byte[bytesToRead]);
335-
this.in.read(buffer);
336-
337338
if (size == 29) {
338-
int i = Util.decodeInteger(buffer.array());
339+
int i = this.decodeInteger(bytesToRead);
339340
size = 29 + i;
340341
} else if (size == 30) {
341-
int i = Util.decodeInteger(buffer.array());
342+
int i = this.decodeInteger(bytesToRead);
342343
size = 285 + i;
343344
} else {
344-
buffer.put(0, (byte) (buffer.get(0) & 0x0F));
345-
int i = Util.decodeInteger(buffer.array());
345+
int i = this.decodeInteger(bytesToRead)
346+
& (0x0FFFFFFF >>> (32 - (8 * bytesToRead)));
346347
size = 65821 + i;
347348
}
348349

349350
return new long[] { size, offset + bytesToRead };
350351
}
351352

353+
private byte[] getByteArray(int length) {
354+
byte[] bytes = new byte[length];
355+
this.mmap.get(bytes);
356+
return bytes;
357+
}
352358
}

‎src/main/java/com/maxmind/maxminddb/Reader.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@
55
import java.io.RandomAccessFile;
66
import java.net.InetAddress;
77
import java.nio.ByteBuffer;
8+
import java.nio.MappedByteBuffer;
89
import java.nio.channels.FileChannel;
10+
import java.nio.channels.FileChannel.MapMode;
911
import java.util.Arrays;
1012

1113
import com.fasterxml.jackson.databind.JsonNode;
@@ -21,14 +23,13 @@ public class Reader {
2123
private final Metadata metadata;
2224
private final FileChannel fc;
2325
private final RandomAccessFile raf;
26+
private final MappedByteBuffer mmap;
2427

2528
public Reader(File database) throws MaxMindDbException, IOException {
2629
this.DEBUG = System.getenv().get("MAXMIND_DB_READER_DEBUG") != null;
2730
this.raf = new RandomAccessFile(database, "r");
2831
this.fc = this.raf.getChannel();
29-
// XXX - we will want
30-
// MappedByteBuffer in = fc.map(FileChannel.MapMode.READ_ONLY, 0,
31-
// fc.size());
32+
this.mmap = this.fc.map(MapMode.READ_ONLY, 0, this.fc.size());
3233

3334
/*
3435
* We need to make sure that whatever chunk we read will have the
@@ -53,12 +54,11 @@ public Reader(File database) throws MaxMindDbException, IOException {
5354
+ "). Is this a valid MaxMind DB file?");
5455
}
5556

56-
Decoder metadataDecoder = new Decoder(this.fc, 0);
57+
Decoder metadataDecoder = new Decoder(this.mmap, 0);
5758

5859
this.metadata = new Metadata(metadataDecoder.decode(start).getNode());
5960

60-
this.decoder = new Decoder(this.fc, this.metadata.searchTreeSize
61-
+ DATA_SECTION_SEPARATOR_SIZE);
61+
this.decoder = new Decoder(this.mmap, this.metadata.searchTreeSize + DATA_SECTION_SEPARATOR_SIZE);
6262

6363
if (this.DEBUG) {
6464
Log.debug(this.metadata.toString());

‎src/test/java/com/maxmind/maxminddb/DecoderTest.java‎

Lines changed: 4 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,10 @@
88
import java.io.RandomAccessFile;
99
import java.math.BigInteger;
1010
import java.nio.ByteBuffer;
11+
import java.nio.MappedByteBuffer;
1112
import java.nio.channels.FileChannel;
13+
import java.nio.channels.FileChannel.MapMode;
1214
import java.nio.charset.Charset;
13-
import java.util.Arrays;
1415
import java.util.HashMap;
1516
import java.util.Map;
1617
import java.util.UUID;
@@ -385,9 +386,10 @@ static <T> void testTypeDecoding(Type type, Map<T, byte[]> tests)
385386

386387
String desc = "decoded " + type.name() + " - " + expect;
387388
FileChannel fc = DecoderTest.getFileChannel(input);
389+
MappedByteBuffer mmap = fc.map(MapMode.READ_ONLY, 0, fc.size());
388390
try {
389391

390-
Decoder decoder = new Decoder(fc, 0);
392+
Decoder decoder = new Decoder(mmap, 0);
391393
decoder.POINTER_TEST_HACK = true;
392394

393395
// XXX - this could be streamlined
@@ -427,62 +429,6 @@ static <T> void testTypeDecoding(Type type, Map<T, byte[]> tests)
427429
}
428430
}
429431

430-
@Test
431-
public void testDecodeInt32() {
432-
Map<Integer, byte[]> map = int32();
433-
for (Map.Entry<Integer, byte[]> entry : map.entrySet()) {
434-
byte[] bytes = Arrays.copyOfRange(entry.getValue(), 2,
435-
entry.getValue().length);
436-
assertEquals("decode int32: " + entry.getKey(), entry.getKey()
437-
.intValue(), Decoder.decodeInt32(bytes).intValue());
438-
}
439-
}
440-
441-
@Test
442-
public void testDecodeUint16() {
443-
Map<Integer, byte[]> map = uint16();
444-
for (Map.Entry<Integer, byte[]> entry : map.entrySet()) {
445-
byte[] bytes = Arrays.copyOfRange(entry.getValue(), 1,
446-
entry.getValue().length);
447-
assertEquals("decode uint16: " + entry.getKey(), entry.getKey()
448-
.intValue(), Decoder.decodeUint16(bytes).intValue());
449-
}
450-
}
451-
452-
// FIXME - these tests should be combined using generics
453-
@Test
454-
public void testDecodeUint32() {
455-
Map<Long, byte[]> map = uint32();
456-
for (Map.Entry<Long, byte[]> entry : map.entrySet()) {
457-
byte[] bytes = Arrays.copyOfRange(entry.getValue(), 1,
458-
entry.getValue().length);
459-
assertEquals("decode uint32: " + entry.getKey(), entry.getKey()
460-
.longValue(), Decoder.decodeUint32(bytes).longValue());
461-
}
462-
}
463-
464-
@Test
465-
public void testDecodeUint64() {
466-
Map<BigInteger, byte[]> map = largeUint(64);
467-
for (Map.Entry<BigInteger, byte[]> entry : map.entrySet()) {
468-
byte[] bytes = Arrays.copyOfRange(entry.getValue(), 2,
469-
entry.getValue().length);
470-
assertEquals("decode uint64: " + entry.getKey(), entry.getKey(),
471-
Decoder.decodeUint64(bytes).bigIntegerValue());
472-
}
473-
}
474-
475-
@Test
476-
public void testDecodeUint128() {
477-
Map<BigInteger, byte[]> map = largeUint(128);
478-
for (Map.Entry<BigInteger, byte[]> entry : map.entrySet()) {
479-
byte[] bytes = Arrays.copyOfRange(entry.getValue(), 2,
480-
entry.getValue().length);
481-
assertEquals("decode uint128: " + entry.getKey(), entry.getKey(),
482-
Decoder.decodeUint128(bytes).bigIntegerValue());
483-
}
484-
}
485-
486432
/*
487433
* I really didn't want to create temporary files for these tests, but it is
488434
* pretty hard to abstract away from the file io system in a way that is

‎src/test/java/com/maxmind/maxminddb/PointerTest.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@
55
import java.io.File;
66
import java.io.IOException;
77
import java.io.RandomAccessFile;
8+
import java.nio.MappedByteBuffer;
89
import java.nio.channels.FileChannel;
10+
import java.nio.channels.FileChannel.MapMode;
911

1012
import org.junit.Test;
1113

@@ -19,8 +21,9 @@ public void testWithPointers() throws MaxMindDbException, IOException {
1921
File file = new File("test-data/pointer.bin");
2022
RandomAccessFile raf = new RandomAccessFile(file, "r");
2123
FileChannel fc = raf.getChannel();
24+
MappedByteBuffer mmap = fc.map(MapMode.READ_ONLY, 0, fc.size());
2225
try {
23-
Decoder decoder = new Decoder(fc, 0);
26+
Decoder decoder = new Decoder(mmap, 0);
2427

2528
ObjectMapper om = new ObjectMapper();
2629

0 commit comments

Comments
 (0)