Skip to content

Commit dd4a117

Browse files
adamretteradamretter
committed
[optimize] Reuse the CRC32 checksum object when compressing many files in the Compression Module
1 parent f103e8b commit dd4a117

1 file changed

Lines changed: 23 additions & 16 deletions

File tree

‎extensions/modules/compression/src/main/java/org/exist/xquery/modules/compression/AbstractCompressFunction.java‎

Lines changed: 23 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -157,14 +157,16 @@ public Sequence eval(final Sequence[] args, final Sequence contextSequence) thro
157157
final OutputStream os = stream(baos, encoding);
158158
final StringBuilderWriter sbWriter = new StringBuilderWriter()) {
159159

160+
final CRC32 chksum = new CRC32();
161+
160162
// iterate through the argument sequence
161163
for (final SequenceIterator i = args[0].iterate(); i.hasNext(); ) {
162164
final Item item = i.nextItem();
163165

164166
if (item instanceof Element element) {
165-
compressElement(os, element, useHierarchy, stripOffset, sbWriter);
167+
compressElement(os, element, useHierarchy, stripOffset, sbWriter, chksum);
166168
} else {
167-
compressFromUri(os, ((AnyURIValue) item).toURI(), useHierarchy, stripOffset, ZipMethod.DEFLATE, null, sbWriter);
169+
compressFromUri(os, ((AnyURIValue) item).toURI(), useHierarchy, stripOffset, ZipMethod.DEFLATE, null, sbWriter, chksum);
168170
}
169171
}
170172

@@ -181,7 +183,7 @@ public Sequence eval(final Sequence[] args, final Sequence contextSequence) thro
181183
}
182184
}
183185

184-
private void compressFromUri(final OutputStream os, final URI uri, final boolean useHierarchy, final String stripOffset, final ZipMethod method, final String resourceName, final StringBuilderWriter sbWriter) throws XPathException
186+
private void compressFromUri(final OutputStream os, final URI uri, final boolean useHierarchy, final String stripOffset, final ZipMethod method, final String resourceName, final StringBuilderWriter sbWriter, final CRC32 chksum) throws XPathException
185187
{
186188
try {
187189
if ("file".equals(uri.getScheme())) {
@@ -194,7 +196,7 @@ private void compressFromUri(final OutputStream os, final URI uri, final boolean
194196

195197
// got a file
196198
final Path file = Paths.get(uri.getPath());
197-
compressFile(os, file, useHierarchy, stripOffset, method, resourceName);
199+
compressFile(os, file, useHierarchy, stripOffset, method, resourceName, chksum);
198200

199201
} else {
200202

@@ -203,7 +205,7 @@ private void compressFromUri(final OutputStream os, final URI uri, final boolean
203205
// try for a collection
204206
try (final Collection collection = context.getBroker().openCollection(xmldburi, LockMode.READ_LOCK)) {
205207
if(collection != null) {
206-
compressCollection(os, collection, useHierarchy, stripOffset, method, sbWriter);
208+
compressCollection(os, collection, useHierarchy, stripOffset, method, sbWriter, chksum);
207209
return;
208210
}
209211
} catch (final PermissionDeniedException | LockException | SAXException | IOException pde) {
@@ -226,7 +228,7 @@ private void compressFromUri(final OutputStream os, final URI uri, final boolean
226228
throw new XPathException(this, "Invalid URI: " + uri.toString());
227229
}
228230

229-
compressResource(os, doc.getDocument(), useHierarchy, stripOffset, method, resourceName, sbWriter);
231+
compressResource(os, doc.getDocument(), useHierarchy, stripOffset, method, resourceName, sbWriter, chksum);
230232
return;
231233
}
232234
} catch (final PermissionDeniedException | LockException | SAXException | IOException pde) {
@@ -249,8 +251,9 @@ private void compressFromUri(final OutputStream os, final URI uri, final boolean
249251
* @param stripOffset a string that should be stripped from the start of the entry name.
250252
* @param method the Zip method.
251253
* @param name the name of the entry.
254+
* @param chksum an object that is used to calculate the checksum.
252255
*/
253-
private void compressFile(final OutputStream os, final Path file, final boolean useHierarchy, final String stripOffset, final ZipMethod method, final String name) throws IOException {
256+
private void compressFile(final OutputStream os, final Path file, final boolean useHierarchy, final String stripOffset, final ZipMethod method, final String name, final CRC32 chksum) throws IOException {
254257

255258
if (!Files.isDirectory(file)) {
256259

@@ -269,6 +272,7 @@ private void compressFile(final OutputStream os, final Path file, final boolean
269272
// close the entry
270273
if (entry instanceof ZipEntry && method == ZipMethod.STORE) {
271274
((ZipEntry) entry).setMethod(ZipOutputStream.STORED);
275+
chksum.reset();
272276
chksum.update(value);
273277
((ZipEntry) entry).setCrc(chksum.getValue());
274278
((ZipEntry) entry).setSize(value.length);
@@ -281,7 +285,7 @@ private void compressFile(final OutputStream os, final Path file, final boolean
281285
} else {
282286

283287
for (final Path child : FileUtils.list(file)) {
284-
compressFile(os, file.resolve(child), useHierarchy, stripOffset, method, null);
288+
compressFile(os, file.resolve(child), useHierarchy, stripOffset, method, null, chksum);
285289
}
286290

287291
}
@@ -296,9 +300,10 @@ private void compressFile(final OutputStream os, final Path file, final boolean
296300
* @param useHierarchy Whether to use a folder hierarchy in the archive file that reflects the collection hierarchy.
297301
* @param stripOffset a string that should be stripped from the start of the entry name.
298302
* @param sbWriter a StringBuilderWriter to reuse
303+
* @param chksum an object that is used to calculate the checksum.
299304
*/
300305
private void compressElement(final OutputStream os, final Element element, final boolean useHierarchy,
301-
final String stripOffset, final StringBuilderWriter sbWriter) throws XPathException {
306+
final String stripOffset, final StringBuilderWriter sbWriter, final CRC32 chksum) throws XPathException {
302307

303308
final String ns = element.getNamespaceURI();
304309
if (!(element.getNodeName().equals("entry") || (ns != null && !ns.isEmpty()))) {
@@ -326,7 +331,7 @@ private void compressElement(final OutputStream os, final Element element, final
326331
if (isNullOrEmpty(uri)) {
327332
throw new XPathException(this, "Entry with type uri must contain a URI.");
328333
}
329-
compressFromUri(os, URI.create(uri), useHierarchy, stripOffset, method, name, sbWriter);
334+
compressFromUri(os, URI.create(uri), useHierarchy, stripOffset, method, name, sbWriter, chksum);
330335
return;
331336
}
332337

@@ -346,7 +351,6 @@ private void compressElement(final OutputStream os, final Element element, final
346351

347352
if (!"collection".equals(type)) {
348353
final byte[] value;
349-
final CRC32 chksum = new CRC32();
350354
final Node content = element.getFirstChild();
351355

352356
if (content == null) {
@@ -379,6 +383,7 @@ private void compressElement(final OutputStream os, final Element element, final
379383

380384
if (entry instanceof ZipEntry && method == ZipMethod.STORE) {
381385
((ZipEntry) entry).setMethod(ZipOutputStream.STORED);
386+
chksum.reset();
382387
chksum.update(value);
383388
((ZipEntry) entry).setCrc(chksum.getValue());
384389
((ZipEntry) entry).setSize(value.length);
@@ -422,8 +427,9 @@ private void getDynamicSerializerOptions(final Serializer serializer) throws SAX
422427
* @param method the Zip method.
423428
* @param name the name of the entry.
424429
* @param sbWriter a StringBuilderWriter to reuse
430+
* @param chksum an object that is used to calculate the checksum.
425431
*/
426-
private void compressResource(final OutputStream os, final DocumentImpl doc, final boolean useHierarchy, final String stripOffset, final ZipMethod method, final String name, final StringBuilderWriter sbWriter) throws IOException, SAXException {
432+
private void compressResource(final OutputStream os, final DocumentImpl doc, final boolean useHierarchy, final String stripOffset, final ZipMethod method, final String name, final StringBuilderWriter sbWriter, final CRC32 chksum) throws IOException, SAXException {
427433
// create an entry in the Tar for the document
428434
final Object entry;
429435
if (name != null) {
@@ -463,9 +469,9 @@ private void compressResource(final OutputStream os, final DocumentImpl doc, fin
463469
}
464470

465471
// close the entry
466-
final CRC32 chksum = new CRC32();
467472
if (entry instanceof ZipEntry && method == ZipMethod.STORE) {
468473
((ZipEntry) entry).setMethod(ZipOutputStream.STORED);
474+
chksum.reset();
469475
chksum.update(value);
470476
((ZipEntry) entry).setCrc(chksum.getValue());
471477
((ZipEntry) entry).setSize(value.length);
@@ -485,8 +491,9 @@ private void compressResource(final OutputStream os, final DocumentImpl doc, fin
485491
* @param stripOffset a string that should be stripped from the start of the entry name.
486492
* @param method the Zip method.
487493
* @param sbWriter a StringBuilderWriter to reuse
494+
* @param chksum an object that is used to calculate the checksum.
488495
*/
489-
private void compressCollection(final OutputStream os, final Collection col, final boolean useHierarchy, final String stripOffset, final ZipMethod method, final StringBuilderWriter sbWriter) throws IOException, SAXException, LockException, PermissionDeniedException {
496+
private void compressCollection(final OutputStream os, final Collection col, final boolean useHierarchy, final String stripOffset, final ZipMethod method, final StringBuilderWriter sbWriter, final CRC32 chksum) throws IOException, SAXException, LockException, PermissionDeniedException {
490497
// iterate over child documents
491498
final DBBroker broker = context.getBroker();
492499
final LockManager lockManager = broker.getBrokerPool().getLockManager();
@@ -495,7 +502,7 @@ private void compressCollection(final OutputStream os, final Collection col, fin
495502
for (final Iterator<DocumentImpl> itChildDocs = childDocs.getDocumentIterator(); itChildDocs.hasNext();) {
496503
final DocumentImpl childDoc = itChildDocs.next();
497504
try (final ManagedDocumentLock updateLock = lockManager.acquireDocumentReadLock(childDoc.getURI())) {
498-
compressResource(os, childDoc, useHierarchy, stripOffset, method, null, sbWriter);
505+
compressResource(os, childDoc, useHierarchy, stripOffset, method, null, sbWriter, chksum);
499506
}
500507
}
501508
// iterate over child collections
@@ -504,7 +511,7 @@ private void compressCollection(final OutputStream os, final Collection col, fin
504511
final XmldbURI childColURI = itChildCols.next();
505512
final Collection childCol = broker.getCollection(col.getURI().append(childColURI));
506513
// recurse
507-
compressCollection(os, childCol, useHierarchy, stripOffset, method, sbWriter);
514+
compressCollection(os, childCol, useHierarchy, stripOffset, method, sbWriter, chksum);
508515
}
509516
}
510517

0 commit comments

Comments
 (0)