Skip to content

Commit 1702cff

Browse files
adamretteradamretter
committed
[optimize] Dispose eagerly of resources that were accumulated during XQuery exeution but are no longer needed in the Result sequence
1 parent 6a09999 commit 1702cff

10 files changed

Lines changed: 530 additions & 84 deletions

File tree

‎exist-core/pom.xml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1112,6 +1112,7 @@
11121112
<include>src/test/java/org/exist/util/io/ByteArrayContentTest.java</include>
11131113
<include>src/main/java/org/exist/util/io/ContentFile.java</include>
11141114
<include>src/main/java/org/exist/util/io/ContentFilePoolObjectFactory.java</include>
1115+
<include>src/main/java/org/exist/util/io/FilterInputStreamCacheMonitor.java</include>
11151116
<include>src/test/java/org/exist/util/io/FilterInputStreamCacheMonitorTest.java</include>
11161117
<include>src/test/java/org/exist/util/io/OverflowToDiskStreamTest.java</include>
11171118
<include>src/main/java/org/exist/util/io/VirtualTempPath.java</include>
@@ -1926,6 +1927,7 @@
19261927
<exclude>src/test/java/org/exist/util/io/ByteArrayContentTest.java</exclude>
19271928
<exclude>src/main/java/org/exist/util/io/ContentFile.java</exclude>
19281929
<exclude>src/main/java/org/exist/util/io/ContentFilePoolObjectFactory.java</exclude>
1930+
<exclude>src/main/java/org/exist/util/io/FilterInputStreamCacheMonitor.java</exclude>
19291931
<exclude>src/test/java/org/exist/util/io/FilterInputStreamCacheMonitorTest.java</exclude>
19301932
<exclude>src/test/java/org/exist/util/io/OverflowToDiskStreamTest.java</exclude>
19311933
<exclude>src/main/java/org/exist/util/io/VirtualTempPath.java</exclude>

‎exist-core/src/main/java/org/exist/test/ExistXmldbEmbeddedServer.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747

4848
import org.exist.EXistException;
4949
import org.exist.TestUtils;
50+
import org.exist.source.StringSource;
5051
import org.exist.storage.serializers.EXistOutputKeys;
5152
import org.exist.util.DatabaseConfigurationException;
5253
import org.exist.xmldb.EXistCollection;
@@ -199,8 +200,7 @@ private void stopXmlDb() throws XMLDBException {
199200

200201

201202
public ResourceSet executeQuery(final String query) throws XMLDBException {
202-
final CompiledExpression compiledQuery = xpathQueryService.compile(query);
203-
return xpathQueryService.execute(compiledQuery);
203+
return xpathQueryService.execute(new StringSource(query));
204204
}
205205

206206
public ResourceSet executeQuery(final String query, final Map<String, Object> externalVariables)

‎exist-core/src/main/java/org/exist/util/io/FilterInputStreamCacheMonitor.java‎

Lines changed: 24 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,28 @@
11
/*
2+
* Elemental
3+
* Copyright (C) 2024, Evolved Binary Ltd
4+
*
5+
* admin@evolvedbinary.com
6+
* https://www.evolvedbinary.com | https://www.elemental.xyz
7+
*
8+
* This library is free software; you can redistribute it and/or
9+
* modify it under the terms of the GNU Lesser General Public
10+
* License as published by the Free Software Foundation; version 2.1.
11+
*
12+
* This library is distributed in the hope that it will be useful,
13+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
14+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15+
* Lesser General Public License for more details.
16+
*
17+
* You should have received a copy of the GNU Lesser General Public
18+
* License along with this library; if not, write to the Free Software
19+
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20+
*
21+
* NOTE: Parts of this file contain code from 'The eXist-db Authors'.
22+
* The original license header is included below.
23+
*
24+
* =====================================================================
25+
*
226
* eXist-db Open Source Native XML Database
327
* Copyright (C) 2001 The eXist-db Authors
428
*
@@ -19,7 +43,6 @@
1943
* License along with this library; if not, write to the Free Software
2044
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
2145
*/
22-
2346
package org.exist.util.io;
2447

2548
import java.util.*;
@@ -44,15 +67,6 @@ public static FilterInputStreamCacheMonitor getInstance() {
4467
return INSTANCE;
4568
}
4669

47-
/**
48-
* Intentionally package private!
49-
*
50-
* Only for use by org.exist.util.io.FilterInputStreamCacheMonitorTest
51-
*/
52-
void clear() {
53-
activeCaches.clear();
54-
}
55-
5670
public void register(final FilterInputStreamCache cache) {
5771
final long now = System.currentTimeMillis();
5872
final FilterInputStreamCacheInfo info = new FilterInputStreamCacheInfo(now, cache);

‎exist-core/src/main/java/org/exist/xmldb/LocalXMLResource.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@
8484
import org.exist.util.serializer.SerializerPool;
8585
import org.exist.xquery.XPathException;
8686
import org.exist.xquery.value.AtomicValue;
87+
import org.exist.xquery.value.Item;
8788
import org.exist.xquery.value.NodeValue;
8889
import org.exist.xquery.value.StringValue;
8990
import org.exist.xquery.value.Type;
@@ -224,6 +225,24 @@ else if (root != null && !(root instanceof NodeValue)) {
224225
}
225226
}
226227

228+
/**
229+
* Allows access to the underlying XDM typed value.
230+
* Should be used sparingly!
231+
*
232+
* @return the Underlying Xdm value.
233+
*/
234+
public @Nullable Item getContentAsXdm() {
235+
if (root instanceof NodeValue) {
236+
return (NodeValue) root;
237+
}
238+
239+
if (value != null) {
240+
return value;
241+
}
242+
243+
return null;
244+
}
245+
227246
private String serialize(final DBBroker broker, final ConsumerE<Serializer, SAXException> toSaxFunction) throws SAXException, IOException {
228247
final Serializer serializer = broker.borrowSerializer();
229248
SAXSerializer saxSerializer = null;

‎exist-core/src/main/java/org/exist/xmldb/LocalXPathQueryService.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,6 @@
7070
import org.exist.xquery.XQueryContext;
7171
import org.exist.xquery.XQueryUtil;
7272
import org.exist.xquery.value.AnyURIValue;
73-
import org.exist.xquery.value.BinaryValue;
7473
import org.exist.xquery.value.Sequence;
7574
import org.exist.xquery.value.SequenceIterator;
7675
import org.w3c.dom.Node;

‎exist-core/src/main/java/org/exist/xquery/XQuery.java‎

Lines changed: 118 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151

5252
import java.io.*;
5353
import java.text.NumberFormat;
54+
import java.util.Iterator;
5455
import java.util.List;
5556
import java.util.Optional;
5657
import java.util.Properties;
@@ -71,11 +72,16 @@
7172
import org.exist.source.Source;
7273
import org.exist.source.StringSource;
7374
import org.exist.storage.DBBroker;
75+
import org.exist.xquery.functions.array.ArrayType;
76+
import org.exist.xquery.functions.map.MapType;
7477
import org.exist.xquery.parser.XQueryLexer;
7578
import org.exist.xquery.parser.XQueryParser;
7679
import org.exist.xquery.parser.XQueryTreeParser;
7780
import org.exist.xquery.util.ExpressionDumper;
81+
import org.exist.xquery.value.BinaryValue;
82+
import org.exist.xquery.value.Item;
7883
import org.exist.xquery.value.Sequence;
84+
import org.exist.xquery.value.SequenceIterator;
7985

8086
import javax.annotation.Nullable;
8187

@@ -425,6 +431,7 @@ public Sequence execute(final DBBroker broker, final CompiledXQuery expression,
425431
context.getProfiler().traceQueryStart();
426432
broker.getBrokerPool().getProcessMonitor().queryStarted(context.getWatchDog());
427433

434+
@Nullable Sequence result = null;
428435
FunctionCall call = null;
429436
try {
430437

@@ -435,7 +442,6 @@ public Sequence execute(final DBBroker broker, final CompiledXQuery expression,
435442
}
436443
}
437444

438-
final Sequence result;
439445
if (expression instanceof LibraryModuleRoot) {
440446
if (functionCall == null) {
441447
if (expression != null) {
@@ -480,11 +486,13 @@ public Sequence execute(final DBBroker broker, final CompiledXQuery expression,
480486
broker.getBrokerPool().getProcessMonitor().queryCompleted(context.getWatchDog());
481487
expression.reset();
482488

483-
if (call != null) {
489+
if(call != null) {
484490
call.reset();
485491
}
486492

487-
if(resetContext) {
493+
runCleanupTasks(context, result);
494+
495+
if (resetContext) {
488496
context.reset();
489497
}
490498
}
@@ -515,12 +523,6 @@ public Sequence execute(final DBBroker broker, final String expression, final Se
515523
final XQueryContext context = new XQueryContext(broker.getBrokerPool());
516524
final CompiledXQuery compiled = compile(context, expression);
517525
return execute(broker, compiled, contextSequence);
518-
// NOTE(AR) we might consider the below cleanup, but what if a binary value is needed from the result sequence?
519-
// try {
520-
// return execute(broker, compiled, contextSequence);
521-
// } finally {
522-
// context.runCleanupTasks();
523-
// }
524526
}
525527

526528
/**
@@ -543,11 +545,112 @@ public Sequence execute(final DBBroker broker, final File file, final Sequence c
543545
final XQueryContext context = new XQueryContext(broker.getBrokerPool());
544546
final CompiledXQuery compiled = compile(context, new FileSource(file.toPath(), true));
545547
return execute(broker, compiled, contextSequence);
546-
// NOTE(AR) we might consider the below cleanup, but what if a binary value is needed from the result sequence?
547-
// try {
548-
// return execute(broker, compiled, contextSequence);
549-
// } finally {
550-
// context.runCleanupTasks();
551-
// }
548+
}
549+
550+
/**
551+
* Runs cleanup tasks after query execution to free any resources that are no longer needed.
552+
*
553+
* @param xqueryContext the XQuery Context.
554+
* @param queryResult the result Sequence of the Query.
555+
*/
556+
private static void runCleanupTasks(final XQueryContext xqueryContext, @Nullable final Sequence queryResult) {
557+
if (queryResult == null) {
558+
// There are no results produced by the query, so we can cleanup everything
559+
xqueryContext.runCleanupTasks();
560+
561+
} else {
562+
// There are results produced, so we can only cleanup resources that don't appear in the result sequence
563+
cleanupOrphanedValues(xqueryContext, queryResult);
564+
}
565+
}
566+
567+
/**
568+
* Runs cleanup tasks that cleanup items that do not appear in the Query Result.
569+
* At present the only items that may need to be cleaned up are BinaryValue types.
570+
*
571+
* @param xqueryContext the XQuery Context.
572+
* @param queryResult the result Sequence of the Query.
573+
*/
574+
private static void cleanupOrphanedValues(final XQueryContext xqueryContext, final Sequence queryResult) {
575+
xqueryContext.runCleanupTasks(false, false, clazz -> clazz.equals(XQueryContext.BinaryValueCleanupTask.class), obj -> {
576+
if (obj instanceof BinaryValue) {
577+
final boolean hasReference = sequenceHasItemReference(queryResult, obj);
578+
return !hasReference;
579+
}
580+
581+
return true;
582+
});
583+
}
584+
585+
/**
586+
* Checks if a Sequence contains an Item.
587+
* The check is performed by reference equality.
588+
*
589+
* @param sequence the Sequence to inspect.
590+
* @param itemReference The item to look for by reference.
591+
*
592+
* @return true if the sequence contains the item, false otherwise.
593+
*/
594+
private static boolean sequenceHasItemReference(final Sequence sequence, final Object itemReference) {
595+
if (sequence instanceof MapType) {
596+
return itemHasItemReference((MapType) sequence, itemReference);
597+
}
598+
599+
if (sequence instanceof ArrayType) {
600+
return itemHasItemReference((ArrayType) sequence, itemReference);
601+
}
602+
603+
try {
604+
final SequenceIterator itItem = sequence.iterate();
605+
while (itItem.hasNext()) {
606+
final Item item = itItem.nextItem();
607+
if (itemHasItemReference(item, itemReference)) {
608+
return true;
609+
}
610+
}
611+
} catch (final XPathException e) {
612+
// no-op
613+
}
614+
615+
return false;
616+
}
617+
618+
/**
619+
* Checks if an Item matches an Item (by reference equality).
620+
* If the Item is a Map or Array, we also checks its members (recursively).
621+
*
622+
* @param item the Item to inspect.
623+
* @param itemReference The item to look for by reference.
624+
*
625+
* @return true if the Item matches or contains the Item reference, false otherwise.
626+
*/
627+
private static boolean itemHasItemReference(final Item item, final Object itemReference) {
628+
if (item == itemReference) {
629+
return true;
630+
}
631+
632+
if (item instanceof MapType) {
633+
final MapType mapType = (MapType) item;
634+
final Iterator<Sequence> itMapValues = mapType.valueIterator();
635+
while (itMapValues.hasNext()) {
636+
final Sequence mapValue = itMapValues.next();
637+
if (sequenceHasItemReference(mapValue, itemReference)) {
638+
return true;
639+
}
640+
}
641+
}
642+
643+
if (item instanceof ArrayType) {
644+
final ArrayType arrayType = (ArrayType) item;
645+
final Iterator<Sequence> itArrayValues = arrayType.iterator();
646+
while (itArrayValues.hasNext()) {
647+
final Sequence arrayValue = itArrayValues.next();
648+
if (sequenceHasItemReference(arrayValue, itemReference)) {
649+
return true;
650+
}
651+
}
652+
}
653+
654+
return false;
552655
}
553656
}

‎exist-core/src/main/java/org/exist/xquery/XQueryContext.java‎

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3471,7 +3471,7 @@ public void registerBinaryValueInstance(final BinaryValue binaryValue) {
34713471
}
34723472

34733473
/**
3474-
* Cleanup Task which is responsible for relasing the streams
3474+
* Cleanup Task which is responsible for releasing the streams
34753475
* of any {@link BinaryValue} which have been used during
34763476
* query execution
34773477
*/
@@ -3603,23 +3603,40 @@ public interface CleanupTask {
36033603

36043604
@Override
36053605
public void runCleanupTasks(final Predicate<Object> predicate) {
3606-
if (importedContextsCleanupTasksFns != null) {
3606+
runCleanupTasks(true, true, (clazz) -> true, predicate);
3607+
}
3608+
3609+
/**
3610+
* Run cleanup tasks.
3611+
*
3612+
* @param runImportedContextsCleanupTasks true if cleanup tasks for imported contexts should be run.
3613+
* @param clearCleanupTasks true if after cleanup tasks have run they should be cleared.
3614+
* @param cleanupTaskFilter a predicate to selectively choose which cleanup tasks are run.
3615+
* @param predicate a predicate to determine what objects should be cleaned up by the cleanup tasks.
3616+
*/
3617+
public void runCleanupTasks(final boolean runImportedContextsCleanupTasks, final boolean clearCleanupTasks, final Predicate<Class<? extends CleanupTask>> cleanupTaskFilter, final Predicate<Object> predicate) {
3618+
if (runImportedContextsCleanupTasks && importedContextsCleanupTasksFns != null) {
36073619
for (final Consumer<Predicate<Object>> importedContextsCleanupTasksFn : importedContextsCleanupTasksFns) {
36083620
importedContextsCleanupTasksFn.accept(predicate);
36093621
}
36103622
importedContextsCleanupTasksFns = null;
36113623
}
36123624

36133625
for (final CleanupTask cleanupTask : cleanupTasks) {
3614-
try {
3615-
cleanupTask.cleanup(this, predicate);
3616-
} catch (final Throwable t) {
3617-
LOG.error("Cleaning up XQueryContext: Ignoring: {}", t.getMessage(), t);
3626+
if (cleanupTaskFilter.test(cleanupTask.getClass())) {
3627+
try {
3628+
cleanupTask.cleanup(this, predicate);
3629+
} catch (final Throwable t) {
3630+
LOG.error("Cleaning up XQueryContext: Ignoring: {}", t.getMessage(), t);
3631+
}
36183632
}
36193633
}
3620-
// now it is safe to clear the cleanup tasks list as we know they have run
3621-
// do not move this anywhere else
3622-
cleanupTasks.clear();
3634+
3635+
if (clearCleanupTasks) {
3636+
// now it is safe to clear the cleanup tasks list as we know they have run
3637+
// do not move this anywhere else
3638+
cleanupTasks.clear();
3639+
}
36233640
}
36243641

36253642
@Immutable

‎exist-core/src/main/java/org/exist/xquery/functions/array/ArrayType.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@
6464
*
6565
* @author Wolf
6666
*/
67-
public class ArrayType extends FunctionReference implements Lookup.LookupSupport {
67+
public class ArrayType extends FunctionReference implements Iterable<Sequence>, Lookup.LookupSupport {
6868

6969
// the signature of the function which is evaluated if the map is called as a function item
7070
private static final FunctionSignature ACCESSOR =
@@ -266,6 +266,11 @@ public Sequence[] toArray() {
266266
return (Sequence[]) RT.seqToPassedArray(vector.seq(), array);
267267
}
268268

269+
@Override
270+
public Iterator<Sequence> iterator() {
271+
return new SeqIterator<>(vector.seq());
272+
}
273+
269274
public int getSize() {
270275
return vector.length();
271276
}

0 commit comments

Comments
 (0)