Merge branch 'main' into xmlns-parsing

Author: GuntherRademacher

basex-core/src/main/java/org/basex/query/QueryParser.java

@@ -539,7 +539,7 @@ private boolean defaultNamespaceDecl(final boolean fixed) throws QueryException
     final boolean elem = wsConsumeWs(ELEMENT);
     if(!elem && !wsConsumeWs(FUNCTION)) return false;
     wsCheck(NAMESPACE);
-    final byte[] uri = uriLiteral();
+    final byte[] uri = elem ? uriLiteral(ANY_URI) : uriLiteral();
     if(eq(XML_URI, uri)) throw error(BINDXMLURI_X_X, uri, XML);
     if(eq(XMLNS_URI, uri)) throw error(BINDXMLURI_X_X, uri, XMLNS);
 
@@ -697,7 +697,6 @@ private void schemaImport() throws QueryException {
     }
     final byte[] uri = uriLiteral();
     if(prefix != null && uri.length == 0) throw error(NSEMPTY);
-    if(!Uri.get(uri).isValid()) throw error(INVURI_X, uri);
     addLocations(new TokenList());
     throw error(IMPLSCHEMA);
   }
@@ -715,7 +714,6 @@ private void moduleImport() throws QueryException {
 
     final byte[] uri = uriLiteral();
     if(uri.length == 0) throw error(NSMODURI);
-    if(!Uri.get(uri).isValid()) throw error(INVURI_X, uri);
     if(sc.imports.contains(token(uri))) throw error(DUPLMODULE_X, uri);
     sc.imports.add(uri);
 
@@ -754,8 +752,7 @@ private boolean addLocations(final TokenList list) throws QueryException {
     if(add) {
       do {
         final byte[] uri = uriLiteral();
-        if(!Uri.get(uri).isValid() || IO.get(string(uri)) instanceof IOContent)
-          throw error(INVURI_X, uri);
+        if(IO.get(string(uri)) instanceof IOContent) throw error(INVURI_X, uri);
         list.add(uri);
       } while(wsConsume(","));
     }
@@ -2087,7 +2084,7 @@ private void validate() throws QueryException {
     final int p = pos;
     if(!wsConsumeWs(VALIDATE)) return;
 
-    if(consume(TYPE)) eQName(sc.elemNS, QNAME_X);
+    if(consume(TYPE)) eQName(eq(sc.elemNS, ANY_URI) ? XS_URI : sc.elemNS, QNAME_X);
     consume(STRICT);
     consume(LAX);
     skipWs();
@@ -2459,12 +2456,14 @@ private ExprInfo simpleNodeTest(final Kind kind, final boolean all) throws Query
       } else {
         NameTest.Scope scope = NameTest.Scope.FULL;
         if(!name.hasPrefix()) {
+          pos = p;
           if(consume(":*")) {
             // name test: prefix:*
             name = new QNm(concat(name.string(), cpToken(':')));
             scope = NameTest.Scope.URI;
           } else if(!eqName) {
-            scope = NameTest.Scope.FLEXIBLE;
+            scope = kind == Kind.ELEMENT && eq(sc.elemNS, ANY_URI) ? NameTest.Scope.LOCAL
+                                                                   : NameTest.Scope.FLEXIBLE;
           }
         }
         // name test: prefix:name, name, Q{uri}name
@@ -2875,11 +2874,15 @@ private byte[] stringLiteral() throws QueryException {
 
   /**
    * Parses the "URILiteral" rule.
+   * @param special tokens that are allowed though not valid URI literals
    * @return query expression
    * @throws QueryException query exception
    */
-  private byte[] uriLiteral() throws QueryException {
-    return normalize(stringLiteral());
+  private byte[] uriLiteral(final byte[]... special) throws QueryException {
+    final byte[] uri = normalize(stringLiteral());
+    for(final byte[] sp : special) if(eq(uri, sp)) return uri;
+    if(!Uri.get(uri).isValid()) throw error(INVURI_X, uri);
+    return uri;
   }
 
   /**
@@ -2895,6 +2898,7 @@ private byte[] bracedURILiteral() throws QueryException {
       entity(token);
     }
     final byte[] ns = normalize(token.toArray());
+    if(!Uri.get(ns).isValid()) throw error(INVURI_X, ns);
     if(eq(ns, XMLNS_URI)) {
       pos = p;
       throw error(ILLEGALEQNAME_X, ns);
@@ -3388,7 +3392,7 @@ private Expr compConstructor() throws QueryException {
    * @throws QueryException query exception
    */
   private Expr compElement() throws QueryException {
-    final Expr name = compName(NOELEMNAME, true, sc.elemNS);
+    final Expr name = compName(NOELEMNAME, true, eq(sc.elemNS, ANY_URI) ? null : sc.elemNS);
     if(name == null) return null;
     skipWs();
     return current('{') ? new CElem(info(), true, name, new Atts(), enclosedExpr()) : null;
@@ -3493,7 +3497,7 @@ private SeqType castTarget() throws QueryException {
     if(wsConsume("(")) {
       type = choiceItemType().type;
     } else {
-      final QNm name = eQName(sc.elemNS, TYPEINVALID);
+      final QNm name = eQName(eq(sc.elemNS, ANY_URI) ? XS_URI : sc.elemNS, TYPEINVALID);
       if(!name.hasURI() && eq(name.local(), token(ENUM))) {
         if(!wsConsume("(")) throw error(WHICHCAST_X, BasicType.similar(name));
         type = enumerationType();
@@ -3582,8 +3586,8 @@ private SeqType itemType() throws QueryException {
         }
       }
     } else {
-      // attach default element namespace
-      if(!name.hasURI()) name.uri(sc.elemNS);
+      // attach default element namespace, or schema namespace if default element namespace is ##any
+      if(!name.hasURI()) name.uri(eq(sc.elemNS, ANY_URI) ? XS_URI : sc.elemNS);
       // basic type
       type = BasicType.get(name, false);
       // declared type
@@ -4831,7 +4835,7 @@ private void resolveQNm(final QNm name, final byte[] elemNS, final InputInfo inf
     if(name.hasPrefix()) {
       name.uri(qc.ns.resolve(name.prefix(), sc));
       if(!name.hasURI()) throw error(NOURI_X, info, name.prefix());
-    } else if(elemNS != null) {
+    } else if(elemNS != null && !Token.eq(elemNS, QueryText.ANY_URI)) {
       name.uri(elemNS);
     }
   }

basex-core/src/main/java/org/basex/query/QueryText.java

@@ -340,6 +340,8 @@ public interface QueryText {
   /** BaseX URI. */ byte[] XQUERY_URI = token(BXMODULES_URL + "xquery");
   /** BaseX URI. */ byte[] XSLT_URI = token(BXMODULES_URL + "xslt");
 
+  /** "Any" URI. */ byte[] ANY_URI = token("##any");
+
   // QUERY PLAN ===================================================================================
 
   /** Query Info. */ String OP = "op";

basex-core/src/main/java/org/basex/query/func/archive/ArchiveCreate.java

@@ -10,7 +10,6 @@
 import java.util.Map.*;
 import java.util.zip.*;
 
-import org.basex.io.out.*;
 import org.basex.query.*;
 import org.basex.query.expr.*;
 import org.basex.query.iter.*;
@@ -29,9 +28,12 @@
 public class ArchiveCreate extends ArchiveFn {
   @Override
   public Item item(final QueryContext qc, final InputInfo ii) throws QueryException {
-    final ArrayOutput ao = new ArrayOutput();
-    create(ao, qc);
-    return B64.get(ao.finish());
+    try(SpillOutput so = new SpillOutput(qc)) {
+      create(so, qc);
+      return so.finish(ARCHIVE_ERROR_X);
+    } catch(final IOException ex) {
+      throw ARCHIVE_ERROR_X.get(info, ex);
+    }
   }
 
   /**
@@ -40,7 +42,7 @@ public Item item(final QueryContext qc, final InputInfo ii) throws QueryExceptio
    * @param qc query context
    * @throws QueryException query exception
    */
-  public final void create(final OutputStream os, final QueryContext qc) throws QueryException {
+  public void create(final OutputStream os, final QueryContext qc) throws QueryException {
     final Map<String, Entry<Item, Item>> files = toFiles(arg(0), arg(1), qc);
     final CreateOptions options = toOptions(arg(2), new CreateOptions(), qc);
     create(files, options, os, qc);

basex-core/src/main/java/org/basex/query/func/archive/ArchiveCreateFrom.java

@@ -7,12 +7,10 @@
 import java.util.AbstractMap.*;
 
 import org.basex.io.*;
-import org.basex.io.out.*;
 import org.basex.query.*;
 import org.basex.query.value.*;
 import org.basex.query.value.item.*;
 import org.basex.query.value.seq.*;
-import org.basex.util.*;
 import org.basex.util.list.*;
 
 /**
@@ -23,7 +21,7 @@
  */
 public final class ArchiveCreateFrom extends ArchiveCreate {
   @Override
-  public B64 item(final QueryContext qc, final InputInfo ii) throws QueryException {
+  public void create(final OutputStream os, final QueryContext qc) throws QueryException {
     final IOFile root = new IOFile(toPath(arg(0), qc));
     final CreateFromOptions options = toOptions(arg(1), new CreateFromOptions(), qc);
     Value entries = arg(2).value(qc);
@@ -47,20 +45,18 @@ public B64 item(final QueryContext qc, final InputInfo ii) throws QueryException
     final int level = level(options);
     final String format = options.get(CreateOptions.FORMAT).toLowerCase(Locale.ENGLISH);
     final String dir = rootDir && root.parent() != null ? root.name() + '/' : "";
-    final ArrayOutput ao = new ArrayOutput();
-    try(ArchiveOut out = ArchiveOut.get(format, info, ao)) {
+    try(ArchiveOut out = ArchiveOut.get(format, info, os)) {
       out.level(level);
       try {
         for(final Item item : entries) {
           final IOFile file = new IOFile(root, toString(item, qc));
           if(!file.exists()) throw FILE_NOT_FOUND_X.get(info, file);
           if(file.isDir()) throw FILE_IS_DIR_X.get(info, file);
-          add(new SimpleEntry<>(item, B64.get(file.read())), out, level, dir, qc);
+          add(new SimpleEntry<>(item, new B64Lazy(file, FILE_NOT_FOUND_X)), out, level, dir, qc);
         }
       } catch(final IOException ex) {
         throw ARCHIVE_ERROR_X.get(info, ex);
       }
     }
-    return B64.get(ao.finish());
   }
 }

basex-core/src/main/java/org/basex/query/func/archive/SpillOutput.java

@@ -0,0 +1,125 @@
+package org.basex.query.func.archive;
+
+import java.io.*;
+import java.util.*;
+
+import org.basex.io.*;
+import org.basex.query.QueryContext;
+import org.basex.query.QueryError;
+import org.basex.query.value.item.*;
+import org.basex.util.*;
+
+/**
+ * Spill output stream.
+ *
+ * This class provides an output stream that buffers data in memory, then spills transparently to a
+ * temporary file if the data exceeds the maximum array size or a given threshold.
+ * The result can be retrieved as a binary item via the {@link #finish} method, which returns a lazy
+ * reference to the temporary file if data was spilled, or an in-memory binary item otherwise. The
+ * temporary file is registered with the query context's resources for automatic deletion when the
+ * query finishes.
+ *
+ * This class is used by {@link ArchiveCreateFrom} and {@link ArchiveCreate}.
+ *
+ * @author BaseX Team, BSD License
+ * @author Vincent Lizzi
+ */
+final class SpillOutput extends OutputStream {
+  /** Query context for registering the temporary file on spill. */
+  private final QueryContext qc;
+  /** Threshold in bytes before spilling to disk. */
+  private final int threshold;
+
+  /** In-memory buffer. */
+  private byte[] buffer = new byte[Array.INITIAL_CAPACITY];
+  /** Number of bytes written to the in-memory buffer. */
+  private int bufSize;
+  /** Disk output stream ({@code null} before spilling). */
+  private FileOutputStream tmpOutput;
+  /** Temporary file ({@code null} before spilling). */
+  private IOFile tmpFile;
+
+  /**
+   * Constructor.
+   * @param qc query context
+   */
+  SpillOutput(final QueryContext qc) {
+    this(qc, Array.MAX_SIZE);
+  }
+
+  /**
+   * Constructor with an explicit spill threshold (used for testing).
+   * @param qc query context
+   * @param threshold spill threshold in bytes
+   */
+  SpillOutput(final QueryContext qc, final int threshold) {
+    this.qc = qc;
+    this.threshold = threshold;
+  }
+
+  @Override
+  public void write(final int b) throws IOException {
+    if(tmpOutput == null && bufSize == threshold) spill();
+    if(tmpOutput != null) {
+      tmpOutput.write(b);
+    } else {
+      if(bufSize == buffer.length) {
+        buffer = Arrays.copyOf(buffer, Array.newCapacity(bufSize));
+      }
+      buffer[bufSize++] = (byte) b;
+    }
+  }
+
+  @Override
+  public void write(final byte[] b, final int off, final int len) throws IOException {
+    if(tmpOutput == null && (long) bufSize + len > threshold) spill();
+    if(tmpOutput != null) {
+      tmpOutput.write(b, off, len);
+    } else {
+      final int newSize = bufSize + len;
+      if(newSize > buffer.length) {
+        buffer = Arrays.copyOf(buffer, Math.max(Array.newCapacity(buffer.length), newSize));
+      }
+      System.arraycopy(b, off, buffer, bufSize, len);
+      bufSize = newSize;
+    }
+  }
+
+  /**
+   * Returns the result as a binary item: a lazy reference to the temporary file
+   * if data was spilled, or an in-memory binary item otherwise.
+   * @param error error to raise if the temporary file cannot be read
+   * @return binary item
+   */
+  B64 finish(final QueryError error) {
+    if(tmpFile != null) return new B64Lazy(tmpFile, error);
+    return B64.get(bufSize == 0 ? Token.EMPTY : bufSize == buffer.length ? buffer :
+      Arrays.copyOf(buffer, bufSize));
+  }
+
+  /**
+   * Closes the disk output stream if one was opened. {@code tmpFile} is intentionally
+   * not nulled here because {@link #finish} may be called after {@code close} and still
+   * needs it to determine whether data was spilled.
+   */
+  @Override
+  public void close() throws IOException {
+    if(tmpOutput != null) {
+      tmpOutput.close();
+      tmpOutput = null;
+    }
+  }
+
+  /**
+   * Spills the in-memory buffer to a temporary file, registers it for deletion
+   * when the query context closes, and switches subsequent writes to disk.
+   * @throws IOException I/O exception
+   */
+  private void spill() throws IOException {
+    tmpFile = new IOFile(File.createTempFile(Prop.NAME + '-', IO.TMPSUFFIX));
+    qc.resources.index(TempFiles.class).add(tmpFile);
+    tmpOutput = tmpFile.outputStream();
+    tmpOutput.write(buffer, 0, bufSize);
+    buffer = null;
+  }
+}

basex-core/src/main/java/org/basex/query/func/archive/TempFiles.java

@@ -0,0 +1,35 @@
+package org.basex.query.func.archive;
+
+import java.util.*;
+
+import org.basex.io.*;
+import org.basex.query.*;
+
+/**
+ * Temporary files created during query evaluation.
+ *
+ * Temporary files that are created during query evaluation and registered in this class will be
+ * deleted when the query context is closed, ensuring that no temporary files are left behind after
+ * the query execution.
+ *
+ * @author BaseX Team, BSD License
+ * @author Vincent Lizzi
+ */
+public final class TempFiles implements QueryResource {
+  /** List of temporary files. */
+  private final List<IOFile> files = new ArrayList<>();
+
+  /**
+   * Adds a temporary file to be deleted on close.
+   * @param file temporary file
+   */
+  synchronized void add(final IOFile file) {
+    files.add(file);
+  }
+
+  @Override
+  public synchronized void close() {
+    for(final IOFile file : files) file.delete();
+    files.clear();
+  }
+}

basex-core/src/main/java/org/basex/query/func/file/FileWriteBinary.java

@@ -46,9 +46,9 @@ final void write(final boolean append, final QueryContext qc) throws QueryExcept
     } else {
       // write full file
       try(BufferOutput out = BufferOutput.get(new FileOutputStream(path.toFile(), append))) {
-        if(arg(1).getClass() == ArchiveCreate.class) {
-          // optimization: stream archive to disk (no support for ArchiveCreateFrom)
-          ((ArchiveCreate) arg(1)).create(out, qc);
+        if(arg(1) instanceof final ArchiveCreate ac) {
+          // optimization: stream archive to disk
+          ac.create(out, qc);
         } else {
           IO.write(toBin(arg(1), qc).input(info), out);
         }