[MOD] Parsing, INTPARSE: Better DTD support (default values, etc.)

Author: ChristianGruen

basex-core/src/main/java/org/basex/build/xml/XMLParser.java

@@ -70,7 +70,11 @@ public final void parse() throws IOException {
     while(true) {
       if(scanner.type == Type.TEXT) {
         final byte[] text = scanner.token.toArray();
-        if(!elms.isEmpty() || fragment || !ws(text)) {
+        // ignore whitespace outside the root element, and ignorable whitespace between the
+        // children of an element with element-only content (matching the default parser)
+        final boolean ignorable = ws(text) && (elms.isEmpty() ? !fragment :
+            scanner.elementContent(elms.peek()));
+        if(!ignorable) {
           if(strips.peek()) scanner.token.trim();
           builder.text(scanner.token.toArray());
         }
@@ -127,8 +131,8 @@ private boolean parseElement() throws IOException {
     nsp.reset();
 
     // get element name
-    byte[] en = consumeToken(Type.ELEMNAME);
-    if(stripNS) en = local(en);
+    final byte[] raw = consumeToken(Type.ELEMNAME);
+    byte[] en = stripNS ? local(raw) : raw;
     skipSpace();
 
     // parse optional attributes
@@ -161,6 +165,9 @@ private boolean parseElement() throws IOException {
       }
     }
 
+    // apply DTD-declared attribute defaults and tokenized-type value normalization
+    scanner.attributes(raw, atts, stripNS);
+
     // send empty element to builder
     if(scanner.type == Type.CLOSE_R_BR) {
       builder.emptyElem(en, atts, nsp);

basex-core/src/main/java/org/basex/build/xml/XMLScanner.java

@@ -49,6 +49,17 @@ private enum Scan {
   private final TokenObjectMap<byte[]> ents = new TokenObjectMap<>();
   /** Index for all PEReferences. */
   private final TokenObjectMap<byte[]> pents = new TokenObjectMap<>();
+  /** Declared attributes per element name (for default values and value normalization). */
+  private final TokenObjectMap<TokenObjectMap<AttDecl>> attDecls = new TokenObjectMap<>();
+  /** Element names declared with element-only content. */
+  private final TokenSet elemContent = new TokenSet();
+
+  /**
+   * Declared attribute.
+   * @param tokenized tokenized (non-CDATA) type flag
+   * @param value default value (or {@code null})
+   */
+  private record AttDecl(boolean tokenized, byte[] value) { }
   /** DTD flag. */
   private final boolean dtd;
   /** Parse fragment. */
@@ -802,7 +813,7 @@ private boolean markupDecl() throws IOException {
       pe = true;
     } else if(consume(ELEM)) { // [45]
       checkS();
-      name(true);
+      final byte[] elem = name(true);
       checkS();
       pe = true;
       if(!consume(EMP) && !consume(ANY)) { // [46]
@@ -814,7 +825,8 @@ private boolean markupDecl() throws IOException {
             while(consume('|')) { s(); name(true); s(); alt = true; }
             check(')');
             if(!consume('*') && alt) throw error(INVEND);
-          } else { // [47] children
+          } else { // [47] children (element-only content)
+            elemContent.add(elem);
             cp();
             while(sep()) cp();
             s();
@@ -830,13 +842,14 @@ private boolean markupDecl() throws IOException {
     } else if(consume(ATTL)) { // [52]
       pe = true;
       checkS();
-      name(true);
+      final byte[] elem = name(true);
       s();
-      while(name(false) != null) { // [53]
+      for(byte[] att; (att = name(false)) != null;) { // [53]
         checkS();
-        if(!consume(CD) && !consume(IDRS) && !consume(IDR) && !consume(ID) &&
-            !consume(ENTS) && !consume(ENT1) && !consume(NMTS) &&
-            !consume(NMT)) { // [56]
+        // [56] AttType: CDATA is the only non-tokenized (StringType) type
+        final boolean tokenized = !consume(CD);
+        if(tokenized && !consume(IDRS) && !consume(IDR) && !consume(ID) &&
+            !consume(ENTS) && !consume(ENT1) && !consume(NMTS) && !consume(NMT)) {
           if(consume(NOT)) { // [57,58]
             checkS(); check('(');
             do { s(); name(true); s(); } while(consume('|'));
@@ -847,14 +860,18 @@ private boolean markupDecl() throws IOException {
           check(')');
         }
 
-        // [54]
+        // [54] DefaultDecl
         pe = true;
         checkS();
+        byte[] value = null;
         if(!consume(REQ) && !consume(IMP)) { // [60]
           if(consume(FIX)) checkS();
           quote = qu();
+          token.reset();
           attValue(consume());
+          value = tokenized ? normalize(token.toArray()) : token.toArray();
         }
+        if(tokenized || value != null) declareAtt(elem, att, tokenized, value);
         s();
       }
       check('>');
@@ -910,6 +927,71 @@ private void occ() throws IOException {
     if(!consume('+') && !consume('?')) consume('*');
   }
 
+  /**
+   * Registers a declared attribute. The first declaration of an attribute is binding.
+   * @param elem element name
+   * @param att attribute name
+   * @param tokenized non-CDATA (tokenized) type flag
+   * @param value default value (or {@code null})
+   */
+  private void declareAtt(final byte[] elem, final byte[] att, final boolean tokenized,
+      final byte[] value) {
+    attDecls.computeIfAbsent(elem, () -> new TokenObjectMap<>()).
+      computeIfAbsent(att, () -> new AttDecl(tokenized, value));
+  }
+
+  /**
+   * Applies declared attribute defaults and normalization. [3.3.2, 3.3.3]
+   * @param elem element name
+   * @param atts attributes assembled from the start tag
+   * @param stripNS strip namespaces
+   */
+  void attributes(final byte[] elem, final Atts atts, final boolean stripNS) {
+    final TokenObjectMap<AttDecl> decls = attDecls.get(elem);
+    if(decls == null) return;
+    // normalize specified values of tokenized-type attributes
+    final int as = atts.size();
+    for(int a = 0; a < as; a++) {
+      final AttDecl decl = decls.get(atts.name(a));
+      if(decl != null && decl.tokenized) atts.value(a, normalize(atts.value(a)));
+    }
+    // add default values for declared attributes that are absent (in declaration order)
+    for(final byte[] att : decls) {
+      final AttDecl decl = decls.get(att);
+      if(decl.value != null && !atts.contains(att)) atts.add(att, decl.value, stripNS);
+    }
+  }
+
+  /**
+   * Indicates whether the named element was declared with element-only content, so that
+   * whitespace between its child elements is ignorable. [3.2.1]
+   * @param elem element name
+   * @return result of check
+   */
+  boolean elementContent(final byte[] elem) {
+    return elemContent.contains(elem);
+  }
+
+  /**
+   * Normalizes an attribute value of a tokenized type. [3.3.3]
+   * @param value value
+   * @return normalized value
+   */
+  private static byte[] normalize(final byte[] value) {
+    final TokenBuilder tb = new TokenBuilder();
+    boolean space = false;
+    for(final byte b : value) {
+      if(b == ' ') {
+        space = true;
+      } else {
+        if(space && !tb.isEmpty()) tb.add(' ');
+        space = false;
+        tb.add(b);
+      }
+    }
+    return tb.finish();
+  }
+
   /**
    * Scans an entity value. [9]
    * @param p pe reference flag

basex-core/src/main/java/org/basex/util/Atts.java

@@ -138,6 +138,17 @@ public byte[] value(final int index) {
     return list[(index << 1) + 1];
   }
 
+  /**
+   * Sets the value at the specified index position.
+   * @param index index
+   * @param value value
+   * @return self reference
+   */
+  public Atts value(final int index, final byte[] value) {
+    list[(index << 1) + 1] = value;
+    return this;
+  }
+
   /**
    * Returns the value for the specified name or {@code null}.
    * @param name name to be found

basex-core/src/test/java/org/basex/build/XMLParserTest.java

@@ -125,24 +125,53 @@ public final class XMLParserTest extends SandboxTest {
 
   /** Internal and default parser must agree on DTD content models. */
   @Test public void dtdContentModelParsers() {
-    set(MainOptions.DTD, true);
-
     // well-formed documents with an internal subset and a matching element tree
-    final String[] docs = {
+    sameOnBothParsers(
         "<!DOCTYPE a [ <!ELEMENT a (b, c)> ]><a><b/><c/></a>",
         "<!DOCTYPE a [ <!ELEMENT a ((b)?, (c)*)> ]><a><c/><c/></a>",
         "<!DOCTYPE a [ <!ELEMENT a (b | c)> ]><a><b/></a>",
         "<!DOCTYPE a [ <!ELEMENT a (#PCDATA | b)*> ]><a>x<b/>y</a>",
-        "<!DOCTYPE a [ <!ELEMENT a (b, (c, d)+, e)> ]><a><b/><c/><d/><e/></a>",
-    };
-    for(final String doc : docs) {
-      set(MainOptions.INTPARSE, false);
-      execute(new CreateDB(NAME, doc));
-      final String def = query(".");
-      set(MainOptions.INTPARSE, true);
-      execute(new CreateDB(NAME, doc));
-      assertEquals(def, query("."), "Parsers disagree on: " + doc);
-    }
+        "<!DOCTYPE a [ <!ELEMENT a (b, (c, d)+, e)> ]><a><b/><c/><d/><e/></a>");
+  }
+
+  /**
+   * Internal and default parser must agree on attribute defaults (incl. #FIXED and enumeration
+   * defaults) and on tokenized-type attribute-value normalization.
+   */
+  @Test public void dtdAttributes() {
+    sameOnBothParsers(
+        // default values, incl. empty and multiple defaults; specified values win
+        "<!DOCTYPE a [ <!ATTLIST a b CDATA \"x\"> ]><a/>",
+        "<!DOCTYPE a [ <!ATTLIST a b CDATA \"x\"> ]><a b=\"y\"/>",
+        "<!DOCTYPE a [ <!ATTLIST a b CDATA \"\"> ]><a/>",
+        "<!DOCTYPE a [ <!ATTLIST a b CDATA \"1\" c CDATA \"2\"> ]><a c=\"X\"/>",
+        // #FIXED and enumeration defaults
+        "<!DOCTYPE a [ <!ATTLIST a b CDATA #FIXED \"x\"> ]><a/>",
+        "<!DOCTYPE a [ <!ATTLIST a b (l | r) \"l\"> ]><a/>",
+        // entity reference in a default value
+        "<!DOCTYPE a [ <!ATTLIST a b CDATA \"&#65;\"> ]><a/>",
+        // tokenized-type normalization (collapse + trim); CDATA is left untouched
+        "<!DOCTYPE a [ <!ATTLIST a b NMTOKEN #IMPLIED> ]><a b=\"  x  \"/>",
+        "<!DOCTYPE a [ <!ATTLIST a b NMTOKENS #IMPLIED> ]><a b=\" x   y \"/>",
+        "<!DOCTYPE a [ <!ATTLIST a b (l | r) #IMPLIED> ]><a b=\" l \"/>",
+        "<!DOCTYPE a [ <!ATTLIST a b CDATA #IMPLIED> ]><a b=\" x   y \"/>",
+        // #IMPLIED / #REQUIRED add nothing
+        "<!DOCTYPE a [ <!ATTLIST a b CDATA #IMPLIED> ]><a/>",
+        "<!DOCTYPE a [ <!ATTLIST a b CDATA #REQUIRED> ]><a b=\"z\"/>");
+  }
+
+  /** Internal and default parser must agree on ignorable (element-content) whitespace. */
+  @Test public void dtdElementContentWhitespace() {
+    sameOnBothParsers(
+        // element-only content: whitespace between children is ignorable and dropped
+        "<!DOCTYPE a [ <!ELEMENT a (b, b)><!ELEMENT b EMPTY> ]><a>\n  <b/>\n  <b/>\n</a>",
+        "<!DOCTYPE a [ <!ELEMENT a (b)><!ELEMENT b (c)><!ELEMENT c EMPTY> ]>" +
+            "<a>\n <b>\n  <c/>\n </b>\n</a>",
+        // mixed content and ANY: whitespace is significant and kept
+        "<!DOCTYPE a [ <!ELEMENT a (#PCDATA | b)*><!ELEMENT b EMPTY> ]><a>\n  <b/>\n</a>",
+        "<!DOCTYPE a [ <!ELEMENT a ANY><!ELEMENT b EMPTY> ]><a>\n  <b/>\n</a>",
+        // non-whitespace text in element content is kept by both (non-validating)
+        "<!DOCTYPE a [ <!ELEMENT a (b)><!ELEMENT b EMPTY> ]><a> x <b/> y </a>");
   }
 
   /** A malformed DTD must report its real cause, not a masked "empty document" error. */
@@ -161,26 +190,6 @@ public final class XMLParserTest extends SandboxTest {
     assertTrue(empty.contains("No input found"), "Unexpected error: " + empty);
   }
 
-  /** Internal parser: ATTLIST declarations, including enumerations and empty default values. */
-  @Test public void dtdAttlist() {
-    set(MainOptions.INTPARSE, true);
-    set(MainOptions.DTD, true);
-
-    // attribute declarations that must be accepted; an empty default value ("") regressed the
-    // scanner before the fix, overrunning the whole declaration
-    final String[] attlists = {
-        "<!ATTLIST a b CDATA \"\">",
-        "<!ATTLIST a b CDATA '' c CDATA '50'>",
-        "<!ATTLIST a b (x | y | z) \"x\">",
-        "<!ATTLIST a b NMTOKEN #IMPLIED c CDATA #REQUIRED d CDATA #FIXED \"1\">",
-        "<!ATTLIST a align (left | right | center | justify | char) \"left\" char CDATA \"\">",
-    };
-    for(final String attlist : attlists) {
-      execute(new CreateDB(NAME, "<!DOCTYPE a [ " + attlist + " ]><a/>"));
-      query(".", "<a/>");
-    }
-  }
-
   /**
    * Internal parser: a complex external DTD subset (parameter entities inside declarations,
    * a parameter-entity-driven conditional section, enumerations and empty default values).
@@ -220,6 +229,22 @@ public final class XMLParserTest extends SandboxTest {
     query("name(*)", "tgroup");
   }
 
+  /**
+   * Asserts that the internal and the default parser produce identical documents.
+   * @param docs document strings (each with an internal DTD subset)
+   */
+  private void sameOnBothParsers(final String... docs) {
+    set(MainOptions.DTD, true);
+    for(final String doc : docs) {
+      set(MainOptions.INTPARSE, false);
+      execute(new CreateDB(NAME, doc));
+      final String def = query(".");
+      set(MainOptions.INTPARSE, true);
+      execute(new CreateDB(NAME, doc));
+      assertEquals(def, query("."), "Parsers disagree on: " + doc);
+    }
+  }
+
   /**
    * Creates a database and returns the resulting error message, or {@code null} on success.
    * @param doc document string