improved entity error handling

Author: rbri

src/main/java/org/htmlunit/cyberneko/HTMLScanner.java

@@ -1270,11 +1270,9 @@ else if (c == '<') {
                     fCurrentEntity.rewind();
                     break;
                 }
-                else {
-                    if (!str.appendCodePoint(c)) {
-                        if (fReportErrors_) {
-                            fErrorReporter.reportError("HTML1005", new Object[] {"&#" + c + ';'});
-                        }
+                else if (!str.appendCodePoint(c)) {
+                    if (fReportErrors_) {
+                        fErrorReporter.reportError("HTML1005", new Object[] {"&#" + c + ';'});
                     }
                 }
             }
@@ -1489,6 +1487,11 @@ protected int scanEntityRef(final XMLString str, final XMLString plainValue, fin
             }
             while (nextChar != -1 && fUnicodeEntitiesParser.parseNumeric(nextChar));
 
+            // EOF finalization for numeric character reference without semicolon
+            if (nextChar == -1) {
+                fUnicodeEntitiesParser.finalizeNumericAtEOF();
+            }
+
             final String match = fUnicodeEntitiesParser.getMatch();
             if (match == null) {
                 fCurrentEntity.rewind(str.length() - 1);
@@ -2567,11 +2570,9 @@ private void scanUntilEndTag(final String tagNameWithLeadingSlash) throws IOExce
                         fScanUntilEndTag.append('\n');
                     }
                 }
-                else {
-                    if (!fScanUntilEndTag.appendCodePoint(c)) {
-                        if (fReportErrors_) {
-                            fErrorReporter.reportError("HTML1005", new Object[] {"&#" + c + ';'});
-                        }
+                else if (!fScanUntilEndTag.appendCodePoint(c)) {
+                    if (fReportErrors_) {
+                        fErrorReporter.reportError("HTML1005", new Object[] {"&#" + c + ';'});
                     }
                 }
             }
@@ -2932,11 +2933,9 @@ else if (c == '>') {
                         fDocumentHandler.comment(fStringBuffer, locationAugs(fCurrentEntity));
                         return SCAN_TRUE;
                     }
-                    else {
-                        if (!fStringBuffer.appendCodePoint(c)) {
-                            if (fReportErrors_) {
-                                fErrorReporter.reportError("HTML1005", new Object[] {"&#" + c + ';'});
-                            }
+                    else if (!fStringBuffer.appendCodePoint(c)) {
+                        if (fReportErrors_) {
+                            fErrorReporter.reportError("HTML1005", new Object[] {"&#" + c + ';'});
                         }
                     }
                 }

src/main/java/org/htmlunit/cyberneko/HTMLUnicodeEntitiesParser.java

@@ -73,7 +73,7 @@ public void setMatchFromCode() {
         // Set the character reference code to 0xFFFD.
         // If the number is greater than 0x10FFFF, then this is
         // a character-reference-outside-unicode-range parse error. Set the character reference code to 0xFFFD.
-        if ((0x00 == code_) || (code_ > 0x10FFFF)) {
+        if ((0x00 >= code_) || (code_ > 0x10FFFF)) {
             match_ = "\uFFFD";
             matchLength_ = consumedCount_;
             return;
@@ -326,4 +326,14 @@ public boolean parseNumeric(final int current) {
         return false;
     }
 
+    public void finalizeNumericAtEOF() {
+        if (match_ != null) {
+            return;
+        }
+        if (state_ == STATE_DECIMAL_CHAR || state_ == STATE_HEXADECIMAL_CHAR) {
+            // treat EOF like "missing semicolon": finalize what we have
+            setMatchFromCode();
+            matchLength_ = consumedCount_;
+        }
+    }
 }

src/test/resources/org/htmlunit/cyberneko/testfiles/entities/test-invalid-entity-at-end.html

@@ -0,0 +1 @@
+<div>abc&#11111111111

src/test/resources/org/htmlunit/cyberneko/testfiles/entities/test-invalid-entity-at-end.html.canonical

@@ -0,0 +1,9 @@
+(html
+(head
+)head
+(body
+(div
+"abc�
+)div
+)body
+)html

src/test/resources/org/htmlunit/cyberneko/testfiles/entities/test-invalid-entity-at-end.html.canonical-frg

@@ -0,0 +1,3 @@
+(div
+"abc�
+)div

src/test/resources/org/htmlunit/cyberneko/testfiles/entities/test-invalid-entity-at-end.html.canonical-html

@@ -0,0 +1 @@
+<html><head></head><body><div>abc�</div></body></html>

src/test/resources/org/htmlunit/cyberneko/testfiles/entities/test-invalid-entity-at-end.html.canonical-xni

@@ -0,0 +1,14 @@
+startDocument [(1,1,0) (1,1,0) false]
+startElement (localpart="html",rawname="html",uri="http://www.w3.org/1999/xhtml") [(-1,-1,-1) (-1,-1,-1) true]
+startElement (localpart="head",rawname="head",uri="http://www.w3.org/1999/xhtml") [(-1,-1,-1) (-1,-1,-1) true]
+endElement (localpart="head",rawname="head",uri="http://www.w3.org/1999/xhtml") [(-1,-1,-1) (-1,-1,-1) true]
+startElement (localpart="body",rawname="body",uri="http://www.w3.org/1999/xhtml") [(-1,-1,-1) (-1,-1,-1) true]
+startElement (localpart="div",rawname="div") [(1,1,0) (1,6,5) false]
+characters 'abc'[(1,6,5) (1,9,8) false]
+characters '�'[(1,9,8) (1,22,21) false]
+characters '
+'[(1,22,21) (2,1,22) false]
+endElement (localpart="div",rawname="div") [(-1,-1,-1) (-1,-1,-1) true]
+endElement (localpart="body",rawname="body",uri="http://www.w3.org/1999/xhtml") [(-1,-1,-1) (-1,-1,-1) true]
+endElement (localpart="html",rawname="html",uri="http://www.w3.org/1999/xhtml") [(-1,-1,-1) (-1,-1,-1) true]
+endDocument [(2,1,22) (2,1,22) false]