* fix translation of 0x98 to \u02DC

* fix surrogate-character-reference parse error handling
* fix consecutive ampersands before namede entity parsing

Author: rbri

src/main/java/org/htmlunit/cyberneko/HTMLScanner.java

@@ -1469,6 +1469,15 @@ protected int scanEntityRef(final XMLString str, final XMLString plainValue, fin
         }
         str.append((char) nextChar);
 
+        if ('&' == nextChar) {
+            fCurrentEntity.rewind(1);
+            if (plainValue != null) {
+                plainValue.append('&');
+            }
+            str.clearAndAppend('&');
+            return returnEntityRefString(str, content);
+        }
+
         if ('#' == nextChar) {
             fUnicodeEntitiesParser.reset();
 

src/main/java/org/htmlunit/cyberneko/HTMLUnicodeEntitiesParser.java

@@ -83,6 +83,7 @@ public void setMatchFromCode() {
         // a surrogate-character-reference parse error. Set the character reference code to 0xFFFD
         if (Character.isSurrogate((char) code_)) {
             match_ = "\uFFFD";
+            matchLength_ = consumedCount_;
             return;
         }
 
@@ -196,7 +197,7 @@ public void setMatchFromCode() {
                 return;
 
             case 0x98:
-                match_ = "\u20DC";
+                match_ = "\u02DC";
                 matchLength_ = consumedCount_;
                 return;
 

src/test/java/org/htmlunit/cyberneko/GeneralTest.java

@@ -315,4 +315,40 @@ public void parseInputSourceIANAEncoding() throws Exception {
                 + ")html";
         assertEquals(expected.trim(), out.toString().trim());
     }
+
+    @Test
+    public void textContentConsecutiveAmpersandsBeforeNamedEntity() throws Exception {
+        final String expected = "(html" + NL
+                + "(head" + NL
+                + ")head" + NL
+                + "(body" + NL
+                + "\"FOO&&&>BAR" + NL
+                + ")body" + NL
+                + ")html";
+        doTest("FOO&&&>BAR", null, expected);
+    }
+
+    @Test
+    public void textContentSurrogateNumericReference() throws Exception {
+        final String expected = "(html" + NL
+                + "(head" + NL
+                + ")head" + NL
+                + "(body" + NL
+                + "\"FOO\uFFFDZOO" + NL
+                + ")body" + NL
+                + ")html";
+        doTest("FOO�ZOO", null, expected);
+    }
+
+    @Test
+    public void textContentWindows1252ControlMapping() throws Exception {
+        final String expected = "(html" + NL
+                + "(head" + NL
+                + ")head" + NL
+                + "(body" + NL
+                + "\"FOO\u02DCZOO" + NL
+                + ")body" + NL
+                + ")html";
+        doTest("FOO˜ZOO", null, expected);
+    }
 }

src/test/java/org/htmlunit/cyberneko/HTMLUnicodeEntitiesParserTest.java

@@ -148,9 +148,12 @@ public void parseSurrogate() {
         while (parser.parseNumeric(input.charAt(i))) {
             i++;
         }
-
+        // A surrogate code point (�) is invalid in HTML, so the parser emits
+        // the replacement character U+FFFD. Because the semicolon-terminated reference
+        // was fully consumed, matchLength equals consumedCount and rewindCount is 0 —
+        // no characters need to be put back into the input stream.
         assertEquals("\uFFFD", parser.getMatch());
-        assertEquals(6, parser.getRewindCount());
+        assertEquals(0, parser.getRewindCount());
     }
 
     @Test