Handle en and em dashes in titles (mozilla#960)

* Handle en and em dashes in titles

* Update remaining regexes

* Add test case

* Update existing test fixtures

An existing test case had a title with an em dash. It started to fail since it wasn't previously detected as a separator.

* Move title separators into a variable

* Use \s instead of whitespace

* Keep titleSeparators in a regex instead of a string

This makes escaping easier to reason about, but has the downside of having to call .source each time we compose it.

* Inline .source access when defining titleSeparators.

---------

Co-authored-by: Gijs Kruitbosch <gijskruitbosch@gmail.com>

Author: robinmetral

Readability.js

@@ -593,14 +593,20 @@ Readability.prototype = {
     }
 
     // If there's a separator in the title, first remove the final part
-    if (/ [\|\-\\\/>»] /.test(curTitle)) {
-      titleHadHierarchicalSeparators = / [\\\/>»] /.test(curTitle);
-      let allSeparators = Array.from(origTitle.matchAll(/ [\|\-\\\/>»] /gi));
+    const titleSeparators = /\|\-–—\\\/>»/.source;
+    if (new RegExp(`\\s[${titleSeparators}]\\s`).test(curTitle)) {
+      titleHadHierarchicalSeparators = /\s[\\\/>»]\s/.test(curTitle);
+      let allSeparators = Array.from(
+        origTitle.matchAll(new RegExp(`\\s[${titleSeparators}]\\s`, "gi"))
+      );
       curTitle = origTitle.substring(0, allSeparators.pop().index);
 
       // If the resulting title is too short, remove the first part instead:
       if (wordCount(curTitle) < 3) {
-        curTitle = origTitle.replace(/^[^\|\-\\\/>»]*[\|\-\\\/>»]/gi, "");
+        curTitle = origTitle.replace(
+          new RegExp(`^[^${titleSeparators}]*[${titleSeparators}]`, "gi"),
+          ""
+        );
       }
     } else if (curTitle.includes(": ")) {
       // Check if we have an heading containing this exact string, so we
@@ -642,7 +648,10 @@ Readability.prototype = {
       curTitleWordCount <= 4 &&
       (!titleHadHierarchicalSeparators ||
         curTitleWordCount !=
-          wordCount(origTitle.replace(/[\|\-\\\/>»]+/g, "")) - 1)
+          wordCount(
+            origTitle.replace(new RegExp(`\\s[${titleSeparators}]\\s`, "g"), "")
+          ) -
+            1)
     ) {
       curTitle = origTitle;
     }

test/test-pages/mercurial/expected-metadata.json

@@ -1,5 +1,5 @@
 {
-  "title": "Shared Mutable History — evolve extension for Mercurial",
+  "title": "Evolve: Shared Mutable History — evolve extension for Mercurial",
   "byline": null,
   "dir": null,
   "excerpt": "Contents",

test/test-pages/title-en-dash/expected-metadata.json

@@ -0,0 +1,9 @@
+{
+  "title": "Lorem ipsum dolor sit amet",
+  "byline": null,
+  "dir": null,
+  "excerpt": "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n        incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,\n        quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo\n        consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse\n        cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non\n        proident, sunt in culpa qui officia deserunt mollit anim id est laborum.",
+  "siteName": null,
+  "publishedTime": null,
+  "readerable": true
+}

test/test-pages/title-en-dash/expected.html

@@ -0,0 +1,6 @@
+<div id="readability-page-1" class="page">
+    <article>
+        <p> Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. </p>
+        <p> Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. </p>
+    </article>
+</div>

test/test-pages/title-en-dash/source.html

@@ -0,0 +1,27 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8"/>
+    <title>Lorem ipsum dolor sit amet – My website</title>
+</head>
+<body>
+<article>
+    <div>
+        Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
+        incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
+        quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+        consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
+        cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
+        proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+    </div>
+    <div>
+        Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
+        incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
+        quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+        consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
+        cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
+        proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+    </div>
+</article>
+</body>
+</html>