Merge pull request freelawproject#294 from freelawproject/293-listed-in-authorities-but-not-actually-cited

Exclude star-pagination text when extracting citations

Author: flooie

CHANGES.md

@@ -12,6 +12,7 @@ Changes:
 
 Fixes:
 - Modifies rendering of AhocorasickTokenizer parameter in API docs II
+- Removed star-pagination markers from extracted text #293
 
 ## Current
 

eyecite/clean.py

@@ -51,7 +51,9 @@ def html(html_content: str) -> str:
             parent::link |
             parent::head |
             parent::page-number |
-            parent::script)]"""
+            parent::script |
+            parent::*[@class="star-pagination"]
+            )]"""
     )
     return " ".join(text)
 

eyecite/models.py

@@ -942,6 +942,14 @@ def __post_init__(self):
                     "Not clear which to apply `clean_steps` to"
                 )
 
+            # I done hacked it bro
+            self.plain_to_markup = SpanUpdater(
+                self.markup_text, self.plain_text
+            )
+            self.markup_to_plain = SpanUpdater(
+                self.plain_text, self.markup_text
+            )
+
             self.source_text = self.markup_text
 
     def identify_emphasis_tags(self):

tests/test_FindTest.py

@@ -828,7 +828,14 @@ def test_find_citations(self):
             # Fix for index error when searching for case name
             ("<p>State v. Luna-Benitez (S53965). Alternative writ issued, dismissed, 342 Or 255</p>",
             [case_citation(volume="342", reporter="Or", page="255")],
-            {'clean_steps': ['html', 'inline_whitespace']})
+            {'clean_steps': ['html', 'inline_whitespace']}),
+            # Test remove text with star-pagination class
+            ("<p>The somewhat similar cases of <i>Crane</i> v. <i>Hyde Park,</i> 135 <span class=\"star-pagination\">*355</span> Mass. 147, and <i>Mahoning County</i> v. <i>Young,</i> 16 U.S. App. 253, also cited by the defendant, likewise turned upon a question of forfeiture for breach of a condition subsequent in a deed to a municipal corporation.</p>",
+             [case_citation(volume="135", reporter="Mass.", page="147",
+                            metadata={"plaintiff": "Crane",
+                                      "defendant": "Hyde Park"}
+                            )],
+             {'clean_steps': ['html', 'inline_whitespace']})
         )
 
         # fmt: on