feat: add clean_newline utility for hyphenated line breaks (#2513)

Author: DevAbdullah90

test_unstructured/cleaners/test_core.py

@@ -147,6 +147,21 @@ def test_clean_extra_whitespace(text, expected):
     assert core.clean(text=text, extra_whitespace=True) == expected
 
 
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("re- \nsearch", "research"),
+        (
+            "This is an example of text contin-\nuing to next line",
+            "This is an example of text continuing to next line",
+        ),
+        ("pre-\n  processing", "preprocessing"),
+    ],
+)
+def test_clean_newline(text, expected):
+    assert core.clean_newline(text) == expected
+
+
 @pytest.mark.parametrize(
     ("text", "expected"),
     [

unstructured/cleaners/core.py

@@ -334,6 +334,16 @@ def clean_extra_whitespace(text: str) -> str:
     return cleaned_text.strip()
 
 
+def clean_newline(text: str) -> str:
+    """Concatenates hyphenated words split across newlines or significant whitespace.
+
+    Example
+    -------
+    re- \nsearch -> research
+    """
+    return re.sub(r"(\w+)-\s+(\w+)", r"\1\2", text)
+
+
 def clean_dashes(text: str) -> str:
     """Cleans dash characters in text.