Fix for undetectable encodings (#10884) (#10905)

Co-authored-by: Minura Punchihewa <49385643+MinuraPunchihewa@users.noreply.github.com>
Co-authored-by: martyna-mindsdb <109554435+martyna-mindsdb@users.noreply.github.com>
Co-authored-by: Zoran Pandovski <zoran.pandovski@gmail.com>
Co-authored-by: Andrey <elkin.andr@gmail.com>

Author: 5 people

mindsdb/integrations/handlers/email_handler/email_ingestor.py

@@ -21,45 +21,48 @@ def __init__(self, email_client: EmailClient, search_options: EmailSearchOptions
         self.search_options = search_options
 
     def _is_tag_visible(self, element):
-        if element.parent.name in ['style', 'script', 'head', 'title', 'meta', '[document]']:
+        if element.parent.name in ["style", "script", "head", "title", "meta", "[document]"]:
             return False
         if isinstance(element, bs4.element.Comment):
             return False
         return True
 
     def _preprocess_raw_html(self, html: str) -> str:
-        soup = BeautifulSoup(html, 'html.parser')
+        soup = BeautifulSoup(html, "html.parser")
         texts = soup.find_all(text=True)
         visible_texts = filter(self._is_tag_visible, texts)
-        return '\n'.join(t.strip() for t in visible_texts)
+        return "\n".join(t.strip() for t in visible_texts)
 
     def _ingest_email_row(self, row: pd.Series) -> dict:
-        if row['body_content_type'] == 'html':
+        if row["body_content_type"] == "html":
             # Extract meaningful text from raw HTML.
-            row['body'] = self._preprocess_raw_html(row['body'])
-        body_str = row['body']
+            row["body"] = self._preprocess_raw_html(row["body"])
+        body_str = row["body"]
         encoding = None
         if isinstance(body_str, bytes):
-            encoding = chardet.detect(body_str)['encoding']
-            if 'windows' in encoding.lower():
+            encoding = chardet.detect(body_str)["encoding"]
+            if encoding is None:
+                # If chardet can't detect the encoding, we default to utf-8.
+                encoding = "utf-8"
+            elif "windows" in encoding.lower():
                 # Easier to treat this at utf-8 since str constructor doesn't support all encodings here:
                 # https://chardet.readthedocs.io/en/latest/supported-encodings.html.
-                encoding = 'utf-8'
+                encoding = "utf-8"
             try:
                 body_str = str(body_str, encoding=encoding)
             except UnicodeDecodeError:
                 # If illegal characters are found, we ignore them.
                 # I encountered this issue with some emails that had a mix of encodings.
-                body_str = row['body'].decode(encoding, errors='ignore')
+                body_str = row["body"].decode(encoding, errors="ignore")
         # We split by paragraph so make sure there aren't too many newlines in a row.
-        body_str = re.sub(r'[\r\n]\s*[\r\n]', '\n\n', body_str)
+        body_str = re.sub(r"[\r\n]\s*[\r\n]", "\n\n", body_str)
         email_data = {
-            'id': row['id'],
-            'body': body_str,
-            'subject': row['subject'],
-            'to_field': row['to_field'],
-            'from_field': row['from_field'],
-            'datetime': row['date']
+            "id": row["id"],
+            "body": body_str,
+            "subject": row["subject"],
+            "to_field": row["to_field"],
+            "from_field": row["from_field"],
+            "datetime": row["date"],
         }
         # Replacing None values {None: ""}
         for key in email_data:
@@ -77,10 +80,12 @@ def ingest(self) -> pd.DataFrame:
         df = pd.DataFrame(all_email_data)
 
         # Replace "(UTC)" with empty string over a pandas DataFrame column
-        if 'datetime' in df.columns:
-            df['datetime'] = df['datetime'].str.replace(' (UTC)', '')
+        if "datetime" in df.columns:
+            df["datetime"] = df["datetime"].str.replace(" (UTC)", "")
 
             # Convert datetime string to datetime object, and normalize timezone to UTC.
-            df['datetime'] = pd.to_datetime(df['datetime'], utc=True, format="%a, %d %b %Y %H:%M:%S %z", errors='coerce')
+            df["datetime"] = pd.to_datetime(
+                df["datetime"], utc=True, format="%a, %d %b %Y %H:%M:%S %z", errors="coerce"
+            )
 
         return df

tests/unused/unit/handler_tests/test_email_handler.py

@@ -28,7 +28,6 @@ def test_connect_already_connected(self):
         assert connection is self.email_handler.connection, "The connection must be the same as the one in the handler."
 
     def test_check_connection(self):
-
         response = self.email_handler.check_connection()
         assert response.success is True, "The response success must be True."
 
@@ -37,30 +36,38 @@ def test_select(self):
         Test the select method of EmailsTable Class
         """
 
-        mock_df = pd.DataFrame({
-            'date': ["Wed, 02 Feb 2022 15:30:00 +0000",
-                     "Thu, 10 Mar 2022 10:45:15 +0530",
-                     "Fri, 16 Dec 2022 20:15:30 -0400"
-                     ],
-            'body_content_type': ['html', 'html', 'text'],
-            "body": ["<html><body><p>Hello, World!</p></body></html>", "<html><body><p>Hello, World!</p></body></html>", "Hello, World!"],
-            "from_field": ["", "", ""],
-            "id": ["", "", ""],
-            "to_field": ["", "", ""],
-            "subject": ["", "", ""],
-        })
+        mock_df = pd.DataFrame(
+            {
+                "date": [
+                    "Wed, 02 Feb 2022 15:30:00 +0000",
+                    "Thu, 10 Mar 2022 10:45:15 +0530",
+                    "Fri, 16 Dec 2022 20:15:30 -0400",
+                ],
+                "body_content_type": ["html", "html", "text"],
+                "body": [
+                    "<html><body><p>Hello, World!</p></body></html>",
+                    "<html><body><p>Hello, World!</p></body></html>",
+                    "Hello, World!",
+                ],
+                "from_field": ["", "", ""],
+                "id": ["", "", ""],
+                "to_field": ["", "", ""],
+                "subject": ["", "", ""],
+            }
+        )
 
         self.emails_table_instance.handler.connection.search_email = MagicMock(return_value=mock_df)
 
-        query = parse_sql('SELECT * FROM emails limit 1')
+        query = parse_sql("SELECT * FROM emails limit 1")
 
         self.emails_table_instance.select(query)
 
-        assert self.emails_table_instance.handler.connection.search_email.called, ("The search_email "
-                                                                                   "method must be called.")
+        assert self.emails_table_instance.handler.connection.search_email.called, (
+            "The search_email method must be called."
+        )
 
         # select using invalid column should raise Exception
-        query = parse_sql('SELECT invalid_column FROM emails limit 1')
+        query = parse_sql("SELECT invalid_column FROM emails limit 1")
 
         with pytest.raises(Exception):
             self.emails_table_instance.select(query)
@@ -73,16 +80,18 @@ def test_insert(self):
         self.emails_table_instance.handler.connection.send_email = MagicMock()
 
         query = parse_sql(
-            'INSERT INTO email_datasource.emails(to_field, subject, body) '
-            'VALUES ("toemail@email.com", "MindsDB", "Hello from MindsDB!")')
+            "INSERT INTO email_datasource.emails(to_field, subject, body) "
+            'VALUES ("toemail@email.com", "MindsDB", "Hello from MindsDB!")'
+        )
 
         self.emails_table_instance.insert(query)
         assert self.emails_table_instance.handler.connection.send_email.called, "The send_email method must be called."
 
         # insert using invalid column should raise Exception
         query = parse_sql(
-            'INSERT INTO email_datasource.emails(to_field, subject, body, invalid_column) '
-            'VALUES ("toemail@email.com", "MindsDB", "blaha" , "invalid")')
+            "INSERT INTO email_datasource.emails(to_field, subject, body, invalid_column) "
+            'VALUES ("toemail@email.com", "MindsDB", "blaha" , "invalid")'
+        )
 
         with pytest.raises(Exception):
             self.emails_table_instance.insert(query)
@@ -94,9 +103,36 @@ def test_get_columns(self):
 
         columns = self.emails_table_instance.get_columns()
         assert isinstance(columns, list), "The returned value must be a list."
-        assert 'id' in columns, "Column 'id' must be in the columns list."
-        assert 'body' in columns, "Column 'body' must be in the columns list."
-        assert 'subject' in columns, "Column 'subject' must be in the columns list."
-        assert 'to_field' in columns, "Column 'to_field' must be in the columns list."
-        assert 'from_field' in columns, "Column 'from_field' must be in the columns list."
-        assert 'datetime' in columns, "Column 'datetime' must be in the columns list."
+        assert "id" in columns, "Column 'id' must be in the columns list."
+        assert "body" in columns, "Column 'body' must be in the columns list."
+        assert "subject" in columns, "Column 'subject' must be in the columns list."
+        assert "to_field" in columns, "Column 'to_field' must be in the columns list."
+        assert "from_field" in columns, "Column 'from_field' must be in the columns list."
+        assert "datetime" in columns, "Column 'datetime' must be in the columns list."
+
+    def test_undetectable_encoding_handling(self):
+        """
+        Test that the email handler can process emails with undetectable encodings
+        without raising exceptions.
+        """
+
+        undetectable_content = b"\x80\x81\x82\x83\x84\x85\x86\x87"
+        mock_df = pd.DataFrame(
+            {
+                "date": ["Wed, 02 Feb 2022 15:30:00 +0000"],
+                "body_content_type": ["text"],
+                "body": [undetectable_content],
+                "from_field": ["test@example.com"],
+                "id": ["test1"],
+                "to_field": ["recipient@example.com"],
+                "subject": ["Test email with undetectable encoding"],
+            }
+        )
+
+        self.emails_table_instance.handler.connection.search_email = MagicMock(return_value=mock_df)
+        query = parse_sql("SELECT * FROM emails limit 1")
+        result = self.emails_table_instance.select(query)
+
+        assert result is not None, "The result must not be None."
+        assert "body" in result.columns, "The body should be in the result columns."
+        assert len(result) > 0, "The result should not be empty."