haystack/test/components/classifiers/test_document_language_classifier.py at 345d9b27732a3e80a22495b6fbcb138c0bb3a502 · devteamaegis/haystack · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
#
# SPDX-License-Identifier: Apache-2.0

import logging

import pytest

from haystack import Document
from haystack.components.classifiers import DocumentLanguageClassifier


class TestDocumentLanguageClassifier:
    def test_init(self):
        component = DocumentLanguageClassifier()
        assert component.languages == ["en"]

    def test_non_document_input(self):
        with pytest.raises(TypeError, match="DocumentLanguageClassifier expects a list of Document as input."):
            classifier = DocumentLanguageClassifier()
            classifier.run(documents="This is an english sentence.")

    def test_single_document(self):
        with pytest.raises(TypeError, match="DocumentLanguageClassifier expects a list of Document as input."):
            classifier = DocumentLanguageClassifier()
            classifier.run(documents=Document(content="This is an english sentence."))

    def test_empty_list(self):
        classifier = DocumentLanguageClassifier()
        result = classifier.run(documents=[])
        assert result == {"documents": []}

    def test_detect_language(self):
        classifier = DocumentLanguageClassifier()
        detected_language = classifier._detect_language(Document(content="This is an english sentence."))
        assert detected_language == "en"

    def test_classify_as_en_and_unmatched(self):
        classifier = DocumentLanguageClassifier()
        english_document = Document(content="This is an english sentence.")
        german_document = Document(content="Ein deutscher Satz ohne Verb.")
        result = classifier.run(documents=[english_document, german_document])
        assert result["documents"][0].meta["language"] == "en"
        assert result["documents"][1].meta["language"] == "unmatched"
        assert "language" not in english_document.meta
        assert "language" not in german_document.meta

    def test_warning_if_no_language_detected(self, caplog):
        with caplog.at_level(logging.WARNING):
            classifier = DocumentLanguageClassifier()
            classifier.run(documents=[Document(content=".")])
            assert "Langdetect cannot detect the language of Document with id" in caplog.text

    def test_content_none_does_not_raise(self):
        """Regression test for https://github.com/deepset-ai/haystack/issues/11418.

        Documents with content=None (blob-only documents) must not raise TypeError.
        They should be classified as 'unmatched' and a warning must be emitted.
        """
        classifier = DocumentLanguageClassifier()
        # Should NOT raise TypeError
        result = classifier.run(documents=[Document(content=None)])
        assert len(result["documents"]) == 1
        assert result["documents"][0].meta["language"] == "unmatched"

    def test_content_none_emits_warning(self, caplog):
        """Regression test: a warning is logged for documents with content=None."""
        with caplog.at_level(logging.WARNING):
            classifier = DocumentLanguageClassifier()
            classifier.run(documents=[Document(content=None)])
            assert "Langdetect cannot detect the language of Document with id" in caplog.text

    def test_mixed_none_and_text_content(self):
        """Documents with content=None and normal documents can coexist in the same batch."""
        classifier = DocumentLanguageClassifier()
        docs = [
            Document(content="This is an english sentence."),
            Document(content=None),
        ]
        result = classifier.run(documents=docs)
        assert result["documents"][0].meta["language"] == "en"
        assert result["documents"][1].meta["language"] == "unmatched"