fix: DocumentRecallEvaluator single_hit reports false 1.0 recall on missing comparison values (#11951)

Co-authored-by: anakin87 <stefanofiorucci@gmail.com>

Author: otiscuilei

haystack/components/evaluators/document_recall.py

@@ -112,33 +112,47 @@ def _get_comparison_value(self, doc: Document) -> Any:
         )
         raise ValueError(msg)
 
-    def _recall_single_hit(self, ground_truth_documents: list[Document], retrieved_documents: list[Document]) -> float:
-        unique_truths = {self._get_comparison_value(g) for g in ground_truth_documents}
-        unique_retrievals = {self._get_comparison_value(p) for p in retrieved_documents}
-        retrieved_ground_truths = unique_truths.intersection(unique_retrievals)
-
-        return float(len(retrieved_ground_truths) > 0)
-
-    def _recall_multi_hit(self, ground_truth_documents: list[Document], retrieved_documents: list[Document]) -> float:
-        unique_truths = {self._get_comparison_value(g) for g in ground_truth_documents}
-        unique_retrievals = {self._get_comparison_value(p) for p in retrieved_documents}
-        retrieved_ground_truths = unique_truths.intersection(unique_retrievals)
+    def _unique_comparison_values(self, documents: list[Document]) -> set:
+        """
+        Collect the unique comparison values of the documents, ignoring missing or empty ones.
+        """
+        return {value for doc in documents if (value := self._get_comparison_value(doc)) not in ("", None)}
 
-        if not unique_truths or unique_truths <= {"", None}:
+    def _comparison_sets(
+        self, ground_truth_documents: list[Document], retrieved_documents: list[Document]
+    ) -> tuple[set, set]:
+        """
+        Collect the unique comparison values of both sides, warning if either has no value to match on.
+        """
+        unique_truths = self._unique_comparison_values(ground_truth_documents)
+        if not unique_truths:
             logger.warning(
                 "There are no ground truth documents or none of them contain a valid comparison value. "
                 "Score will be set to 0."
             )
-            return 0.0
 
-        if not unique_retrievals or unique_retrievals <= {"", None}:
+        unique_retrievals = self._unique_comparison_values(retrieved_documents)
+        if not unique_retrievals:
             logger.warning(
                 "There are no retrieved documents or none of them contain a valid comparison value. "
                 "Score will be set to 0."
             )
+
+        return unique_truths, unique_retrievals
+
+    def _recall_single_hit(self, ground_truth_documents: list[Document], retrieved_documents: list[Document]) -> float:
+        unique_truths, unique_retrievals = self._comparison_sets(ground_truth_documents, retrieved_documents)
+        if not unique_truths or not unique_retrievals:
+            return 0.0
+
+        return float(len(unique_truths.intersection(unique_retrievals)) > 0)
+
+    def _recall_multi_hit(self, ground_truth_documents: list[Document], retrieved_documents: list[Document]) -> float:
+        unique_truths, unique_retrievals = self._comparison_sets(ground_truth_documents, retrieved_documents)
+        if not unique_truths or not unique_retrievals:
             return 0.0
 
-        return len(retrieved_ground_truths) / len(unique_truths)
+        return len(unique_truths.intersection(unique_retrievals)) / len(unique_truths)
 
     @component.output_types(score=float, individual_scores=list[float])
     def run(

releasenotes/notes/document-recall-single-hit-missing-values-f78265c3caa11ded.yaml

@@ -0,0 +1,7 @@
+---
+fixes:
+  - |
+    Fixed ``DocumentRecallEvaluator`` returning wrong scores when the configured
+    ``document_comparison_field`` is missing or empty on some documents. Those documents were
+    considered a match for each other, so unrelated documents could count as correctly retrieved.
+    They are now ignored when computing the score.

test/components/evaluators/test_document_recall.py

@@ -152,6 +152,46 @@ def test_from_dict(self):
         new_evaluator = default_from_dict(DocumentRecallEvaluator, data)
         assert new_evaluator.mode == RecallMode.SINGLE_HIT
 
+    def test_empty_ground_truth_documents(self, evaluator):
+        ground_truth_documents = [[]]
+        retrieved_documents = [[Document(content="test")]]
+        score = evaluator.run(ground_truth_documents, retrieved_documents)
+        assert score == {"individual_scores": [0.0], "score": 0.0}
+
+    def test_empty_retrieved_documents(self, evaluator):
+        ground_truth_documents = [[Document(content="test")]]
+        retrieved_documents = [[]]
+        score = evaluator.run(ground_truth_documents, retrieved_documents)
+        assert score == {"individual_scores": [0.0], "score": 0.0}
+
+    def test_empty_string_ground_truth_documents(self, evaluator):
+        ground_truth_documents = [[Document(content="")]]
+        retrieved_documents = [[Document(content="test")]]
+        score = evaluator.run(ground_truth_documents, retrieved_documents)
+        assert score == {"individual_scores": [0.0], "score": 0.0}
+
+    def test_empty_string_retrieved_documents(self, evaluator):
+        ground_truth_documents = [[Document(content="test")]]
+        retrieved_documents = [[Document(content="")]]
+        score = evaluator.run(ground_truth_documents, retrieved_documents)
+        assert score == {"individual_scores": [0.0], "score": 0.0}
+
+    def test_no_match_when_comparison_field_missing_on_all_documents(self):
+        evaluator = DocumentRecallEvaluator(mode=RecallMode.SINGLE_HIT, document_comparison_field="meta.file_id")
+        result = evaluator.run(
+            ground_truth_documents=[[Document(content="real ground truth")]],
+            retrieved_documents=[[Document(content="totally unrelated doc")]],
+        )
+        assert result == {"individual_scores": [0.0], "score": 0.0}
+
+    def test_no_match_when_comparison_field_missing_on_some_documents(self):
+        evaluator = DocumentRecallEvaluator(mode=RecallMode.SINGLE_HIT, document_comparison_field="meta.file_id")
+        result = evaluator.run(
+            ground_truth_documents=[[Document(meta={"file_id": "f1"}), Document(content="no file_id")]],
+            retrieved_documents=[[Document(content="also no file_id"), Document(meta={"file_id": "f2"})]],
+        )
+        assert result == {"individual_scores": [0.0], "score": 0.0}
+
 
 class TestDocumentRecallEvaluatorMultiHit:
     @pytest.fixture
@@ -265,3 +305,19 @@ def test_empty_string_retrieved_documents(self, evaluator):
         retrieved_documents = [[Document(content="")]]
         score = evaluator.run(ground_truth_documents, retrieved_documents)
         assert score == {"individual_scores": [0.0], "score": 0.0}
+
+    def test_no_match_when_comparison_field_missing_on_some_documents(self):
+        evaluator = DocumentRecallEvaluator(mode=RecallMode.MULTI_HIT, document_comparison_field="meta.file_id")
+        result = evaluator.run(
+            ground_truth_documents=[[Document(meta={"file_id": "f1"}), Document(content="no file_id")]],
+            retrieved_documents=[[Document(content="also no file_id"), Document(meta={"file_id": "f2"})]],
+        )
+        assert result == {"individual_scores": [0.0], "score": 0.0}
+
+    def test_ground_truth_missing_comparison_field_excluded_from_denominator(self):
+        evaluator = DocumentRecallEvaluator(mode=RecallMode.MULTI_HIT, document_comparison_field="meta.file_id")
+        result = evaluator.run(
+            ground_truth_documents=[[Document(meta={"file_id": "f1"}), Document(content="no file_id")]],
+            retrieved_documents=[[Document(meta={"file_id": "f1"})]],
+        )
+        assert result == {"individual_scores": [1.0], "score": 1.0}