diff --git a/haystack/components/evaluators/document_recall.py b/haystack/components/evaluators/document_recall.py index 83a4e044cc6..f42cccd8950 100644 --- a/haystack/components/evaluators/document_recall.py +++ b/haystack/components/evaluators/document_recall.py @@ -112,33 +112,47 @@ def _get_comparison_value(self, doc: Document) -> Any: ) raise ValueError(msg) - def _recall_single_hit(self, ground_truth_documents: list[Document], retrieved_documents: list[Document]) -> float: - unique_truths = {self._get_comparison_value(g) for g in ground_truth_documents} - unique_retrievals = {self._get_comparison_value(p) for p in retrieved_documents} - retrieved_ground_truths = unique_truths.intersection(unique_retrievals) - - return float(len(retrieved_ground_truths) > 0) - - def _recall_multi_hit(self, ground_truth_documents: list[Document], retrieved_documents: list[Document]) -> float: - unique_truths = {self._get_comparison_value(g) for g in ground_truth_documents} - unique_retrievals = {self._get_comparison_value(p) for p in retrieved_documents} - retrieved_ground_truths = unique_truths.intersection(unique_retrievals) + def _unique_comparison_values(self, documents: list[Document]) -> set: + """ + Collect the unique comparison values of the documents, ignoring missing or empty ones. + """ + return {value for doc in documents if (value := self._get_comparison_value(doc)) not in ("", None)} - if not unique_truths or unique_truths <= {"", None}: + def _comparison_sets( + self, ground_truth_documents: list[Document], retrieved_documents: list[Document] + ) -> tuple[set, set]: + """ + Collect the unique comparison values of both sides, warning if either has no value to match on. + """ + unique_truths = self._unique_comparison_values(ground_truth_documents) + if not unique_truths: logger.warning( "There are no ground truth documents or none of them contain a valid comparison value. " "Score will be set to 0." ) - return 0.0 - if not unique_retrievals or unique_retrievals <= {"", None}: + unique_retrievals = self._unique_comparison_values(retrieved_documents) + if not unique_retrievals: logger.warning( "There are no retrieved documents or none of them contain a valid comparison value. " "Score will be set to 0." ) + + return unique_truths, unique_retrievals + + def _recall_single_hit(self, ground_truth_documents: list[Document], retrieved_documents: list[Document]) -> float: + unique_truths, unique_retrievals = self._comparison_sets(ground_truth_documents, retrieved_documents) + if not unique_truths or not unique_retrievals: + return 0.0 + + return float(len(unique_truths.intersection(unique_retrievals)) > 0) + + def _recall_multi_hit(self, ground_truth_documents: list[Document], retrieved_documents: list[Document]) -> float: + unique_truths, unique_retrievals = self._comparison_sets(ground_truth_documents, retrieved_documents) + if not unique_truths or not unique_retrievals: return 0.0 - return len(retrieved_ground_truths) / len(unique_truths) + return len(unique_truths.intersection(unique_retrievals)) / len(unique_truths) @component.output_types(score=float, individual_scores=list[float]) def run( diff --git a/releasenotes/notes/document-recall-single-hit-missing-values-f78265c3caa11ded.yaml b/releasenotes/notes/document-recall-single-hit-missing-values-f78265c3caa11ded.yaml new file mode 100644 index 00000000000..fec4da48af8 --- /dev/null +++ b/releasenotes/notes/document-recall-single-hit-missing-values-f78265c3caa11ded.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Fixed ``DocumentRecallEvaluator`` returning wrong scores when the configured + ``document_comparison_field`` is missing or empty on some documents. Those documents were + considered a match for each other, so unrelated documents could count as correctly retrieved. + They are now ignored when computing the score. diff --git a/test/components/evaluators/test_document_recall.py b/test/components/evaluators/test_document_recall.py index f29c18cb24b..05eb94dc919 100644 --- a/test/components/evaluators/test_document_recall.py +++ b/test/components/evaluators/test_document_recall.py @@ -152,6 +152,46 @@ def test_from_dict(self): new_evaluator = default_from_dict(DocumentRecallEvaluator, data) assert new_evaluator.mode == RecallMode.SINGLE_HIT + def test_empty_ground_truth_documents(self, evaluator): + ground_truth_documents = [[]] + retrieved_documents = [[Document(content="test")]] + score = evaluator.run(ground_truth_documents, retrieved_documents) + assert score == {"individual_scores": [0.0], "score": 0.0} + + def test_empty_retrieved_documents(self, evaluator): + ground_truth_documents = [[Document(content="test")]] + retrieved_documents = [[]] + score = evaluator.run(ground_truth_documents, retrieved_documents) + assert score == {"individual_scores": [0.0], "score": 0.0} + + def test_empty_string_ground_truth_documents(self, evaluator): + ground_truth_documents = [[Document(content="")]] + retrieved_documents = [[Document(content="test")]] + score = evaluator.run(ground_truth_documents, retrieved_documents) + assert score == {"individual_scores": [0.0], "score": 0.0} + + def test_empty_string_retrieved_documents(self, evaluator): + ground_truth_documents = [[Document(content="test")]] + retrieved_documents = [[Document(content="")]] + score = evaluator.run(ground_truth_documents, retrieved_documents) + assert score == {"individual_scores": [0.0], "score": 0.0} + + def test_no_match_when_comparison_field_missing_on_all_documents(self): + evaluator = DocumentRecallEvaluator(mode=RecallMode.SINGLE_HIT, document_comparison_field="meta.file_id") + result = evaluator.run( + ground_truth_documents=[[Document(content="real ground truth")]], + retrieved_documents=[[Document(content="totally unrelated doc")]], + ) + assert result == {"individual_scores": [0.0], "score": 0.0} + + def test_no_match_when_comparison_field_missing_on_some_documents(self): + evaluator = DocumentRecallEvaluator(mode=RecallMode.SINGLE_HIT, document_comparison_field="meta.file_id") + result = evaluator.run( + ground_truth_documents=[[Document(meta={"file_id": "f1"}), Document(content="no file_id")]], + retrieved_documents=[[Document(content="also no file_id"), Document(meta={"file_id": "f2"})]], + ) + assert result == {"individual_scores": [0.0], "score": 0.0} + class TestDocumentRecallEvaluatorMultiHit: @pytest.fixture @@ -265,3 +305,19 @@ def test_empty_string_retrieved_documents(self, evaluator): retrieved_documents = [[Document(content="")]] score = evaluator.run(ground_truth_documents, retrieved_documents) assert score == {"individual_scores": [0.0], "score": 0.0} + + def test_no_match_when_comparison_field_missing_on_some_documents(self): + evaluator = DocumentRecallEvaluator(mode=RecallMode.MULTI_HIT, document_comparison_field="meta.file_id") + result = evaluator.run( + ground_truth_documents=[[Document(meta={"file_id": "f1"}), Document(content="no file_id")]], + retrieved_documents=[[Document(content="also no file_id"), Document(meta={"file_id": "f2"})]], + ) + assert result == {"individual_scores": [0.0], "score": 0.0} + + def test_ground_truth_missing_comparison_field_excluded_from_denominator(self): + evaluator = DocumentRecallEvaluator(mode=RecallMode.MULTI_HIT, document_comparison_field="meta.file_id") + result = evaluator.run( + ground_truth_documents=[[Document(meta={"file_id": "f1"}), Document(content="no file_id")]], + retrieved_documents=[[Document(meta={"file_id": "f1"})]], + ) + assert result == {"individual_scores": [1.0], "score": 1.0}