From 0011ad5c146c8c589386e4095a7378a8a13e1282 Mon Sep 17 00:00:00 2001 From: otiscuilei Date: Thu, 9 Jul 2026 03:36:04 +0800 Subject: [PATCH 1/2] fix: DocumentRecallEvaluator single_hit false 1.0 on missing comparison values --- .../components/evaluators/document_recall.py | 14 ++++++++ ...e-hit-missing-values-f78265c3caa11ded.yaml | 7 ++++ .../evaluators/test_document_recall.py | 34 +++++++++++++++++++ 3 files changed, 55 insertions(+) create mode 100644 releasenotes/notes/document-recall-single-hit-missing-values-f78265c3caa11ded.yaml diff --git a/haystack/components/evaluators/document_recall.py b/haystack/components/evaluators/document_recall.py index 83a4e044cc6..e4dc2bcffc5 100644 --- a/haystack/components/evaluators/document_recall.py +++ b/haystack/components/evaluators/document_recall.py @@ -117,6 +117,20 @@ def _recall_single_hit(self, ground_truth_documents: list[Document], retrieved_d unique_retrievals = {self._get_comparison_value(p) for p in retrieved_documents} retrieved_ground_truths = unique_truths.intersection(unique_retrievals) + if not unique_truths or unique_truths <= {"", None}: + logger.warning( + "There are no ground truth documents or none of them contain a valid comparison value. " + "Score will be set to 0." + ) + return 0.0 + + if not unique_retrievals or unique_retrievals <= {"", None}: + logger.warning( + "There are no retrieved documents or none of them contain a valid comparison value. " + "Score will be set to 0." + ) + return 0.0 + return float(len(retrieved_ground_truths) > 0) def _recall_multi_hit(self, ground_truth_documents: list[Document], retrieved_documents: list[Document]) -> float: diff --git a/releasenotes/notes/document-recall-single-hit-missing-values-f78265c3caa11ded.yaml b/releasenotes/notes/document-recall-single-hit-missing-values-f78265c3caa11ded.yaml new file mode 100644 index 00000000000..14d63083c22 --- /dev/null +++ b/releasenotes/notes/document-recall-single-hit-missing-values-f78265c3caa11ded.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Fixed ``DocumentRecallEvaluator`` in ``single_hit`` mode reporting a false recall of 1.0 when the + configured ``document_comparison_field`` is missing (or empty) on all documents. Missing/empty + comparison values are now excluded before matching, so the score is 0.0 in that case, mirroring + the behavior of ``multi_hit`` mode. diff --git a/test/components/evaluators/test_document_recall.py b/test/components/evaluators/test_document_recall.py index f29c18cb24b..a84d025bc44 100644 --- a/test/components/evaluators/test_document_recall.py +++ b/test/components/evaluators/test_document_recall.py @@ -152,6 +152,40 @@ def test_from_dict(self): new_evaluator = default_from_dict(DocumentRecallEvaluator, data) assert new_evaluator.mode == RecallMode.SINGLE_HIT + def test_empty_ground_truth_documents(self, evaluator): + ground_truth_documents = [[]] + retrieved_documents = [[Document(content="test")]] + score = evaluator.run(ground_truth_documents, retrieved_documents) + assert score == {"individual_scores": [0.0], "score": 0.0} + + def test_empty_retrieved_documents(self, evaluator): + ground_truth_documents = [[Document(content="test")]] + retrieved_documents = [[]] + score = evaluator.run(ground_truth_documents, retrieved_documents) + assert score == {"individual_scores": [0.0], "score": 0.0} + + def test_empty_string_ground_truth_documents(self, evaluator): + ground_truth_documents = [[Document(content="")]] + retrieved_documents = [[Document(content="test")]] + score = evaluator.run(ground_truth_documents, retrieved_documents) + assert score == {"individual_scores": [0.0], "score": 0.0} + + def test_empty_string_retrieved_documents(self, evaluator): + ground_truth_documents = [[Document(content="test")]] + retrieved_documents = [[Document(content="")]] + score = evaluator.run(ground_truth_documents, retrieved_documents) + assert score == {"individual_scores": [0.0], "score": 0.0} + + def test_missing_comparison_field_on_both_sides(self): + # When the configured comparison field is absent on all documents, every comparison value is + # None. Such missing values must not be treated as a match (previously reported a false 1.0). + evaluator = DocumentRecallEvaluator(mode=RecallMode.SINGLE_HIT, document_comparison_field="meta.file_id") + result = evaluator.run( + ground_truth_documents=[[Document(content="real ground truth")]], + retrieved_documents=[[Document(content="totally unrelated doc")]], + ) + assert result == {"individual_scores": [0.0], "score": 0.0} + class TestDocumentRecallEvaluatorMultiHit: @pytest.fixture From 6bce552dbd87bc74983732d31a1aa36317784381 Mon Sep 17 00:00:00 2001 From: anakin87 Date: Fri, 17 Jul 2026 09:58:24 +0200 Subject: [PATCH 2/2] fix: ignore missing comparison values in DocumentRecallEvaluator --- .../components/evaluators/document_recall.py | 50 +++++++++---------- ...e-hit-missing-values-f78265c3caa11ded.yaml | 8 +-- .../evaluators/test_document_recall.py | 28 +++++++++-- 3 files changed, 54 insertions(+), 32 deletions(-) diff --git a/haystack/components/evaluators/document_recall.py b/haystack/components/evaluators/document_recall.py index e4dc2bcffc5..f42cccd8950 100644 --- a/haystack/components/evaluators/document_recall.py +++ b/haystack/components/evaluators/document_recall.py @@ -112,47 +112,47 @@ def _get_comparison_value(self, doc: Document) -> Any: ) raise ValueError(msg) - def _recall_single_hit(self, ground_truth_documents: list[Document], retrieved_documents: list[Document]) -> float: - unique_truths = {self._get_comparison_value(g) for g in ground_truth_documents} - unique_retrievals = {self._get_comparison_value(p) for p in retrieved_documents} - retrieved_ground_truths = unique_truths.intersection(unique_retrievals) + def _unique_comparison_values(self, documents: list[Document]) -> set: + """ + Collect the unique comparison values of the documents, ignoring missing or empty ones. + """ + return {value for doc in documents if (value := self._get_comparison_value(doc)) not in ("", None)} - if not unique_truths or unique_truths <= {"", None}: + def _comparison_sets( + self, ground_truth_documents: list[Document], retrieved_documents: list[Document] + ) -> tuple[set, set]: + """ + Collect the unique comparison values of both sides, warning if either has no value to match on. + """ + unique_truths = self._unique_comparison_values(ground_truth_documents) + if not unique_truths: logger.warning( "There are no ground truth documents or none of them contain a valid comparison value. " "Score will be set to 0." ) - return 0.0 - if not unique_retrievals or unique_retrievals <= {"", None}: + unique_retrievals = self._unique_comparison_values(retrieved_documents) + if not unique_retrievals: logger.warning( "There are no retrieved documents or none of them contain a valid comparison value. " "Score will be set to 0." ) - return 0.0 - return float(len(retrieved_ground_truths) > 0) + return unique_truths, unique_retrievals - def _recall_multi_hit(self, ground_truth_documents: list[Document], retrieved_documents: list[Document]) -> float: - unique_truths = {self._get_comparison_value(g) for g in ground_truth_documents} - unique_retrievals = {self._get_comparison_value(p) for p in retrieved_documents} - retrieved_ground_truths = unique_truths.intersection(unique_retrievals) - - if not unique_truths or unique_truths <= {"", None}: - logger.warning( - "There are no ground truth documents or none of them contain a valid comparison value. " - "Score will be set to 0." - ) + def _recall_single_hit(self, ground_truth_documents: list[Document], retrieved_documents: list[Document]) -> float: + unique_truths, unique_retrievals = self._comparison_sets(ground_truth_documents, retrieved_documents) + if not unique_truths or not unique_retrievals: return 0.0 - if not unique_retrievals or unique_retrievals <= {"", None}: - logger.warning( - "There are no retrieved documents or none of them contain a valid comparison value. " - "Score will be set to 0." - ) + return float(len(unique_truths.intersection(unique_retrievals)) > 0) + + def _recall_multi_hit(self, ground_truth_documents: list[Document], retrieved_documents: list[Document]) -> float: + unique_truths, unique_retrievals = self._comparison_sets(ground_truth_documents, retrieved_documents) + if not unique_truths or not unique_retrievals: return 0.0 - return len(retrieved_ground_truths) / len(unique_truths) + return len(unique_truths.intersection(unique_retrievals)) / len(unique_truths) @component.output_types(score=float, individual_scores=list[float]) def run( diff --git a/releasenotes/notes/document-recall-single-hit-missing-values-f78265c3caa11ded.yaml b/releasenotes/notes/document-recall-single-hit-missing-values-f78265c3caa11ded.yaml index 14d63083c22..fec4da48af8 100644 --- a/releasenotes/notes/document-recall-single-hit-missing-values-f78265c3caa11ded.yaml +++ b/releasenotes/notes/document-recall-single-hit-missing-values-f78265c3caa11ded.yaml @@ -1,7 +1,7 @@ --- fixes: - | - Fixed ``DocumentRecallEvaluator`` in ``single_hit`` mode reporting a false recall of 1.0 when the - configured ``document_comparison_field`` is missing (or empty) on all documents. Missing/empty - comparison values are now excluded before matching, so the score is 0.0 in that case, mirroring - the behavior of ``multi_hit`` mode. + Fixed ``DocumentRecallEvaluator`` returning wrong scores when the configured + ``document_comparison_field`` is missing or empty on some documents. Those documents were + considered a match for each other, so unrelated documents could count as correctly retrieved. + They are now ignored when computing the score. diff --git a/test/components/evaluators/test_document_recall.py b/test/components/evaluators/test_document_recall.py index a84d025bc44..05eb94dc919 100644 --- a/test/components/evaluators/test_document_recall.py +++ b/test/components/evaluators/test_document_recall.py @@ -176,9 +176,7 @@ def test_empty_string_retrieved_documents(self, evaluator): score = evaluator.run(ground_truth_documents, retrieved_documents) assert score == {"individual_scores": [0.0], "score": 0.0} - def test_missing_comparison_field_on_both_sides(self): - # When the configured comparison field is absent on all documents, every comparison value is - # None. Such missing values must not be treated as a match (previously reported a false 1.0). + def test_no_match_when_comparison_field_missing_on_all_documents(self): evaluator = DocumentRecallEvaluator(mode=RecallMode.SINGLE_HIT, document_comparison_field="meta.file_id") result = evaluator.run( ground_truth_documents=[[Document(content="real ground truth")]], @@ -186,6 +184,14 @@ def test_missing_comparison_field_on_both_sides(self): ) assert result == {"individual_scores": [0.0], "score": 0.0} + def test_no_match_when_comparison_field_missing_on_some_documents(self): + evaluator = DocumentRecallEvaluator(mode=RecallMode.SINGLE_HIT, document_comparison_field="meta.file_id") + result = evaluator.run( + ground_truth_documents=[[Document(meta={"file_id": "f1"}), Document(content="no file_id")]], + retrieved_documents=[[Document(content="also no file_id"), Document(meta={"file_id": "f2"})]], + ) + assert result == {"individual_scores": [0.0], "score": 0.0} + class TestDocumentRecallEvaluatorMultiHit: @pytest.fixture @@ -299,3 +305,19 @@ def test_empty_string_retrieved_documents(self, evaluator): retrieved_documents = [[Document(content="")]] score = evaluator.run(ground_truth_documents, retrieved_documents) assert score == {"individual_scores": [0.0], "score": 0.0} + + def test_no_match_when_comparison_field_missing_on_some_documents(self): + evaluator = DocumentRecallEvaluator(mode=RecallMode.MULTI_HIT, document_comparison_field="meta.file_id") + result = evaluator.run( + ground_truth_documents=[[Document(meta={"file_id": "f1"}), Document(content="no file_id")]], + retrieved_documents=[[Document(content="also no file_id"), Document(meta={"file_id": "f2"})]], + ) + assert result == {"individual_scores": [0.0], "score": 0.0} + + def test_ground_truth_missing_comparison_field_excluded_from_denominator(self): + evaluator = DocumentRecallEvaluator(mode=RecallMode.MULTI_HIT, document_comparison_field="meta.file_id") + result = evaluator.run( + ground_truth_documents=[[Document(meta={"file_id": "f1"}), Document(content="no file_id")]], + retrieved_documents=[[Document(meta={"file_id": "f1"})]], + ) + assert result == {"individual_scores": [1.0], "score": 1.0}