From bf1bcb77228add326f2f35c104dd64f4b0ff7635 Mon Sep 17 00:00:00 2001 From: otiscuilei Date: Thu, 9 Jul 2026 09:39:46 +0800 Subject: [PATCH 1/2] fix: raise clear error for empty inputs in retrieval/answer evaluators AnswerExactMatchEvaluator, DocumentMAPEvaluator, DocumentMRREvaluator and DocumentRecallEvaluator averaged their per-query scores by dividing by the input length without guarding against an empty batch. Passing two empty lists (which satisfies the equal-length check) therefore raised an opaque ZeroDivisionError. They now raise a descriptive ValueError, matching the existing contract of DocumentNDCGEvaluator. --- .../evaluators/answer_exact_match.py | 3 ++ .../components/evaluators/document_map.py | 4 +++ .../components/evaluators/document_mrr.py | 4 +++ .../components/evaluators/document_recall.py | 4 +++ ...ty-input-value-error-c2e8b41d7a56f309.yaml | 7 +++++ .../evaluators/test_evaluators_empty_input.py | 30 +++++++++++++++++++ 6 files changed, 52 insertions(+) create mode 100644 releasenotes/notes/evaluators-empty-input-value-error-c2e8b41d7a56f309.yaml create mode 100644 test/components/evaluators/test_evaluators_empty_input.py diff --git a/haystack/components/evaluators/answer_exact_match.py b/haystack/components/evaluators/answer_exact_match.py index a2bc57e549b..580bb0dac51 100644 --- a/haystack/components/evaluators/answer_exact_match.py +++ b/haystack/components/evaluators/answer_exact_match.py @@ -53,6 +53,9 @@ def run(self, ground_truth_answers: list[str], predicted_answers: list[str]) -> - `score` - A number from 0.0 to 1.0 that represents the proportion of questions where any predicted answer matched one of the ground truth answers. """ + if len(ground_truth_answers) == 0 or len(predicted_answers) == 0: + raise ValueError("ground_truth_answers and predicted_answers must be provided.") + if not len(ground_truth_answers) == len(predicted_answers): raise ValueError("The length of ground_truth_answers and predicted_answers must be the same.") diff --git a/haystack/components/evaluators/document_map.py b/haystack/components/evaluators/document_map.py index 668ffa4c96f..27863755e80 100644 --- a/haystack/components/evaluators/document_map.py +++ b/haystack/components/evaluators/document_map.py @@ -108,6 +108,10 @@ def run( - `individual_scores` - A list of numbers from 0.0 to 1.0 that represents how high retrieved documents are ranked. """ + if len(ground_truth_documents) == 0 or len(retrieved_documents) == 0: + msg = "ground_truth_documents and retrieved_documents must be provided." + raise ValueError(msg) + if len(ground_truth_documents) != len(retrieved_documents): msg = "The length of ground_truth_documents and retrieved_documents must be the same." raise ValueError(msg) diff --git a/haystack/components/evaluators/document_mrr.py b/haystack/components/evaluators/document_mrr.py index db51583521a..3550768115e 100644 --- a/haystack/components/evaluators/document_mrr.py +++ b/haystack/components/evaluators/document_mrr.py @@ -106,6 +106,10 @@ def run( - `individual_scores` - A list of numbers from 0.0 to 1.0 that represents how high the first retrieved document is ranked. """ + if len(ground_truth_documents) == 0 or len(retrieved_documents) == 0: + msg = "ground_truth_documents and retrieved_documents must be provided." + raise ValueError(msg) + if len(ground_truth_documents) != len(retrieved_documents): msg = "The length of ground_truth_documents and retrieved_documents must be the same." raise ValueError(msg) diff --git a/haystack/components/evaluators/document_recall.py b/haystack/components/evaluators/document_recall.py index 83a4e044cc6..917885e6e3f 100644 --- a/haystack/components/evaluators/document_recall.py +++ b/haystack/components/evaluators/document_recall.py @@ -158,6 +158,10 @@ def run( - `individual_scores` - A list of numbers from 0.0 to 1.0 that represents the proportion of matching documents retrieved. If the mode is `single_hit`, the individual scores are 0 or 1. """ + if len(ground_truth_documents) == 0 or len(retrieved_documents) == 0: + msg = "ground_truth_documents and retrieved_documents must be provided." + raise ValueError(msg) + if len(ground_truth_documents) != len(retrieved_documents): msg = "The length of ground_truth_documents and retrieved_documents must be the same." raise ValueError(msg) diff --git a/releasenotes/notes/evaluators-empty-input-value-error-c2e8b41d7a56f309.yaml b/releasenotes/notes/evaluators-empty-input-value-error-c2e8b41d7a56f309.yaml new file mode 100644 index 00000000000..52db133f504 --- /dev/null +++ b/releasenotes/notes/evaluators-empty-input-value-error-c2e8b41d7a56f309.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + ``AnswerExactMatchEvaluator``, ``DocumentMAPEvaluator``, ``DocumentMRREvaluator`` and + ``DocumentRecallEvaluator`` now raise a descriptive ``ValueError`` when called with + empty input lists, instead of failing with an opaque ``ZeroDivisionError``. This + matches the existing behavior of ``DocumentNDCGEvaluator``. diff --git a/test/components/evaluators/test_evaluators_empty_input.py b/test/components/evaluators/test_evaluators_empty_input.py new file mode 100644 index 00000000000..ecbef4e5c88 --- /dev/null +++ b/test/components/evaluators/test_evaluators_empty_input.py @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: 2022-present deepset GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +from haystack.components.evaluators import ( + AnswerExactMatchEvaluator, + DocumentMAPEvaluator, + DocumentMRREvaluator, + DocumentRecallEvaluator, +) + + +@pytest.mark.parametrize( + "evaluator, kwargs", + [ + (AnswerExactMatchEvaluator(), {"ground_truth_answers": [], "predicted_answers": []}), + (DocumentMAPEvaluator(), {"ground_truth_documents": [], "retrieved_documents": []}), + (DocumentMRREvaluator(), {"ground_truth_documents": [], "retrieved_documents": []}), + (DocumentRecallEvaluator(), {"ground_truth_documents": [], "retrieved_documents": []}), + ], +) +def test_run_with_empty_inputs_raises_value_error(evaluator, kwargs): + # Empty (equal-length) inputs previously fell through the length check and + # crashed with a bare ZeroDivisionError when averaging over zero items. + # They must instead raise a descriptive ValueError, matching + # DocumentNDCGEvaluator's contract. + with pytest.raises(ValueError, match="must be provided"): + evaluator.run(**kwargs) From f2b4b1235ea17989fe151162e2b73f4c13bb21ce Mon Sep 17 00:00:00 2001 From: otiscuilei Date: Sun, 26 Jul 2026 20:10:42 +0800 Subject: [PATCH 2/2] test: move empty-input tests into the existing evaluator test files --- .../evaluators/test_answer_exact_match.py | 9 ++++++ .../evaluators/test_document_map.py | 9 ++++++ .../evaluators/test_document_mrr.py | 9 ++++++ .../evaluators/test_document_recall.py | 9 ++++++ .../evaluators/test_evaluators_empty_input.py | 30 ------------------- 5 files changed, 36 insertions(+), 30 deletions(-) delete mode 100644 test/components/evaluators/test_evaluators_empty_input.py diff --git a/test/components/evaluators/test_answer_exact_match.py b/test/components/evaluators/test_answer_exact_match.py index 581ef9742e0..1718eb93e30 100644 --- a/test/components/evaluators/test_answer_exact_match.py +++ b/test/components/evaluators/test_answer_exact_match.py @@ -71,3 +71,12 @@ def test_run_with_different_lengths(): with pytest.raises(ValueError): evaluator.run(ground_truth_answers=["Berlin", "Paris"], predicted_answers=["Berlin"]) + + +def test_run_with_empty_inputs(): + evaluator = AnswerExactMatchEvaluator() + + # Empty (equal-length) inputs previously fell through the length check and + # crashed with a bare ZeroDivisionError when averaging over zero items. + with pytest.raises(ValueError, match="must be provided"): + evaluator.run(ground_truth_answers=[], predicted_answers=[]) diff --git a/test/components/evaluators/test_document_map.py b/test/components/evaluators/test_document_map.py index fcd54804b6d..2041f0accc2 100644 --- a/test/components/evaluators/test_document_map.py +++ b/test/components/evaluators/test_document_map.py @@ -147,3 +147,12 @@ def test_run_with_different_lengths(): ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]], retrieved_documents=[[Document(content="Berlin")]], ) + + +def test_run_with_empty_inputs(): + evaluator = DocumentMAPEvaluator() + + # Empty (equal-length) inputs previously fell through the length check and + # crashed with a bare ZeroDivisionError when averaging over zero items. + with pytest.raises(ValueError, match="must be provided"): + evaluator.run(ground_truth_documents=[], retrieved_documents=[]) diff --git a/test/components/evaluators/test_document_mrr.py b/test/components/evaluators/test_document_mrr.py index af41d16e9ae..2c15a753984 100644 --- a/test/components/evaluators/test_document_mrr.py +++ b/test/components/evaluators/test_document_mrr.py @@ -150,3 +150,12 @@ def test_run_with_different_lengths(): ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]], retrieved_documents=[[Document(content="Berlin")]], ) + + +def test_run_with_empty_inputs(): + evaluator = DocumentMRREvaluator() + + # Empty (equal-length) inputs previously fell through the length check and + # crashed with a bare ZeroDivisionError when averaging over zero items. + with pytest.raises(ValueError, match="must be provided"): + evaluator.run(ground_truth_documents=[], retrieved_documents=[]) diff --git a/test/components/evaluators/test_document_recall.py b/test/components/evaluators/test_document_recall.py index f29c18cb24b..96bcdfa66e9 100644 --- a/test/components/evaluators/test_document_recall.py +++ b/test/components/evaluators/test_document_recall.py @@ -68,6 +68,15 @@ def test_run_with_nested_meta_comparison(): assert result == {"individual_scores": [0.5], "score": 0.5} +def test_run_with_empty_inputs(): + evaluator = DocumentRecallEvaluator() + + # Empty (equal-length) inputs previously fell through the length check and + # crashed with a bare ZeroDivisionError when averaging over zero items. + with pytest.raises(ValueError, match="must be provided"): + evaluator.run(ground_truth_documents=[], retrieved_documents=[]) + + class TestDocumentRecallEvaluatorSingleHit: @pytest.fixture def evaluator(self): diff --git a/test/components/evaluators/test_evaluators_empty_input.py b/test/components/evaluators/test_evaluators_empty_input.py deleted file mode 100644 index ecbef4e5c88..00000000000 --- a/test/components/evaluators/test_evaluators_empty_input.py +++ /dev/null @@ -1,30 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 - -import pytest - -from haystack.components.evaluators import ( - AnswerExactMatchEvaluator, - DocumentMAPEvaluator, - DocumentMRREvaluator, - DocumentRecallEvaluator, -) - - -@pytest.mark.parametrize( - "evaluator, kwargs", - [ - (AnswerExactMatchEvaluator(), {"ground_truth_answers": [], "predicted_answers": []}), - (DocumentMAPEvaluator(), {"ground_truth_documents": [], "retrieved_documents": []}), - (DocumentMRREvaluator(), {"ground_truth_documents": [], "retrieved_documents": []}), - (DocumentRecallEvaluator(), {"ground_truth_documents": [], "retrieved_documents": []}), - ], -) -def test_run_with_empty_inputs_raises_value_error(evaluator, kwargs): - # Empty (equal-length) inputs previously fell through the length check and - # crashed with a bare ZeroDivisionError when averaging over zero items. - # They must instead raise a descriptive ValueError, matching - # DocumentNDCGEvaluator's contract. - with pytest.raises(ValueError, match="must be provided"): - evaluator.run(**kwargs)