diff --git a/haystack/components/evaluators/answer_exact_match.py b/haystack/components/evaluators/answer_exact_match.py index a2bc57e549..580bb0dac5 100644 --- a/haystack/components/evaluators/answer_exact_match.py +++ b/haystack/components/evaluators/answer_exact_match.py @@ -53,6 +53,9 @@ def run(self, ground_truth_answers: list[str], predicted_answers: list[str]) -> - `score` - A number from 0.0 to 1.0 that represents the proportion of questions where any predicted answer matched one of the ground truth answers. """ + if len(ground_truth_answers) == 0 or len(predicted_answers) == 0: + raise ValueError("ground_truth_answers and predicted_answers must be provided.") + if not len(ground_truth_answers) == len(predicted_answers): raise ValueError("The length of ground_truth_answers and predicted_answers must be the same.") diff --git a/haystack/components/evaluators/document_map.py b/haystack/components/evaluators/document_map.py index 668ffa4c96..27863755e8 100644 --- a/haystack/components/evaluators/document_map.py +++ b/haystack/components/evaluators/document_map.py @@ -108,6 +108,10 @@ def run( - `individual_scores` - A list of numbers from 0.0 to 1.0 that represents how high retrieved documents are ranked. """ + if len(ground_truth_documents) == 0 or len(retrieved_documents) == 0: + msg = "ground_truth_documents and retrieved_documents must be provided." + raise ValueError(msg) + if len(ground_truth_documents) != len(retrieved_documents): msg = "The length of ground_truth_documents and retrieved_documents must be the same." raise ValueError(msg) diff --git a/haystack/components/evaluators/document_mrr.py b/haystack/components/evaluators/document_mrr.py index db51583521..3550768115 100644 --- a/haystack/components/evaluators/document_mrr.py +++ b/haystack/components/evaluators/document_mrr.py @@ -106,6 +106,10 @@ def run( - `individual_scores` - A list of numbers from 0.0 to 1.0 that represents how high the first retrieved document is ranked. """ + if len(ground_truth_documents) == 0 or len(retrieved_documents) == 0: + msg = "ground_truth_documents and retrieved_documents must be provided." + raise ValueError(msg) + if len(ground_truth_documents) != len(retrieved_documents): msg = "The length of ground_truth_documents and retrieved_documents must be the same." raise ValueError(msg) diff --git a/haystack/components/evaluators/document_recall.py b/haystack/components/evaluators/document_recall.py index f42cccd895..68e591d7bf 100644 --- a/haystack/components/evaluators/document_recall.py +++ b/haystack/components/evaluators/document_recall.py @@ -172,6 +172,10 @@ def run( - `individual_scores` - A list of numbers from 0.0 to 1.0 that represents the proportion of matching documents retrieved. If the mode is `single_hit`, the individual scores are 0 or 1. """ + if len(ground_truth_documents) == 0 or len(retrieved_documents) == 0: + msg = "ground_truth_documents and retrieved_documents must be provided." + raise ValueError(msg) + if len(ground_truth_documents) != len(retrieved_documents): msg = "The length of ground_truth_documents and retrieved_documents must be the same." raise ValueError(msg) diff --git a/releasenotes/notes/evaluators-empty-input-value-error-c2e8b41d7a56f309.yaml b/releasenotes/notes/evaluators-empty-input-value-error-c2e8b41d7a56f309.yaml new file mode 100644 index 0000000000..52db133f50 --- /dev/null +++ b/releasenotes/notes/evaluators-empty-input-value-error-c2e8b41d7a56f309.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + ``AnswerExactMatchEvaluator``, ``DocumentMAPEvaluator``, ``DocumentMRREvaluator`` and + ``DocumentRecallEvaluator`` now raise a descriptive ``ValueError`` when called with + empty input lists, instead of failing with an opaque ``ZeroDivisionError``. This + matches the existing behavior of ``DocumentNDCGEvaluator``. diff --git a/test/components/evaluators/test_answer_exact_match.py b/test/components/evaluators/test_answer_exact_match.py index 581ef9742e..1718eb93e3 100644 --- a/test/components/evaluators/test_answer_exact_match.py +++ b/test/components/evaluators/test_answer_exact_match.py @@ -71,3 +71,12 @@ def test_run_with_different_lengths(): with pytest.raises(ValueError): evaluator.run(ground_truth_answers=["Berlin", "Paris"], predicted_answers=["Berlin"]) + + +def test_run_with_empty_inputs(): + evaluator = AnswerExactMatchEvaluator() + + # Empty (equal-length) inputs previously fell through the length check and + # crashed with a bare ZeroDivisionError when averaging over zero items. + with pytest.raises(ValueError, match="must be provided"): + evaluator.run(ground_truth_answers=[], predicted_answers=[]) diff --git a/test/components/evaluators/test_document_map.py b/test/components/evaluators/test_document_map.py index fcd54804b6..2041f0accc 100644 --- a/test/components/evaluators/test_document_map.py +++ b/test/components/evaluators/test_document_map.py @@ -147,3 +147,12 @@ def test_run_with_different_lengths(): ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]], retrieved_documents=[[Document(content="Berlin")]], ) + + +def test_run_with_empty_inputs(): + evaluator = DocumentMAPEvaluator() + + # Empty (equal-length) inputs previously fell through the length check and + # crashed with a bare ZeroDivisionError when averaging over zero items. + with pytest.raises(ValueError, match="must be provided"): + evaluator.run(ground_truth_documents=[], retrieved_documents=[]) diff --git a/test/components/evaluators/test_document_mrr.py b/test/components/evaluators/test_document_mrr.py index af41d16e9a..2c15a75398 100644 --- a/test/components/evaluators/test_document_mrr.py +++ b/test/components/evaluators/test_document_mrr.py @@ -150,3 +150,12 @@ def test_run_with_different_lengths(): ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]], retrieved_documents=[[Document(content="Berlin")]], ) + + +def test_run_with_empty_inputs(): + evaluator = DocumentMRREvaluator() + + # Empty (equal-length) inputs previously fell through the length check and + # crashed with a bare ZeroDivisionError when averaging over zero items. + with pytest.raises(ValueError, match="must be provided"): + evaluator.run(ground_truth_documents=[], retrieved_documents=[]) diff --git a/test/components/evaluators/test_document_recall.py b/test/components/evaluators/test_document_recall.py index 05eb94dc91..b04124c7c6 100644 --- a/test/components/evaluators/test_document_recall.py +++ b/test/components/evaluators/test_document_recall.py @@ -68,6 +68,15 @@ def test_run_with_nested_meta_comparison(): assert result == {"individual_scores": [0.5], "score": 0.5} +def test_run_with_empty_inputs(): + evaluator = DocumentRecallEvaluator() + + # Empty (equal-length) inputs previously fell through the length check and + # crashed with a bare ZeroDivisionError when averaging over zero items. + with pytest.raises(ValueError, match="must be provided"): + evaluator.run(ground_truth_documents=[], retrieved_documents=[]) + + class TestDocumentRecallEvaluatorSingleHit: @pytest.fixture def evaluator(self):