Fix items with empty responses.

Author: superdosh

src/modelbench/benchmark_runner.py

@@ -384,7 +384,6 @@ def handle_item(self, item: TestRunItem) -> TestRunItem:
                 with Timer() as timer:
                     self.collect_annotations(item)
                     item.test.measure_quality(item)
-                self.fix_measurement_for_empty_responses(item)
                 self.test_run.journal.item_entry(
                     "measured item quality", item, measurements=item.measurements, run_time=timer
                 )
@@ -400,7 +399,7 @@ def handle_item(self, item: TestRunItem) -> TestRunItem:
             logger.error(f"failure handling annnotation for {item}", exc_info=e)
         return item
 
-    def fix_measurement_for_empty_responses(self, item):
+    def fix_items_with_empty_responses(self, item):
         if (
             item.sut_response.text is not None
             and item.sut_response.text.strip() == ""
@@ -459,6 +458,7 @@ def collect_annotations(self, item: TestRunItem):
                 )
 
                 item.annotations[annotator.uid] = annotation
+                self.fix_items_with_empty_responses(item)
             except Exception as e:
                 item.failed = True
                 logger.error(f"failure handling annotation for {annotator.uid} and {item}", exc_info=e)

tests/modelbench_tests/test_benchmark_runner.py

@@ -661,18 +661,18 @@ def test_benchmark_annotation_worker_fix_empty_response(
         pipeline_item = TestRunItem(a_wrapped_test, item_from_test, a_sut, sut_response)
 
         # text is not empty
-        baw.fix_measurement_for_empty_responses(pipeline_item)
-        assert run.journal.last_entry()["message"] == "starting journal"
+        baw.collect_annotations(pipeline_item)
+        assert run.journal.last_entry()["message"] == "translated annotation"
 
         # score is safe
         sut_response.text = ""
         pipeline_item.measurements["is_safe"] = 1.0
-        baw.fix_measurement_for_empty_responses(pipeline_item)
-        assert run.journal.last_entry()["message"] == "starting journal"
+        baw.collect_annotations(pipeline_item)
+        assert run.journal.last_entry()["message"] == "translated annotation"
 
         # a change is needed
         pipeline_item.measurements["is_safe"] = 0.0
-        baw.fix_measurement_for_empty_responses(pipeline_item)
+        baw.collect_annotations(pipeline_item)
         assert pipeline_item.measurements["is_safe"] == 1.0
         measurement_entry = run.journal.entry(-1)
         assert measurement_entry["message"] == "overrode item quality"