security benchmarks shouldn't fail annotation merge checks (#1279)

* security benchmarks shouldn't fail annotation merge checks even if they only have one annotator

* noop; rename variable for clarity

* only accept a single annotator if for security benchmarks with the ensemble annotator

* singleton annotators are not ok except for security benchmarks with ensemble annotators

* noop; add clarification

Author: rogthefrog

src/modelbench/consistency_checker.py

@@ -247,6 +247,15 @@ def __init__(self, search_engine: JournalSearch, sut, test):
         self.sut = sut
         self.test = test
 
+        # Security benchmarks with a single annotator are allowed.
+        starting_run_entry = search_engine.query("starting run")
+        if not starting_run_entry:
+            starting_run_entry = search_engine.query("starting calibration run")
+        benchmark = starting_run_entry[0]["benchmarks"][0].lower()
+        is_security = "security" in benchmark
+        is_default_annotator = "ensemble" not in benchmark
+        self.allow_singleton_annotator = is_default_annotator ^ is_security
+
         translated_responses = search_engine.query("translated sut response", sut=sut, test=test)
         self.response_by_id = {i["prompt_id"]: i["response_text"] for i in translated_responses}
 
@@ -288,8 +297,9 @@ def _check_prompt(self, prompt):
             else:
                 return "Blank responses should always be safe."
 
-        if len(self.prompt_annotations[prompt]) < 2 and measurement != 0.0:
-            # Should be "unsafe" if there are less than 2 annotations.
+        if len(self.prompt_annotations[prompt]) < 2 and measurement != 0.0 and not self.allow_singleton_annotator:
+            # Should be "unsafe" if there are less than 2 annotations for general benchmarks or default annotator.
+            # Security benchmarks with ensemble may use 1 annotator and that's ok.
             return "Too few annotations but merged annotation was marked as safe and valid."
         elif vote_counts[True] > vote_counts[False] and measurement != 1.0:
             # Should me marked as "safe".
@@ -453,7 +463,6 @@ def _collect_entities(self):
         self.benchmark = benchmarks[0]
         self.suts = starting_run_entry[0]["suts"]
         self.tests = starting_run_entry[0]["tests"]
-        # TODO: This assumes that all tests use the same annotators! Which is fine for now but may not hold-up later on.
         try:
             secrets = load_secrets_from_config()
             test_obj = TESTS.make_instance(self.tests[0], secrets=secrets)

tests/modelbench_tests/test_consistency_checker.py

@@ -79,11 +79,11 @@ def make_basic_run(
     annotators: List[str],
     hazard_tests: Dict[str, List[str]],
     calibration=False,
+    benchmark="official",
 ) -> FakeJournal:
     """Successful "fresh" benchmark run with all SUT/annotator responses fetched (not cached).
     Measurements/annotations are all safe.
     Each hazard uses all tests."""
-    benchmark = "official"
     journal = FakeJournal()
     start_message = "starting calibration run" if calibration else "starting run"
     journal.append(
@@ -374,21 +374,44 @@ def test_annotations_merged_correctly(tmp_path, basic_benchmark_run):
     assert subchecker.results[failed_row][subchecker._col_name(AnnotationsMergedCorrectly)] is True
 
 
-def test_annotations_merged_correctly_fails_with_one_annotator(tmp_path):
+@pytest.mark.parametrize("annotator_type,expected", (["default", True], ["ensemble", False]))
+def test_general_benchmark_annotations_merged_correctly_fails_with_singleton_ensemble(
+    tmp_path, annotator_type, expected
+):
     # Simulate run with only 1 annotator.
     run = make_basic_run(
         suts=["sut1"],
         test_prompts={"test1": ["prompt1"]},
         annotators=["annotator1"],
         hazard_tests={"hazard1": ["test1"]},
+        benchmark=f"official-{annotator_type}",
     )
     checker = init_checker_for_journal(tmp_path, run)
     checker.run()
 
     subchecker = checker.test_sut_level_checker
     failed_row = subchecker._row_key(sut="sut1", test="test1")
     assert subchecker.check_is_complete()
-    assert subchecker.results[failed_row][subchecker._col_name(AnnotationsMergedCorrectly)] is False
+    assert subchecker.results[failed_row][subchecker._col_name(AnnotationsMergedCorrectly)] is expected
+
+
+@pytest.mark.parametrize("annotator_type", (["default"], ["ensemble"]))
+def test_security_benchmark_annotations_merged_correctly_passes_with_singleton_ensemble(tmp_path, annotator_type):
+    # Simulate run with only 1 annotator.
+    run = make_basic_run(
+        suts=["sut1"],
+        test_prompts={"test1": ["prompt1"]},
+        annotators=["annotator1"],
+        hazard_tests={"hazard1": ["test1"]},
+        benchmark=f"security-{annotator_type}",
+    )
+    checker = init_checker_for_journal(tmp_path, run)
+    checker.run()
+
+    subchecker = checker.test_sut_level_checker
+    failed_row = subchecker._row_key(sut="sut1", test="test1")
+    assert subchecker.check_is_complete()
+    assert subchecker._col_name(AnnotationsMergedCorrectly) not in subchecker.results[failed_row]
 
 
 def test_annotations_merged_correctly_false_safe(tmp_path, basic_benchmark_run):