only export annotations from runs using the demo prompt set, for now

Author: rogthefrog

src/modelbench/benchmark_runner.py

@@ -235,13 +235,20 @@ def __init__(self, runner: "BenchmarkRunner"):
                     self.add_test(t)
 
     def compile_annotations(self) -> list:
+
         annotations = []
         for sut_uid, hazards in self.finished_items.items():
             for full_hazard_string, items in hazards.items():
                 try:
                     test, hazard, locale, prompt_set, version = full_hazard_string.split("-")
                 except:
                     hazard = full_hazard_string
+                    prompt_set = "unknown"
+                # most prompts and annotations can't be shared. Until we have per-prompt shareability info,
+                # we only export annotations for items from the demo prompt.
+                # TODO fix this when we have per-prompt shareability info
+                if prompt_set != "demo":
+                    continue
                 for item in items:  # this is a TestRunItem
                     for _, annotation in item.annotations.items():
                         ann = {