GraniteGuardian-based metric for attaq

Signed-off-by: Jonathan Bnayahu <bnayahu@il.ibm.com>

Author: bnayahu

prepare/benchmarks/safety.py

@@ -4,10 +4,27 @@
 
 benchmark = Benchmark(
     subsets={
-        "attaq": DatasetRecipe(card="cards.attaq"),
-        "provoq": DatasetRecipe(card="cards.safety.provoq"),
-        "airbench": DatasetRecipe(card="cards.safety.airbench2024"),
-        "ailuminate": DatasetRecipe(card="cards.safety.mlcommons_ailuminate"),
+        "attaq": DatasetRecipe(
+            card="cards.safety.attaq_gg",
+            template_card_index="default",
+            max_test_instances=500,
+        ),
+        "provoq": DatasetRecipe(
+            card="cards.safety.provoq",
+            template_card_index="default",
+            group_by=["group"],
+            max_test_instances=500,
+        ),
+        "airbench": DatasetRecipe(
+            card="cards.safety.airbench2024",
+            template_card_index="default",
+            max_test_instances=500,
+        ),
+        "ailuminate": DatasetRecipe(
+            card="cards.safety.mlcommons_ailuminate",
+            template_card_index="default",
+            max_test_instances=500,
+        ),
     }
 )
 

prepare/cards/safety/attaq_gg.py

@@ -0,0 +1,27 @@
+from unitxt import add_to_catalog
+from unitxt.blocks import InputOutputTemplate, LoadHF, Task, TaskCard, TemplatesDict
+from unitxt.operators import Shuffle
+from unitxt.splitters import RenameSplits
+from unitxt.test_utils.card import test_card
+
+card = TaskCard(
+    loader=LoadHF(path="ibm/AttaQ"),
+    preprocess_steps=[
+        RenameSplits(mapper={"train": "test"}),
+        Shuffle(page_size=2800),
+    ],
+    task=Task(
+        input_fields={"input": str},
+        reference_fields={"label": str},
+        prediction_type=str,
+        metrics=[
+            "metrics.granite_guardian.assistant_risk.harm[prediction_type=str,user_message_field=input,assistant_message_field=prediction]",
+        ],
+    ),
+    templates=TemplatesDict(
+        {"default": InputOutputTemplate(input_format="{input}", output_format="")}
+    ),
+)
+
+test_card(card, strict=False, demos_taken_from="test", num_demos=0)
+add_to_catalog(card, "cards.safety.attaq_gg", overwrite=True)

src/unitxt/catalog/benchmarks/safety.json

@@ -3,19 +3,30 @@
     "subsets": {
         "attaq": {
             "__type__": "dataset_recipe",
-            "card": "cards.attaq"
+            "card": "cards.safety.attaq_gg",
+            "template_card_index": "default",
+            "max_test_instances": 500
         },
         "provoq": {
             "__type__": "dataset_recipe",
-            "card": "cards.safety.provoq"
+            "card": "cards.safety.provoq",
+            "template_card_index": "default",
+            "group_by": [
+                "group"
+            ],
+            "max_test_instances": 500
         },
         "airbench": {
             "__type__": "dataset_recipe",
-            "card": "cards.safety.airbench2024"
+            "card": "cards.safety.airbench2024",
+            "template_card_index": "default",
+            "max_test_instances": 500
         },
         "ailuminate": {
             "__type__": "dataset_recipe",
-            "card": "cards.safety.mlcommons_ailuminate"
+            "card": "cards.safety.mlcommons_ailuminate",
+            "template_card_index": "default",
+            "max_test_instances": 500
         }
     }
 }

src/unitxt/catalog/cards/safety/attaq_gg.json

@@ -0,0 +1,42 @@
+{
+    "__type__": "task_card",
+    "loader": {
+        "__type__": "load_hf",
+        "path": "ibm/AttaQ"
+    },
+    "preprocess_steps": [
+        {
+            "__type__": "rename_splits",
+            "mapper": {
+                "train": "test"
+            }
+        },
+        {
+            "__type__": "shuffle",
+            "page_size": 2800
+        }
+    ],
+    "task": {
+        "__type__": "task",
+        "input_fields": {
+            "input": "str"
+        },
+        "reference_fields": {
+            "label": "str"
+        },
+        "prediction_type": "str",
+        "metrics": [
+            "metrics.granite_guardian.assistant_risk.harm[prediction_type=str,user_message_field=input,assistant_message_field=prediction]"
+        ]
+    },
+    "templates": {
+        "__type__": "templates_dict",
+        "items": {
+            "default": {
+                "__type__": "input_output_template",
+                "input_format": "{input}",
+                "output_format": ""
+            }
+        }
+    }
+}