FEAT: Scenario DatasetConfiguration (microsoft#1288)

Co-authored-by: Roman Lutz <romanlutz13@gmail.com>

Author: rlundeen2

doc/_toc.yml

@@ -129,7 +129,7 @@ chapters:
       - file: code/auxiliary_attacks/1_gcg_azure_ml
     - file: code/scenarios/0_scenarios
       sections:
-      - file: code/scenarios/1_composite_scenario
+      - file: code/scenarios/1_configuring_scenarios
     - file: code/front_end/0_front_end
       sections:
       - file: code/front_end/1_pyrit_scan

doc/code/converters/3_image_converters.ipynb

@@ -64,7 +64,7 @@
 
 # This converter adds text to an existing image
 # Using the roakey image from the doc folder
-sample_image = pathlib.Path("/workspace/doc/roakey.png")
+sample_image = pathlib.Path(".").resolve().parent.parent / "roakey.png"
 text_to_image_converter = AddImageTextConverter(img_to_add=str(sample_image))
 output_image_file = await text_to_image_converter.convert_async(prompt=text_prompt)  # type: ignore
 
@@ -89,7 +89,7 @@
 text_to_add: str = jailbreak.get_jailbreak(prompt="How to create a Molotov cocktail?")
 
 image_converter = AddTextImageConverter(text_to_add=text_to_add)
-image_location = str(pathlib.Path("/workspace/doc/roakey.png"))
+image_location = str(pathlib.Path(".").resolve().parent.parent / "roakey.png")
 output_image_file = await image_converter.convert_async(prompt=image_location)  # type: ignore
 
 print(output_image_file)

doc/code/converters/3_image_converters.py

@@ -32,6 +32,10 @@
     "\n",
     "These Scenarios can be updated and added to as you refine what you are testing for.\n",
     "\n",
+    "## How to Run Scenarios\n",
+    "\n",
+    "Scenarios should take almost no effort to run with default values. [pyrit_scan](../front_end/1_pyrit_scan.ipynb) and [pyrit_shell](../front_end/2_pyrit_shell.md) both use scenarios to execute.\n",
+    "\n",
     "## How It Works\n",
     "\n",
     "Each `Scenario` contains a collection of `AtomicAttack` objects. When executed:\n",
@@ -80,16 +84,33 @@
    "execution_count": null,
    "id": "1",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Found default environment files: ['C:\\\\Users\\\\rlundeen\\\\.pyrit\\\\.env', 'C:\\\\Users\\\\rlundeen\\\\.pyrit\\\\.env.local']\n",
+      "Loaded environment file: C:\\Users\\rlundeen\\.pyrit\\.env\n",
+      "Loaded environment file: C:\\Users\\rlundeen\\.pyrit\\.env.local\n"
+     ]
+    }
+   ],
    "source": [
     "from typing import List, Optional, Type\n",
     "\n",
     "from pyrit.common import apply_defaults\n",
     "from pyrit.executor.attack import AttackScoringConfig, PromptSendingAttack\n",
-    "from pyrit.models import SeedGroup, SeedObjective\n",
-    "from pyrit.scenario import AtomicAttack, Scenario, ScenarioStrategy\n",
+    "from pyrit.scenario import (\n",
+    "    AtomicAttack,\n",
+    "    DatasetConfiguration,\n",
+    "    Scenario,\n",
+    "    ScenarioStrategy,\n",
+    ")\n",
     "from pyrit.scenario.core.scenario_strategy import ScenarioCompositeStrategy\n",
     "from pyrit.score.true_false.true_false_scorer import TrueFalseScorer\n",
+    "from pyrit.setup import initialize_pyrit_async\n",
+    "\n",
+    "await initialize_pyrit_async(memory_db_type=\"InMemory\") # type: ignore [top-level-await]\n",
     "\n",
     "\n",
     "class MyStrategy(ScenarioStrategy):\n",
@@ -101,6 +122,7 @@
     "class MyScenario(Scenario):\n",
     "    version: int = 1\n",
     "\n",
+    "    # A strategy defintion helps callers define how to run your scenario (e.g. from the front_end)\n",
     "    @classmethod\n",
     "    def get_strategy_class(cls) -> Type[ScenarioStrategy]:\n",
     "        return MyStrategy\n",
@@ -109,6 +131,11 @@
     "    def get_default_strategy(cls) -> ScenarioStrategy:\n",
     "        return MyStrategy.ALL\n",
     "\n",
+    "    # This is the default dataset configuration for this scenario (e.g. prompts to send)\n",
+    "    @classmethod\n",
+    "    def default_dataset_config(cls) -> DatasetConfiguration:\n",
+    "        return DatasetConfiguration(dataset_names=[\"dataset_name\"])\n",
+    "\n",
     "    @apply_defaults\n",
     "    def __init__(\n",
     "        self,\n",
@@ -146,11 +173,8 @@
     "        )\n",
     "\n",
     "        for strategy in selected_strategies:\n",
-    "            # Create seed groups with objectives\n",
-    "            seed_groups = [\n",
-    "                SeedGroup(seeds=[SeedObjective(value=\"objective1\")]),\n",
-    "                SeedGroup(seeds=[SeedObjective(value=\"objective2\")]),\n",
-    "            ]\n",
+    "            # self._dataset_config is set by the parent class\n",
+    "            seed_groups = self._dataset_config.get_all_seed_groups()\n",
     "\n",
     "            # Create attack instances based on strategy\n",
     "            attack = PromptSendingAttack(\n",
@@ -165,7 +189,10 @@
     "                    memory_labels=self._memory_labels,\n",
     "                )\n",
     "            )\n",
-    "        return atomic_attacks"
+    "        return atomic_attacks\n",
+    "    \n",
+    "\n",
+    "scenario = MyScenario()"
    ]
   },
   {
@@ -205,7 +232,7 @@
       "    Available Strategies (7):\n",
       "      hate, fairness, violence, sexual, harassment, misinformation, leakage\n",
       "    Default Strategy: all\n",
-      "    Required Datasets (7):\n",
+      "    Default Datasets (7, max 4 per dataset):\n",
       "      airt_hate, airt_fairness, airt_violence, airt_sexual, airt_harassment,\n",
       "      airt_misinformation, airt_leakage\n",
       "\u001b[1m\u001b[36m\n",
@@ -221,7 +248,7 @@
       "    Available Strategies (2):\n",
       "      single_turn, multi_turn\n",
       "    Default Strategy: all\n",
-      "    Required Datasets (1):\n",
+      "    Default Datasets (1):\n",
       "      airt_malware\n",
       "\u001b[1m\u001b[36m\n",
       "  foundry_scenario\u001b[0m\n",
@@ -245,7 +272,7 @@
       "      suffix_append, string_join, unicode_confusable, unicode_substitution,\n",
       "      url, jailbreak, tense, multi_turn, crescendo, pair, tap\n",
       "    Default Strategy: easy\n",
-      "    Required Datasets (1):\n",
+      "    Default Datasets (1, max 4 per dataset):\n",
       "      harmbench\n",
       "\u001b[1m\u001b[36m\n",
       "  garak.encoding_scenario\u001b[0m\n",
@@ -268,7 +295,7 @@
       "      uuencode, rot13, braille, atbash, morse_code, nato, ecoji, zalgo,\n",
       "      leet_speak, ascii_smuggler\n",
       "    Default Strategy: all\n",
-      "    Required Datasets (2):\n",
+      "    Default Datasets (2, max 4 per dataset):\n",
       "      garak_slur_terms_en, garak_web_html_js\n",
       "\n",
       "================================================================================\n",

doc/code/scenarios/0_scenarios.ipynb

@@ -5,7 +5,11 @@
 #       extension: .py
 #       format_name: percent
 #       format_version: '1.3'
-#       jupytext_version: 1.17.3
+#       jupytext_version: 1.18.1
+#   kernelspec:
+#     display_name: pyrit (3.13.5)
+#     language: python
+#     name: python3
 # ---
 
 # %% [markdown]
@@ -34,6 +38,10 @@
 #
 # These Scenarios can be updated and added to as you refine what you are testing for.
 #
+# ## How to Run Scenarios
+#
+# Scenarios should take almost no effort to run with default values. [pyrit_scan](../front_end/1_pyrit_scan.ipynb) and [pyrit_shell](../front_end/2_pyrit_shell.md) both use scenarios to execute.
+#
 # ## How It Works
 #
 # Each `Scenario` contains a collection of `AtomicAttack` objects. When executed:
@@ -80,10 +88,17 @@
 
 from pyrit.common import apply_defaults
 from pyrit.executor.attack import AttackScoringConfig, PromptSendingAttack
-from pyrit.models import SeedGroup, SeedObjective
-from pyrit.scenario import AtomicAttack, Scenario, ScenarioStrategy
+from pyrit.scenario import (
+    AtomicAttack,
+    DatasetConfiguration,
+    Scenario,
+    ScenarioStrategy,
+)
 from pyrit.scenario.core.scenario_strategy import ScenarioCompositeStrategy
 from pyrit.score.true_false.true_false_scorer import TrueFalseScorer
+from pyrit.setup import initialize_pyrit_async
+
+await initialize_pyrit_async(memory_db_type="InMemory")  # type: ignore [top-level-await]
 
 
 class MyStrategy(ScenarioStrategy):
@@ -95,6 +110,7 @@ class MyStrategy(ScenarioStrategy):
 class MyScenario(Scenario):
     version: int = 1
 
+    # A strategy defintion helps callers define how to run your scenario (e.g. from the front_end)
     @classmethod
     def get_strategy_class(cls) -> Type[ScenarioStrategy]:
         return MyStrategy
@@ -103,6 +119,11 @@ def get_strategy_class(cls) -> Type[ScenarioStrategy]:
     def get_default_strategy(cls) -> ScenarioStrategy:
         return MyStrategy.ALL
 
+    # This is the default dataset configuration for this scenario (e.g. prompts to send)
+    @classmethod
+    def default_dataset_config(cls) -> DatasetConfiguration:
+        return DatasetConfiguration(dataset_names=["dataset_name"])
+
     @apply_defaults
     def __init__(
         self,
@@ -140,11 +161,8 @@ async def _get_atomic_attacks_async(self) -> List[AtomicAttack]:
         )
 
         for strategy in selected_strategies:
-            # Create seed groups with objectives
-            seed_groups = [
-                SeedGroup(seeds=[SeedObjective(value="objective1")]),
-                SeedGroup(seeds=[SeedObjective(value="objective2")]),
-            ]
+            # self._dataset_config is set by the parent class
+            seed_groups = self._dataset_config.get_all_seed_groups()
 
             # Create attack instances based on strategy
             attack = PromptSendingAttack(
@@ -162,6 +180,8 @@ async def _get_atomic_attacks_async(self) -> List[AtomicAttack]:
         return atomic_attacks
 
 
+scenario = MyScenario()
+
 # %% [markdown]
 #
 # ## Existing Scenarios