MAINT Scenario Namespace & Directory Changes (microsoft#1197)

Co-authored-by: Victor Valbuena <vvalbuena@microsoft.com>

Author: ValbuenaVC

doc/api.rst

@@ -531,10 +531,10 @@ API Reference
     TrueFalseScoreAggregator
     TrueFalseScorer
 
-:py:mod:`pyrit.scenarios`
+:py:mod:`pyrit.scenario`
 =========================
 
-.. automodule:: pyrit.scenarios
+.. automodule:: pyrit.scenario
     :no-members:
     :no-inherited-members:
 

doc/code/front_end/1_pyrit_scan.ipynb

@@ -247,7 +247,7 @@
     "\n",
     "You need a single scenario to run, you need two things:\n",
     "\n",
-    "1. A Scenario. Many are defined in `pyrit.scenarios.scenarios`. But you can also define your own in initialization_scripts.\n",
+    "1. A Scenario. Many are defined in `pyrit.scenario.scenarios`. But you can also define your own in initialization_scripts.\n",
     "2. Initializers (which can be supplied via `--initializers` or `--initialization-scripts`). Scenarios often don't need many arguments, but they can be configured in different ways. And at the very least, most need an `objective_target` (the thing you're running a scan against).\n",
     "3. Scenario Strategies (optional). These are supplied by the `--scenario-strategies` flag and tell the scenario what to test, but they are always optional. Also note you can obtain these by running `--list-scenarios`\n",
     "\n",
@@ -413,8 +413,8 @@
    "source": [
     "# my_custom_scenarios.py\n",
     "from pyrit.common.apply_defaults import apply_defaults\n",
-    "from pyrit.scenarios import Scenario\n",
-    "from pyrit.scenarios.scenario_strategy import ScenarioStrategy\n",
+    "from pyrit.scenario import Scenario\n",
+    "from pyrit.scenario.scenario_strategy import ScenarioStrategy\n",
     "\n",
     "\n",
     "class MyCustomStrategy(ScenarioStrategy):\n",

doc/code/front_end/1_pyrit_scan.py

@@ -59,7 +59,7 @@
 #
 # You need a single scenario to run, you need two things:
 #
-# 1. A Scenario. Many are defined in `pyrit.scenarios.scenarios`. But you can also define your own in initialization_scripts.
+# 1. A Scenario. Many are defined in `pyrit.scenario.scenarios`. But you can also define your own in initialization_scripts.
 # 2. Initializers (which can be supplied via `--initializers` or `--initialization-scripts`). Scenarios often don't need many arguments, but they can be configured in different ways. And at the very least, most need an `objective_target` (the thing you're running a scan against).
 # 3. Scenario Strategies (optional). These are supplied by the `--scenario-strategies` flag and tell the scenario what to test, but they are always optional. Also note you can obtain these by running `--list-scenarios`
 #
@@ -124,8 +124,8 @@
 
 # %%
 # my_custom_scenarios.py
-from pyrit.scenarios import Scenario
-from pyrit.scenarios.scenario_strategy import ScenarioStrategy
+from pyrit.scenario import Scenario
+from pyrit.scenario.scenario_strategy import ScenarioStrategy
 
 
 class MyCustomStrategy(ScenarioStrategy):

doc/code/scenarios/scenarios.ipynb

@@ -41,7 +41,7 @@
     "\n",
     "## Creating Custom Scenarios\n",
     "\n",
-    "To create a custom scenario, extend the `Scenario` base class. See [`FoundryScenario`](../../../pyrit/scenarios/scenarios/foundry_scenario.py) for an example.\n",
+    "To create a custom scenario, extend the `Scenario` base class. See [`FoundryScenario`](../../../pyrit/scenario/scenarios/foundry_scenario.py) for an example.\n",
     "\n",
     "## Using Scenarios\n",
     "\n",
@@ -64,7 +64,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "a1dd9e091cb741e59cd9408027ba6e2e",
+       "model_id": "a99d236b595c4934ad564e9562190ff0",
        "version_major": 2,
        "version_minor": 0
       },
@@ -101,7 +101,7 @@
       "\u001b[1m  🎯 Target Information\u001b[0m\n",
       "\u001b[36m    • Target Type: OpenAIChatTarget\u001b[0m\n",
       "\u001b[36m    • Target Model: gpt4o\u001b[0m\n",
-      "\u001b[36m    • Target Endpoint: https://pyrit-github-pipeline.openai.azure.com/openai/deployments/gpt-40/chat/completions?api-version=2024-10-21\u001b[0m\n",
+      "\u001b[36m    • Target Endpoint: https://pyrit-japan-test.openai.azure.com/openai/deployments/gpt-4o-japan-nilfilter/chat/completions?api-version=2024-10-21\u001b[0m\n",
       "\n",
       "\u001b[1m  📊 Scorer Information\u001b[0m\n",
       "\u001b[36m    • Scorer Type: TrueFalseCompositeScorer\u001b[0m\n",
@@ -118,7 +118,7 @@
       "\u001b[1m  📈 Summary\u001b[0m\n",
       "\u001b[32m    • Total Strategies: 4\u001b[0m\n",
       "\u001b[32m    • Total Attack Results: 8\u001b[0m\n",
-      "\u001b[32m    • Overall Success Rate: 12%\u001b[0m\n",
+      "\u001b[32m    • Overall Success Rate: 0%\u001b[0m\n",
       "\u001b[32m    • Unique Objectives: 2\u001b[0m\n",
       "\n",
       "\u001b[1m\u001b[36m▼ Per-Strategy Breakdown\u001b[0m\n",
@@ -130,7 +130,7 @@
       "\n",
       "\u001b[1m  🔸 Strategy: base64\u001b[0m\n",
       "\u001b[33m    • Number of Results: 2\u001b[0m\n",
-      "\u001b[33m    • Success Rate: 50%\u001b[0m\n",
+      "\u001b[32m    • Success Rate: 0%\u001b[0m\n",
       "\n",
       "\u001b[1m  🔸 Strategy: binary\u001b[0m\n",
       "\u001b[33m    • Number of Results: 2\u001b[0m\n",
@@ -147,8 +147,9 @@
    ],
    "source": [
     "from pyrit.prompt_target import OpenAIChatTarget\n",
-    "from pyrit.scenarios import FoundryScenario, FoundryStrategy, ScenarioCompositeStrategy\n",
-    "from pyrit.scenarios.printer.console_printer import ConsoleScenarioResultPrinter\n",
+    "from pyrit.scenario.core import ScenarioCompositeStrategy\n",
+    "from pyrit.scenario.printer.console_printer import ConsoleScenarioResultPrinter\n",
+    "from pyrit.scenario.scenarios import FoundryScenario, FoundryStrategy\n",
     "from pyrit.setup import IN_MEMORY, initialize_pyrit\n",
     "\n",
     "initialize_pyrit(\n",
@@ -174,10 +175,8 @@
     "\n",
     "\n",
     "# Create a scenario from the pre-configured Foundry scenario\n",
-    "foundry_scenario = FoundryScenario(\n",
-    "    objectives=objectives\n",
-    ")\n",
-    "await foundry_scenario.initialize_async( # type: ignore\n",
+    "foundry_scenario = FoundryScenario(objectives=objectives)\n",
+    "await foundry_scenario.initialize_async(  # type: ignore\n",
     "    objective_target=objective_target,\n",
     "    scenario_strategies=scenario_strategies,\n",
     "    max_concurrency=10,\n",
@@ -186,8 +185,8 @@
     "print(f\"Created scenario: {foundry_scenario.name}\")\n",
     "\n",
     "# Execute the entire scenario\n",
-    "foundry_results = await foundry_scenario.run_async() # type: ignore\n",
-    "await printer.print_summary_async(foundry_results) # type: ignore"
+    "foundry_results = await foundry_scenario.run_async()  # type: ignore\n",
+    "await printer.print_summary_async(foundry_results)  # type: ignore"
    ]
   },
   {
@@ -226,7 +225,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.11"
+   "version": "3.11.14"
   }
  },
  "nbformat": 4,

doc/code/scenarios/scenarios.py

@@ -45,7 +45,7 @@
 #
 # ## Creating Custom Scenarios
 #
-# To create a custom scenario, extend the `Scenario` base class. See [`FoundryScenario`](../../../pyrit/scenarios/scenarios/foundry_scenario.py) for an example.
+# To create a custom scenario, extend the `Scenario` base class. See [`FoundryScenario`](../../../pyrit/scenario/scenarios/foundry_scenario.py) for an example.
 #
 # ## Using Scenarios
 #
@@ -54,8 +54,9 @@
 
 # %%
 from pyrit.prompt_target import OpenAIChatTarget
-from pyrit.scenarios import FoundryScenario, FoundryStrategy, ScenarioCompositeStrategy
-from pyrit.scenarios.printer.console_printer import ConsoleScenarioResultPrinter
+from pyrit.scenario.core import ScenarioCompositeStrategy
+from pyrit.scenario.printer.console_printer import ConsoleScenarioResultPrinter
+from pyrit.scenario.scenarios import FoundryScenario, FoundryStrategy
 from pyrit.setup import IN_MEMORY, initialize_pyrit
 
 initialize_pyrit(

doc/code/setup/2_resiliency.ipynb

@@ -156,7 +156,7 @@
    ],
    "source": [
     "from pyrit.prompt_target import OpenAIChatTarget\n",
-    "from pyrit.scenarios import FoundryScenario, FoundryStrategy\n",
+    "from pyrit.scenario import FoundryScenario, FoundryStrategy\n",
     "from pyrit.setup import IN_MEMORY, initialize_pyrit\n",
     "\n",
     "initialize_pyrit(memory_db_type=IN_MEMORY)\n",

doc/code/setup/2_resiliency.py

@@ -124,7 +124,7 @@
 
 # %%
 from pyrit.prompt_target import OpenAIChatTarget
-from pyrit.scenarios import FoundryScenario, FoundryStrategy
+from pyrit.scenario import FoundryScenario, FoundryStrategy
 from pyrit.setup import IN_MEMORY, initialize_pyrit
 
 initialize_pyrit(memory_db_type=IN_MEMORY)

pyrit/cli/frontend_core.py

@@ -214,7 +214,7 @@ async def run_scenario_async(
     Note:
         Initializers from PyRITContext will be run before the scenario executes.
     """
-    from pyrit.scenarios.printer.console_printer import ConsoleScenarioResultPrinter
+    from pyrit.scenario.printer.console_printer import ConsoleScenarioResultPrinter
     from pyrit.setup import initialize_pyrit
 
     # Ensure context is initialized first (loads registries)

pyrit/cli/pyrit_shell.py

@@ -296,7 +296,7 @@ def do_print_scenario(self, arg):
                 print(f"\n{'#' * 80}")
                 print(f"Scenario Run #{idx}: {command}")
                 print(f"{'#' * 80}")
-                from pyrit.scenarios.printer.console_printer import (
+                from pyrit.scenario.printer.console_printer import (
                     ConsoleScenarioResultPrinter,
                 )
 
@@ -313,7 +313,7 @@ def do_print_scenario(self, arg):
                 command, result = self._scenario_history[scenario_num - 1]
                 print(f"\nScenario Run #{scenario_num}: {command}")
                 print("=" * 80)
-                from pyrit.scenarios.printer.console_printer import (
+                from pyrit.scenario.printer.console_printer import (
                     ConsoleScenarioResultPrinter,
                 )
 

pyrit/cli/scenario_registry.py

@@ -7,7 +7,7 @@
 Scenario registry for discovering and instantiating PyRIT scenarios.
 
 This module provides functionality to discover all available Scenario subclasses
-from the pyrit.scenarios.scenarios module and from user-defined initialization scripts.
+from the pyrit.scenario.scenarios module and from user-defined initialization scripts.
 
 PERFORMANCE OPTIMIZATION:
 This module uses lazy imports to minimize overhead during CLI operations:
@@ -35,7 +35,7 @@
 # Lazy import to avoid loading heavy scenario modules when just listing scenarios
 if TYPE_CHECKING:
     from pyrit.cli.frontend_core import ScenarioInfo
-    from pyrit.scenarios.scenario import Scenario
+    from pyrit.scenario.core import Scenario
 
 logger = logging.getLogger(__name__)
 
@@ -45,7 +45,7 @@ class ScenarioRegistry:
     Registry for discovering and managing available scenarios.
 
     This class discovers all Scenario subclasses from:
-    1. Built-in scenarios in pyrit.scenarios.scenarios module
+    1. Built-in scenarios in pyrit.scenario.scenarios module
     2. User-defined scenarios from initialization scripts (set via globals)
 
     Scenarios are identified by their simple name (e.g., "encoding_scenario", "foundry_scenario").
@@ -65,15 +65,15 @@ def _ensure_discovered(self) -> None:
 
     def _discover_builtin_scenarios(self) -> None:
         """
-        Discover all built-in scenarios from pyrit.scenarios.scenarios module.
+        Discover all built-in scenarios from pyrit.scenario.scenarios module.
 
         This method dynamically imports all modules in the scenarios package
         and registers any Scenario subclasses found.
         """
-        from pyrit.scenarios.scenario import Scenario
+        from pyrit.scenario.core import Scenario
 
         try:
-            import pyrit.scenarios.scenarios as scenarios_package
+            import pyrit.scenario.scenarios as scenarios_package
 
             # Get the path to the scenarios package
             package_file = scenarios_package.__file__
@@ -94,7 +94,7 @@ def _discover_builtin_scenarios(self) -> None:
 
                 try:
                     # Import the module
-                    full_module_name = f"pyrit.scenarios.scenarios.{module_name}"
+                    full_module_name = f"pyrit.scenario.scenarios.{module_name}"
                     module = importlib.import_module(full_module_name)
 
                     # Find all Scenario subclasses in the module
@@ -121,7 +121,7 @@ def discover_user_scenarios(self) -> None:
 
         User scenarios will override built-in scenarios with the same name.
         """
-        from pyrit.scenarios.scenario import Scenario
+        from pyrit.scenario.core import Scenario
 
         try:
             # Check the global namespace for Scenario subclasses
@@ -142,8 +142,8 @@ def discover_user_scenarios(self) -> None:
                 # Look for Scenario subclasses in the module
                 for name, obj in inspect.getmembers(module, inspect.isclass):
                     if issubclass(obj, Scenario) and obj is not Scenario:
-                        # Check if this is a user-defined class (not from pyrit.scenarios.scenarios)
-                        if not obj.__module__.startswith("pyrit.scenarios.scenarios"):
+                        # Check if this is a user-defined class (not from pyrit.scenario.scenarios)
+                        if not obj.__module__.startswith("pyrit.scenario.scenarios"):
                             # Convert class name to snake_case for scenario name
                             scenario_name = self._class_name_to_scenario_name(obj.__name__)
                             self._scenarios[scenario_name] = obj