Skip to content

Commit 7e8518a

Browse files
rlundeen2rlundeen2
authored
[MAINT BREAKING]: Renaming scenarios (microsoft#1301)
1 parent 698d848 commit 7e8518a

41 files changed

Lines changed: 599 additions & 415 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎build_scripts/validate_jupyter_book.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,21 @@ def validate_api_rst_modules(modules: List[Tuple[str, List[str]]], repo_root: Pa
7979
repo_root / module_path / "__init__.py",
8080
]
8181

82+
# For pyrit.scenario.* modules, also check in pyrit.scenario.scenarios.*
83+
# These are virtual modules registered via sys.modules aliasing
84+
if module_name.startswith("pyrit.scenario.") and module_name != "pyrit.scenario.scenarios":
85+
# e.g., pyrit.scenario.airt -> pyrit.scenario.scenarios.airt
86+
scenarios_path = module_name.replace("pyrit.scenario.", "pyrit.scenario.scenarios.", 1)
87+
scenarios_module_path = scenarios_path.replace(".", os.sep)
88+
possible_paths.extend(
89+
[
90+
repo_root / f"{scenarios_module_path}.py",
91+
repo_root / scenarios_module_path / "__init__.py",
92+
]
93+
)
94+
8295
module_exists = any(p.exists() for p in possible_paths)
96+
8397
if not module_exists:
8498
errors.append(f"Module file not found for '{module_name}': checked {[str(p) for p in possible_paths]}")
8599
continue

‎doc/api.rst‎

Lines changed: 47 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -582,12 +582,56 @@ API Reference
582582
:toctree: _autosummary/
583583

584584
AtomicAttack
585-
EncodingScenario
586-
FoundryStrategy
587-
FoundryScenario
585+
DatasetConfiguration
588586
Scenario
587+
ScenarioCompositeStrategy
589588
ScenarioStrategy
590589

590+
:py:mod:`pyrit.scenario.airt`
591+
=============================
592+
593+
.. automodule:: pyrit.scenario.airt
594+
:no-members:
595+
:no-inherited-members:
596+
597+
.. autosummary::
598+
:nosignatures:
599+
:toctree: _autosummary/
600+
601+
ContentHarms
602+
ContentHarmsStrategy
603+
Cyber
604+
CyberStrategy
605+
606+
:py:mod:`pyrit.scenario.foundry`
607+
================================
608+
609+
.. automodule:: pyrit.scenario.foundry
610+
:no-members:
611+
:no-inherited-members:
612+
613+
.. autosummary::
614+
:nosignatures:
615+
:toctree: _autosummary/
616+
617+
Foundry
618+
FoundryScenario
619+
FoundryStrategy
620+
621+
:py:mod:`pyrit.scenario.garak`
622+
==============================
623+
624+
.. automodule:: pyrit.scenario.garak
625+
:no-members:
626+
:no-inherited-members:
627+
628+
.. autosummary::
629+
:nosignatures:
630+
:toctree: _autosummary/
631+
632+
Encoding
633+
EncodingStrategy
634+
591635
:py:mod:`pyrit.setup`
592636
=====================
593637

‎doc/code/front_end/1_pyrit_scan.ipynb‎

Lines changed: 94 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,12 @@
3131
" [--list-initializers] [--database DATABASE]\n",
3232
" [--initializers INITIALIZERS [INITIALIZERS ...]]\n",
3333
" [--initialization-scripts INITIALIZATION_SCRIPTS [INITIALIZATION_SCRIPTS ...]]\n",
34+
" [--env-files ENV_FILES [ENV_FILES ...]]\n",
3435
" [--strategies SCENARIO_STRATEGIES [SCENARIO_STRATEGIES ...]]\n",
3536
" [--max-concurrency MAX_CONCURRENCY]\n",
3637
" [--max-retries MAX_RETRIES] [--memory-labels MEMORY_LABELS]\n",
38+
" [--dataset-names DATASET_NAMES [DATASET_NAMES ...]]\n",
39+
" [--max-dataset-size MAX_DATASET_SIZE]\n",
3740
" [scenario_name]\n",
3841
"\n",
3942
"PyRIT Scanner - Run security scenarios against AI systems\n",
@@ -44,15 +47,15 @@
4447
" pyrit_scan --list-initializers\n",
4548
"\n",
4649
" # Run a scenario with built-in initializers\n",
47-
" pyrit_scan foundry_scenario --initializers openai_objective_target load_default_datasets\n",
50+
" pyrit_scan foundry --initializers openai_objective_target load_default_datasets\n",
4851
"\n",
4952
" # Run with custom initialization scripts\n",
50-
" pyrit_scan garak.encoding_scenario --initialization-scripts ./my_config.py\n",
53+
" pyrit_scan garak.encoding --initialization-scripts ./my_config.py\n",
5154
"\n",
5255
" # Run specific strategies or options\n",
53-
" pyrit scan foundry_scenario --strategies base64 rot13 --initializers openai_objective_target\n",
54-
" pyrit_scan foundry_scenario --initializers openai_objective_target --max-concurrency 10 --max-retries 3\n",
55-
" pyrit_scan garak.encoding_scenario --initializers openai_objective_target --memory-labels '{\"run_id\":\"test123\"}'\n",
56+
" pyrit scan foundry --strategies base64 rot13 --initializers openai_objective_target\n",
57+
" pyrit_scan foundry --initializers openai_objective_target --max-concurrency 10 --max-retries 3\n",
58+
" pyrit_scan garak.encoding --initializers openai_objective_target --memory-labels '{\"run_id\":\"test123\"}'\n",
5659
"\n",
5760
"positional arguments:\n",
5861
" scenario_name Name of the scenario to run\n",
@@ -72,7 +75,11 @@
7275
" --initialization-scripts INITIALIZATION_SCRIPTS [INITIALIZATION_SCRIPTS ...]\n",
7376
" Paths to custom Python initialization scripts to run\n",
7477
" before the scenario\n",
75-
" --strategies SCENARIO_STRATEGIES [SCENARIO_STRATEGIES ...], -s SCENARIO_STRATEGIES [SCENARIO_STRATEGIES ...]\n",
78+
" --env-files ENV_FILES [ENV_FILES ...]\n",
79+
" Paths to environment files to load in order (e.g.,\n",
80+
" .env.production .env.local). Later files override\n",
81+
" earlier ones.\n",
82+
" --strategies, -s SCENARIO_STRATEGIES [SCENARIO_STRATEGIES ...]\n",
7683
" List of strategy names to run (e.g., base64 rot13)\n",
7784
" --max-concurrency MAX_CONCURRENCY\n",
7885
" Maximum number of concurrent attack executions (must\n",
@@ -82,7 +89,16 @@
8289
" be >= 0)\n",
8390
" --memory-labels MEMORY_LABELS\n",
8491
" Additional labels as JSON string (e.g.,\n",
85-
" '{\"experiment\": \"test1\"}')\n"
92+
" '{\"experiment\": \"test1\"}')\n",
93+
" --dataset-names DATASET_NAMES [DATASET_NAMES ...]\n",
94+
" List of dataset names to use instead of scenario\n",
95+
" defaults (e.g., harmbench advbench). Creates a new\n",
96+
" dataset config; fetches all items unless --max-\n",
97+
" dataset-size is also specified\n",
98+
" --max-dataset-size MAX_DATASET_SIZE\n",
99+
" Maximum number of items to use from the dataset (must\n",
100+
" be >= 1). Limits new datasets if --dataset-names\n",
101+
" provided, otherwise overrides scenario's default limit\n"
86102
]
87103
}
88104
],
@@ -111,12 +127,15 @@
111127
"output_type": "stream",
112128
"text": [
113129
"Starting PyRIT...\n",
130+
"Found default environment files: ['C:\\\\Users\\\\rlundeen\\\\.pyrit\\\\.env', 'C:\\\\Users\\\\rlundeen\\\\.pyrit\\\\.env.local']\n",
131+
"Loaded environment file: C:\\Users\\rlundeen\\.pyrit\\.env\n",
132+
"Loaded environment file: C:\\Users\\rlundeen\\.pyrit\\.env.local\n",
114133
"\n",
115134
"Available Scenarios:\n",
116135
"================================================================================\n",
117136
"\u001b[1m\u001b[36m\n",
118-
" airt.content_harms_scenario\u001b[0m\n",
119-
" Class: ContentHarmsScenario\n",
137+
" airt.content_harms\u001b[0m\n",
138+
" Class: ContentHarms\n",
120139
" Description:\n",
121140
" Content Harms Scenario implementation for PyRIT. This scenario contains\n",
122141
" various harm-based checks that you can run to get a quick idea about\n",
@@ -126,51 +145,43 @@
126145
" Available Strategies (7):\n",
127146
" hate, fairness, violence, sexual, harassment, misinformation, leakage\n",
128147
" Default Strategy: all\n",
129-
" Required Datasets (7):\n",
148+
" Default Datasets (7, max 4 per dataset):\n",
130149
" airt_hate, airt_fairness, airt_violence, airt_sexual, airt_harassment,\n",
131150
" airt_misinformation, airt_leakage\n",
132151
"\u001b[1m\u001b[36m\n",
133-
" airt.cyber_scenario\u001b[0m\n",
134-
" Class: CyberScenario\n",
152+
" airt.cyber\u001b[0m\n",
153+
" Class: Cyber\n",
135154
" Description:\n",
136155
" Cyber scenario implementation for PyRIT. This scenario tests how willing\n",
137156
" models are to exploit cybersecurity harms by generating malware. The\n",
138-
" CyberScenario class contains different variations of the malware\n",
139-
" generation techniques.\n",
157+
" Cyber class contains different variations of the malware generation\n",
158+
" techniques.\n",
140159
" Aggregate Strategies:\n",
141160
" - all\n",
142161
" Available Strategies (2):\n",
143162
" single_turn, multi_turn\n",
144163
" Default Strategy: all\n",
145-
" Required Datasets (1):\n",
164+
" Default Datasets (1, max 4 per dataset):\n",
146165
" airt_malware\n",
147166
"\u001b[1m\u001b[36m\n",
148-
" foundry_scenario\u001b[0m\n",
167+
" foundry\u001b[0m\n",
149168
" Class: FoundryScenario\n",
150169
" Description:\n",
151-
" FoundryScenario is a preconfigured scenario that automatically generates\n",
152-
" multiple AtomicAttack instances based on the specified attack\n",
153-
" strategies. It supports both single-turn attacks (with various\n",
154-
" converters) and multi-turn attacks (Crescendo, RedTeaming), making it\n",
155-
" easy to quickly test a target against multiple attack vectors. The\n",
156-
" scenario can expand difficulty levels (EASY, MODERATE, DIFFICULT) into\n",
157-
" their constituent attack strategies, or you can specify individual\n",
158-
" strategies directly. Note this is not the same as the Foundry AI Red\n",
159-
" Teaming Agent. This is a PyRIT contract so their library can make use of\n",
160-
" PyRIT in a consistent way.\n",
170+
" Deprecated alias for Foundry. This class is deprecated and will be\n",
171+
" removed in version 0.13.0. Use `Foundry` instead.\n",
161172
" Aggregate Strategies:\n",
162173
" - all, easy, moderate, difficult\n",
163-
" Available Strategies (23):\n",
174+
" Available Strategies (25):\n",
164175
" ansi_attack, ascii_art, ascii_smuggler, atbash, base64, binary, caesar,\n",
165176
" character_space, char_swap, diacritic, flip, leetspeak, morse, rot13,\n",
166177
" suffix_append, string_join, unicode_confusable, unicode_substitution,\n",
167-
" url, jailbreak, tense, multi_turn, crescendo\n",
178+
" url, jailbreak, tense, multi_turn, crescendo, pair, tap\n",
168179
" Default Strategy: easy\n",
169-
" Required Datasets (1):\n",
180+
" Default Datasets (1, max 4 per dataset):\n",
170181
" harmbench\n",
171182
"\u001b[1m\u001b[36m\n",
172-
" garak.encoding_scenario\u001b[0m\n",
173-
" Class: EncodingScenario\n",
183+
" garak.encoding\u001b[0m\n",
184+
" Class: Encoding\n",
174185
" Description:\n",
175186
" Encoding Scenario implementation for PyRIT. This scenario tests how\n",
176187
" resilient models are to various encoding attacks by encoding potentially\n",
@@ -189,7 +200,7 @@
189200
" uuencode, rot13, braille, atbash, morse_code, nato, ecoji, zalgo,\n",
190201
" leet_speak, ascii_smuggler\n",
191202
" Default Strategy: all\n",
192-
" Required Datasets (2):\n",
203+
" Default Datasets (2, max 3 per dataset):\n",
193204
" garak_slur_terms_en, garak_web_html_js\n",
194205
"\n",
195206
"================================================================================\n",
@@ -309,7 +320,7 @@
309320
"Or concretely:\n",
310321
"\n",
311322
"```shell\n",
312-
"!pyrit_scan foundry_scenario --initializers simple openai_objective_target --scenario-strategies base64\n",
323+
"!pyrit_scan foundry --initializers simple openai_objective_target --scenario-strategies base64\n",
313324
"```\n",
314325
"\n",
315326
"Example with a basic configuration that runs the Foundry scenario against the objective target defined in `openai_objective_target` (which just is an OpenAIChatTarget with `DEFAULT_OPENAI_FRONTEND_ENDPOINT` and `DEFAULT_OPENAI_FRONTEND_KEY`)."
@@ -326,29 +337,22 @@
326337
"output_type": "stream",
327338
"text": [
328339
"Starting PyRIT...\n",
340+
"Found default environment files: ['C:\\\\Users\\\\rlundeen\\\\.pyrit\\\\.env', 'C:\\\\Users\\\\rlundeen\\\\.pyrit\\\\.env.local']\n",
341+
"Loaded environment file: C:\\Users\\rlundeen\\.pyrit\\.env\n",
342+
"Loaded environment file: C:\\Users\\rlundeen\\.pyrit\\.env.local\n",
329343
"Running 1 initializer(s)...\n",
344+
"Found default environment files: ['C:\\\\Users\\\\rlundeen\\\\.pyrit\\\\.env', 'C:\\\\Users\\\\rlundeen\\\\.pyrit\\\\.env.local']\n",
345+
"Loaded environment file: C:\\Users\\rlundeen\\.pyrit\\.env\n",
346+
"Loaded environment file: C:\\Users\\rlundeen\\.pyrit\\.env.local\n",
330347
"\n",
331-
"Running scenario: foundry_scenario\n",
348+
"Running scenario: foundry\n",
332349
"\n",
333-
"\u001b[36m====================================================================================================\u001b[0m\n",
334-
"\n",
335-
"Error: 'charmap' codec can't encode character '\\U0001f4ca' in position 41: character maps to <undefined>\n"
336-
]
337-
},
338-
{
339-
"name": "stderr",
340-
"output_type": "stream",
341-
"text": [
342-
"\n",
343-
"Executing Foundry Scenario: 0%| | 0/2 [00:00<?, ?attack/s]\n",
344-
"Executing Foundry Scenario: 50%|##### | 1/2 [00:39<00:39, 39.30s/attack]\n",
345-
"Executing Foundry Scenario: 100%|##########| 2/2 [01:12<00:00, 35.76s/attack]\n",
346-
"Executing Foundry Scenario: 100%|##########| 2/2 [01:12<00:00, 36.29s/attack]\n"
350+
"Error: SeedGroup at index 0 is missing an objective. Use seed_group.set_objective(value) to set one.\n"
347351
]
348352
}
349353
],
350354
"source": [
351-
"!pyrit_scan foundry_scenario --initializers openai_objective_target --strategies base64"
355+
"!pyrit_scan foundry --initializers openai_objective_target --strategies base64"
352356
]
353357
},
354358
{
@@ -359,17 +363,17 @@
359363
"Or with all options and multiple initializers and multiple strategies:\n",
360364
"\n",
361365
"```shell\n",
362-
"pyrit_scan foundry_scenario --database InMemory --initializers simple objective_target objective_list --scenario-strategies easy crescendo\n",
366+
"pyrit_scan foundry --database InMemory --initializers simple objective_target objective_list --scenario-strategies easy crescendo\n",
363367
"```\n",
364368
"\n",
365369
"You can also override scenario execution parameters:\n",
366370
"\n",
367371
"```shell\n",
368372
"# Override concurrency and retry settings\n",
369-
"pyrit_scan foundry_scenario --initializers simple objective_target --max-concurrency 10 --max-retries 3\n",
373+
"pyrit_scan foundry --initializers simple objective_target --max-concurrency 10 --max-retries 3\n",
370374
"\n",
371375
"# Add custom memory labels for tracking (must be valid JSON)\n",
372-
"pyrit_scan foundry_scenario --initializers simple objective_target --memory-labels '{\"experiment\": \"test1\", \"version\": \"v2\", \"researcher\": \"alice\"}'\n",
376+
"pyrit_scan foundry --initializers simple objective_target --memory-labels '{\"experiment\": \"test1\", \"version\": \"v2\", \"researcher\": \"alice\"}'\n",
373377
"```\n",
374378
"\n",
375379
"Available CLI parameter overrides:\n",
@@ -401,12 +405,33 @@
401405
"metadata": {
402406
"lines_to_next_cell": 2
403407
},
404-
"outputs": [],
408+
"outputs": [
409+
{
410+
"name": "stdout",
411+
"output_type": "stream",
412+
"text": [
413+
"Found default environment files: ['C:\\\\Users\\\\rlundeen\\\\.pyrit\\\\.env', 'C:\\\\Users\\\\rlundeen\\\\.pyrit\\\\.env.local']\n",
414+
"Loaded environment file: C:\\Users\\rlundeen\\.pyrit\\.env\n",
415+
"Loaded environment file: C:\\Users\\rlundeen\\.pyrit\\.env.local\n"
416+
]
417+
},
418+
{
419+
"data": {
420+
"text/plain": [
421+
"<__main__.MyCustomScenario at 0x19e7c2a70e0>"
422+
]
423+
},
424+
"execution_count": null,
425+
"metadata": {},
426+
"output_type": "execute_result"
427+
}
428+
],
405429
"source": [
406430
"# my_custom_scenarios.py\n",
407-
"from pyrit.common.apply_defaults import apply_defaults\n",
408-
"from pyrit.scenario import Scenario\n",
431+
"from pyrit.common import apply_defaults\n",
432+
"from pyrit.scenario import DatasetConfiguration, Scenario\n",
409433
"from pyrit.scenario.core.scenario_strategy import ScenarioStrategy\n",
434+
"from pyrit.setup import initialize_pyrit_async\n",
410435
"\n",
411436
"\n",
412437
"class MyCustomStrategy(ScenarioStrategy):\n",
@@ -417,7 +442,6 @@
417442
" Strategy2 = (\"strategy2\", set[str]())\n",
418443
"\n",
419444
"\n",
420-
"@apply_defaults\n",
421445
"class MyCustomScenario(Scenario):\n",
422446
" \"\"\"My custom scenario that does XYZ.\"\"\"\n",
423447
"\n",
@@ -429,6 +453,12 @@
429453
" def get_default_strategy(cls):\n",
430454
" return MyCustomStrategy.ALL\n",
431455
"\n",
456+
" @classmethod\n",
457+
" def default_dataset_config(cls) -> DatasetConfiguration:\n",
458+
" # Return default dataset configuration for this scenario\n",
459+
" return DatasetConfiguration(dataset_names=[\"harmbench\"])\n",
460+
"\n",
461+
" @apply_defaults\n",
432462
" def __init__(self, *, scenario_result_id=None, **kwargs):\n",
433463
" # Scenario-specific configuration only - no runtime parameters\n",
434464
" super().__init__(\n",
@@ -440,8 +470,13 @@
440470
" # ... your scenario-specific initialization code\n",
441471
"\n",
442472
" async def _get_atomic_attacks_async(self):\n",
443-
" # Build and return your atomic attacks\n",
444-
" return []"
473+
" # Build and return your atomic attacks based on self._scenario_composites\n",
474+
" # Example: create attacks for each strategy composite\n",
475+
" return []\n",
476+
"\n",
477+
"\n",
478+
"await initialize_pyrit_async(memory_db_type=\"InMemory\") # type: ignore\n",
479+
"MyCustomScenario()"
445480
]
446481
},
447482
{
@@ -474,7 +509,7 @@
474509
"name": "python",
475510
"nbconvert_exporter": "python",
476511
"pygments_lexer": "ipython3",
477-
"version": "3.12.11"
512+
"version": "3.13.5"
478513
}
479514
},
480515
"nbformat": 4,

0 commit comments

Comments
 (0)