Skip to content

Commit 4fdbbd4

Browse files
rlundeen2rlundeen2
authored
MAINT: Creating an interface for scenarios (microsoft#1184)
1 parent d168b60 commit 4fdbbd4

17 files changed

Lines changed: 570 additions & 532 deletions

‎doc/code/front_end/0_cli.ipynb‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -286,7 +286,7 @@
286286
"\u001b[1m 🎯 Target Information\u001b[0m\n",
287287
"\u001b[36m • Target Type: OpenAIChatTarget\u001b[0m\n",
288288
"\u001b[36m • Target Model: gpt-4o\u001b[0m\n",
289-
"\u001b[36m • Target Endpoint: https://airt-blackhat-2-aoaio.openai.azure.com/openai/deployments/gpt-4o-blackhat/chat/completions?api-version=2025-01-01-preview\u001b[0m\n",
289+
"\u001b[36m • Target Endpoint: https://pyrit-github-pipeline.openai.azure.com/openai/deployments/gpt-40/chat/completions?api-version=2024-10-21\u001b[0m\n",
290290
"\n",
291291
"\u001b[1m 📊 Scorer Information\u001b[0m\n",
292292
"\u001b[36m • Scorer Type: TrueFalseCompositeScorer\u001b[0m\n",
@@ -301,14 +301,18 @@
301301
"\u001b[1m\u001b[36m▼ Overall Statistics\u001b[0m\n",
302302
"\u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n",
303303
"\u001b[1m 📈 Summary\u001b[0m\n",
304-
"\u001b[32m • Total Strategies: 1\u001b[0m\n",
305-
"\u001b[32m • Total Attack Results: 4\u001b[0m\n",
304+
"\u001b[32m • Total Strategies: 2\u001b[0m\n",
305+
"\u001b[32m • Total Attack Results: 8\u001b[0m\n",
306306
"\u001b[32m • Overall Success Rate: 0%\u001b[0m\n",
307307
"\u001b[32m • Unique Objectives: 4\u001b[0m\n",
308308
"\n",
309309
"\u001b[1m\u001b[36m▼ Per-Strategy Breakdown\u001b[0m\n",
310310
"\u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n",
311311
"\n",
312+
"\u001b[1m 🔸 Strategy: baseline\u001b[0m\n",
313+
"\u001b[33m • Number of Results: 4\u001b[0m\n",
314+
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
315+
"\n",
312316
"\u001b[1m 🔸 Strategy: base64\u001b[0m\n",
313317
"\u001b[33m • Number of Results: 4\u001b[0m\n",
314318
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
@@ -322,9 +326,10 @@
322326
"output_type": "stream",
323327
"text": [
324328
"\n",
325-
"Executing Foundry Scenario: 0%| | 0/1 [00:00<?, ?attack/s]\n",
326-
"Executing Foundry Scenario: 100%|██████████| 1/1 [00:15<00:00, 15.67s/attack]\n",
327-
"Executing Foundry Scenario: 100%|██████████| 1/1 [00:15<00:00, 15.67s/attack]\n"
329+
"Executing Foundry Scenario: 0%| | 0/2 [00:00<?, ?attack/s]\n",
330+
"Executing Foundry Scenario: 50%|█████ | 1/2 [00:17<00:17, 17.44s/attack]\n",
331+
"Executing Foundry Scenario: 100%|██████████| 2/2 [00:41<00:00, 21.11s/attack]\n",
332+
"Executing Foundry Scenario: 100%|██████████| 2/2 [00:41<00:00, 20.56s/attack]\n"
328333
]
329334
}
330335
],

‎doc/code/front_end/0_cli.py‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,6 @@
66
# format_name: percent
77
# format_version: '1.3'
88
# jupytext_version: 1.17.3
9-
# kernelspec:
10-
# display_name: pyrit-dev
11-
# language: python
12-
# name: python3
139
# ---
1410

1511
# %% [markdown]

‎doc/code/scenarios/scenarios.ipynb‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -64,12 +64,12 @@
6464
{
6565
"data": {
6666
"application/vnd.jupyter.widget-view+json": {
67-
"model_id": "ecb802c8f9964212b3f6c3ff7a416e79",
67+
"model_id": "a1dd9e091cb741e59cd9408027ba6e2e",
6868
"version_major": 2,
6969
"version_minor": 0
7070
},
7171
"text/plain": [
72-
"Executing Foundry Scenario: 0%| | 0/3 [00:00<?, ?attack/s]"
72+
"Executing Foundry Scenario: 0%| | 0/4 [00:00<?, ?attack/s]"
7373
]
7474
},
7575
"metadata": {},
@@ -101,7 +101,7 @@
101101
"\u001b[1m 🎯 Target Information\u001b[0m\n",
102102
"\u001b[36m • Target Type: OpenAIChatTarget\u001b[0m\n",
103103
"\u001b[36m • Target Model: gpt4o\u001b[0m\n",
104-
"\u001b[36m • Target Endpoint: https://pyrit-github-pipeline.openai.azure.com/openai/deployments/gpt-40/chat/completions\u001b[0m\n",
104+
"\u001b[36m • Target Endpoint: https://pyrit-github-pipeline.openai.azure.com/openai/deployments/gpt-40/chat/completions?api-version=2024-10-21\u001b[0m\n",
105105
"\n",
106106
"\u001b[1m 📊 Scorer Information\u001b[0m\n",
107107
"\u001b[36m • Scorer Type: TrueFalseCompositeScorer\u001b[0m\n",
@@ -116,14 +116,18 @@
116116
"\u001b[1m\u001b[36m▼ Overall Statistics\u001b[0m\n",
117117
"\u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n",
118118
"\u001b[1m 📈 Summary\u001b[0m\n",
119-
"\u001b[32m • Total Strategies: 3\u001b[0m\n",
120-
"\u001b[32m • Total Attack Results: 6\u001b[0m\n",
121-
"\u001b[32m • Overall Success Rate: 16%\u001b[0m\n",
119+
"\u001b[32m • Total Strategies: 4\u001b[0m\n",
120+
"\u001b[32m • Total Attack Results: 8\u001b[0m\n",
121+
"\u001b[32m • Overall Success Rate: 12%\u001b[0m\n",
122122
"\u001b[32m • Unique Objectives: 2\u001b[0m\n",
123123
"\n",
124124
"\u001b[1m\u001b[36m▼ Per-Strategy Breakdown\u001b[0m\n",
125125
"\u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n",
126126
"\n",
127+
"\u001b[1m 🔸 Strategy: baseline\u001b[0m\n",
128+
"\u001b[33m • Number of Results: 2\u001b[0m\n",
129+
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
130+
"\n",
127131
"\u001b[1m 🔸 Strategy: base64\u001b[0m\n",
128132
"\u001b[33m • Number of Results: 2\u001b[0m\n",
129133
"\u001b[33m • Success Rate: 50%\u001b[0m\n",
@@ -171,12 +175,13 @@
171175
"\n",
172176
"# Create a scenario from the pre-configured Foundry scenario\n",
173177
"foundry_scenario = FoundryScenario(\n",
178+
" objectives=objectives\n",
179+
")\n",
180+
"await foundry_scenario.initialize_async( # type: ignore\n",
174181
" objective_target=objective_target,\n",
175-
" max_concurrency=10,\n",
176182
" scenario_strategies=scenario_strategies,\n",
177-
" objectives=objectives\n",
183+
" max_concurrency=10,\n",
178184
")\n",
179-
"await foundry_scenario.initialize_async() # type: ignore\n",
180185
"\n",
181186
"print(f\"Created scenario: {foundry_scenario.name}\")\n",
182187
"\n",

‎doc/code/scenarios/scenarios.py‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -81,13 +81,12 @@
8181

8282

8383
# Create a scenario from the pre-configured Foundry scenario
84-
foundry_scenario = FoundryScenario(
84+
foundry_scenario = FoundryScenario(objectives=objectives)
85+
await foundry_scenario.initialize_async( # type: ignore
8586
objective_target=objective_target,
86-
max_concurrency=10,
8787
scenario_strategies=scenario_strategies,
88-
objectives=objectives,
88+
max_concurrency=10,
8989
)
90-
await foundry_scenario.initialize_async() # type: ignore
9190

9291
print(f"Created scenario: {foundry_scenario.name}")
9392

‎doc/code/setup/2_resiliency.ipynb‎

Lines changed: 48 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -94,37 +94,31 @@
9494
"Like target-level retry, JSON retry is **automatically applied** where needed. It's particularly useful when:\n",
9595
"- Working with targets that return structured data\n",
9696
"- LLMs occasionally generate malformed JSON\n",
97-
"- You need automatic recovery from parsing errors"
98-
]
99-
},
100-
{
101-
"cell_type": "markdown",
102-
"id": "3",
103-
"metadata": {},
104-
"source": [
97+
"- You need automatic recovery from parsing errors\n",
98+
"\n",
99+
"\n",
105100
"## 3. Scenario-Level Retries: High-Level Workflow Resiliency\n",
106101
"\n",
107-
"\"### What is Scenario-Level Retry?\\n\",\n",
108-
" \"\\n\",\n",
109-
" \"Scenario-level retry is the **highest-level** retry mechanism in PyRIT. When enabled via the `max_retries` parameter, it allows an entire scenario execution to automatically retry if an exception occurs during the workflow. \\n\", \n",
102+
"### What is Scenario-Level Retry?\n",
103+
"\n",
104+
"Scenario-level retry is the **highest-level** retry mechanism in PyRIT. When enabled via the `max_retries` parameter, it allows an entire scenario execution to automatically retry if an exception occurs during the workflow.\n",
110105
"\n",
111106
"But also note, if you rerun the same scenario manually, that follows the same logic and is always an option.\n",
112107
"\n",
113108
"### Key Features\n",
114109
"\n",
115-
"\"\\n\",\n",
116-
" \"- **Picks up where it left off**: On retry, the scenario skips already-completed objectives and continues from the point of exception\\n\",\n",
117-
" \"- **Broad exception handling**: Catches any exception during scenario execution (network issues, target failures, scoring errors, etc.)\\n\",\n",
118-
" \"- **Configurable attempts**: Set `max_retries` to control how many additional attempts are allowed\\n\",\n",
110+
"- **Picks up where it left off**: On retry, the scenario skips already-completed objectives and continues from the point of exception\n",
111+
"- **Broad exception handling**: Catches any exception during scenario execution (network issues, target failures, scoring errors, etc.)\n",
112+
"- **Configurable attempts**: Set `max_retries` to control how many additional attempts are allowed\n",
119113
"- **Progress tracking**: The `number_tries` field in `ScenarioResult` tracks total attempts\n",
120114
"\n",
121115
"### How It Works\n",
122116
"\n",
123117
"When you call `scenario.run_async()`, PyRIT:\n",
124118
"\n",
125-
"\"1. **Initial Attempt**: Executes all atomic attacks in sequence\\n\",\n",
126-
" \"2. **On Exception**: If an exception occurs, checks if retries remain\\n\",\n",
127-
" \"3. **Retry with Resume**: On retry, queries memory to identify completed objectives and skips them\\n\",\n",
119+
"1. **Initial Attempt**: Executes all atomic attacks in sequence\n",
120+
"2. **On Exception**: If an exception occurs, checks if retries remain\n",
121+
"3. **Retry with Resume**: On retry, queries memory to identify completed objectives and skips them\n",
128122
"4. **Continue from Exception Point**: Executes only the remaining objectives\n",
129123
"5. **Repeat**: Continues retrying until success or `max_retries` exhausted\n",
130124
"\n",
@@ -134,18 +128,18 @@
134128
{
135129
"cell_type": "code",
136130
"execution_count": null,
137-
"id": "4",
131+
"id": "3",
138132
"metadata": {},
139133
"outputs": [
140134
{
141135
"data": {
142136
"application/vnd.jupyter.widget-view+json": {
143-
"model_id": "7e7ec69c6fe6410daa8f213e916da7ca",
137+
"model_id": "585bf11361ad4e14bcf466f6948d1756",
144138
"version_major": 2,
145139
"version_minor": 0
146140
},
147141
"text/plain": [
148-
"Executing Foundry Scenario: 0%| | 0/1 [00:00<?, ?attack/s]"
142+
"Executing Foundry Scenario: 0%| | 0/2 [00:00<?, ?attack/s]"
149143
]
150144
},
151145
"metadata": {},
@@ -156,7 +150,7 @@
156150
"output_type": "stream",
157151
"text": [
158152
"Scenario completed after 1 attempt(s)\n",
159-
"Total results: 1\n"
153+
"Total results: 2\n"
160154
]
161155
}
162156
],
@@ -170,15 +164,15 @@
170164
"objective_target = OpenAIChatTarget(model_name=\"gpt-4o\")\n",
171165
"\n",
172166
"# Create a scenario with retry configuration\n",
173-
"scenario = FoundryScenario(\n",
167+
"scenario = FoundryScenario()\n",
168+
"\n",
169+
"await scenario.initialize_async( # type: ignore\n",
174170
" objective_target=objective_target,\n",
175171
" max_concurrency=5,\n",
176-
" max_retries=3, # Allow up to 3 retries (4 total attempts)\n",
172+
" max_retries=3,\n",
177173
" scenario_strategies=[FoundryStrategy.Base64],\n",
178174
")\n",
179175
"\n",
180-
"await scenario.initialize_async() # type: ignore\n",
181-
"\n",
182176
"# Execute with automatic retry after exceptions\n",
183177
"result = await scenario.run_async() # type: ignore\n",
184178
"\n",
@@ -188,7 +182,7 @@
188182
},
189183
{
190184
"cell_type": "markdown",
191-
"id": "5",
185+
"id": "4",
192186
"metadata": {},
193187
"source": [
194188
"\n",
@@ -206,8 +200,8 @@
206200
"\n",
207201
"Use scenario-level retries when:\n",
208202
"\n",
209-
"\"- ✅ Running long-duration test campaigns that might encounter transient exceptions\\n\",\n",
210-
" \"- ✅ Testing against unreliable targets or networks\\n\",\n",
203+
"- ✅ Running long-duration test campaigns that might encounter transient exceptions\n",
204+
"- ✅ Testing against unreliable targets or networks\n",
211205
"- ✅ You want to ensure comprehensive test coverage despite intermittent issues\n",
212206
"- ✅ You need workflow-level resilience (e.g., partial completion + retry)\n",
213207
"\n",
@@ -220,7 +214,7 @@
220214
},
221215
{
222216
"cell_type": "markdown",
223-
"id": "6",
217+
"id": "5",
224218
"metadata": {},
225219
"source": [
226220
"### Manual Scenario Resumption\n",
@@ -259,7 +253,7 @@
259253
},
260254
{
261255
"cell_type": "markdown",
262-
"id": "7",
256+
"id": "6",
263257
"metadata": {},
264258
"source": [
265259
"### Resume from Partial Completion\n",
@@ -287,7 +281,7 @@
287281
},
288282
{
289283
"cell_type": "markdown",
290-
"id": "8",
284+
"id": "7",
291285
"metadata": {},
292286
"source": [
293287
"## Understanding the Retry Hierarchy\n",
@@ -297,29 +291,29 @@
297291
"### Core Retry Mechanisms\n",
298292
"\n",
299293
"```\n",
300-
"\"┌─────────────────────────────────────────────────────────────┐\\n\",\n",
301-
" \"│ Scenario-Level Retry (max_retries) │\\n\",\n",
302-
" \"│ • Handles ANY exception in the entire workflow │\\n\",\n",
303-
" \"│ • Resumes from point of exception │\\n\",\n",
294+
"┌─────────────────────────────────────────────────────────────┐\n",
295+
"│ Scenario-Level Retry (max_retries) │\n",
296+
"│ • Handles ANY exception in the entire workflow │\n",
297+
"│ • Resumes from point of exception │\n",
304298
"│ • Configurable per scenario │\n",
305299
"│ │\n",
306-
"│ ┌───────────────────────────────────────────────────────┐ │\n",
307-
"│ │ AtomicAttack Execution │ │\n",
308-
"│ │ │ │\n",
309-
"│ │ ┌─────────────────────────────────────────────────┐ │ │\n",
310-
"│ │ │ JSON-Level Retry (pyrit_json_retry) │ │ │\n",
311-
"│ │ │ • Handles invalid JSON responses │ │ │\n",
312-
"│ │ │ • No exponential backoff (immediate retry) │ │ │\n",
313-
"│ │ │ • Uses target call, so includes target retry │ │ │\n",
314-
"│ │ │ │ │ │\n",
315-
"│ │ │ ┌──────────────────────────────────────────┐ │ │ │\n",
316-
"│ │ │ │ Target-Level Retry (pyrit_target_retry) │ │ │ │\n",
317-
"│ │ │ │ • Handles rate limits, empty responses │ │ │ │\n",
318-
"│ │ │ │ • Exponential backoff │ │ │ │\n",
319-
"│ │ │ │ • Configured via environment variables │ │ │ │\n",
320-
"│ │ │ └──────────────────────────────────────────┘ │ │ │\n",
321-
"│ │ └─────────────────────────────────────────────────┘ │ │\n",
322-
"│ └───────────────────────────────────────────────────────┘ │\n",
300+
"│ ┌───────────────────────────────────────────────────────┐ │\n",
301+
"│ │ AtomicAttack Execution │ │\n",
302+
"│ │ │ │\n",
303+
"│ │ ┌─────────────────────────────────────────────────┐ │ │\n",
304+
"│ │ │ JSON-Level Retry (pyrit_json_retry) │ │ │\n",
305+
"│ │ │ • Handles invalid JSON responses │ │ │\n",
306+
"│ │ │ • No exponential backoff (immediate retry) │ │ │\n",
307+
"│ │ │ • Uses target call, so includes target retry │ │ │\n",
308+
"│ │ │ │ │ │\n",
309+
"│ │ │ ┌──────────────────────────────────────────┐ │ │ │\n",
310+
"│ │ │ │ Target-Level Retry (pyrit_target_retry) │ │ │ │\n",
311+
"│ │ │ │ • Handles rate limits, empty responses │ │ │ │\n",
312+
"│ │ │ │ • Exponential backoff │ │ │ │\n",
313+
"│ │ │ │ • Configured via environment variables │ │ │ │\n",
314+
"│ │ │ └──────────────────────────────────────────┘ │ │ │\n",
315+
"│ │ └─────────────────────────────────────────────────┘ │ │\n",
316+
"│ └───────────────────────────────────────────────────────┘ │\n",
323317
"└─────────────────────────────────────────────────────────────┘\n",
324318
"```\n",
325319
"\n",
@@ -336,7 +330,7 @@
336330
},
337331
{
338332
"cell_type": "markdown",
339-
"id": "9",
333+
"id": "8",
340334
"metadata": {},
341335
"source": [
342336
"## Best Practices\n",

0 commit comments

Comments
 (0)