Skip to content

Commit 1e386ae

Browse files
hannahwestra25Copilot
hannahwestra25
and
Copilot
authored
DOC: Update 0_scenarios.ipynb output with adaptive.text_adaptive scenario (#1936)
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 01f4abc commit 1e386ae

2 files changed

Lines changed: 37 additions & 117 deletions

File tree

‎doc/code/scenarios/0_scenarios.ipynb‎

Lines changed: 29 additions & 115 deletions
Original file line numberDiff line numberDiff line change
@@ -73,13 +73,15 @@
7373
" - `name`: Descriptive name for your scenario\n",
7474
" - `version`: Integer version number\n",
7575
" - `strategy_class`: The strategy enum class for this scenario\n",
76-
" - `objective_scorer_identifier`: Identifier dict for the scoring mechanism (optional)\n",
76+
" - `default_strategy`: The default strategy member (typically `YourStrategy.ALL` or `YourStrategy.DEFAULT`)\n",
77+
" - `default_dataset_config`: A `DatasetConfiguration` specifying the scenario's default datasets\n",
78+
" - `objective_scorer`: The scorer used to judge responses\n",
7779
" - `scenario_result_id`: Optional ID to resume an existing scenario (optional)\n",
7880
"\n",
7981
"5. **Initialization**: Call `await scenario.initialize_async()` to populate atomic attacks:\n",
8082
" - `objective_target`: The target system being tested (required)\n",
8183
" - `scenario_strategies`: List of strategies to execute (optional, defaults to ALL)\n",
82-
" - `max_concurrency`: Number of concurrent operations (default: 1)\n",
84+
" - `max_concurrency`: Number of concurrent operations (default: 4)\n",
8385
" - `max_retries`: Number of retry attempts on failure (default: 0)\n",
8486
" - `memory_labels`: Optional labels for tracking (optional)\n",
8587
" - `include_baseline`: Whether to prepend a baseline attack (defaults to the scenario type's\n",
@@ -189,118 +191,6 @@
189191
"id": "3",
190192
"metadata": {},
191193
"outputs": [
192-
{
193-
"name": "stderr",
194-
"output_type": "stream",
195-
"text": [
196-
"TargetRegistry entry 'objective_scorer_chat' not found. Falling back to default OpenAIChatTarget.\n"
197-
]
198-
},
199-
{
200-
"name": "stderr",
201-
"output_type": "stream",
202-
"text": [
203-
"Using fallback default objective scorer: TrueFalseInverterScorer with chat target: OpenAIChatTarget\n"
204-
]
205-
},
206-
{
207-
"name": "stderr",
208-
"output_type": "stream",
209-
"text": [
210-
"TargetRegistry entry 'objective_scorer_chat' not found. Falling back to default OpenAIChatTarget.\n"
211-
]
212-
},
213-
{
214-
"name": "stderr",
215-
"output_type": "stream",
216-
"text": [
217-
"Using fallback default objective scorer: TrueFalseInverterScorer with chat target: OpenAIChatTarget\n"
218-
]
219-
},
220-
{
221-
"name": "stderr",
222-
"output_type": "stream",
223-
"text": [
224-
"TargetRegistry entry 'objective_scorer_chat' not found. Falling back to default OpenAIChatTarget.\n"
225-
]
226-
},
227-
{
228-
"name": "stderr",
229-
"output_type": "stream",
230-
"text": [
231-
"TargetRegistry entry 'adversarial_chat' not found. Falling back to default OpenAIChatTarget.\n"
232-
]
233-
},
234-
{
235-
"name": "stderr",
236-
"output_type": "stream",
237-
"text": [
238-
"TargetRegistry entry 'objective_scorer_chat' not found. Falling back to default OpenAIChatTarget.\n"
239-
]
240-
},
241-
{
242-
"name": "stderr",
243-
"output_type": "stream",
244-
"text": [
245-
"TargetRegistry entry 'objective_scorer_chat' not found. Falling back to default OpenAIChatTarget.\n"
246-
]
247-
},
248-
{
249-
"name": "stderr",
250-
"output_type": "stream",
251-
"text": [
252-
"Using fallback default objective scorer: TrueFalseInverterScorer with chat target: OpenAIChatTarget\n"
253-
]
254-
},
255-
{
256-
"name": "stderr",
257-
"output_type": "stream",
258-
"text": [
259-
"TargetRegistry entry 'objective_scorer_chat' not found. Falling back to default OpenAIChatTarget.\n"
260-
]
261-
},
262-
{
263-
"name": "stderr",
264-
"output_type": "stream",
265-
"text": [
266-
"TargetRegistry entry 'adversarial_chat' not found. Falling back to default OpenAIChatTarget.\n"
267-
]
268-
},
269-
{
270-
"name": "stderr",
271-
"output_type": "stream",
272-
"text": [
273-
"TargetRegistry entry 'objective_scorer_chat' not found. Falling back to default OpenAIChatTarget.\n"
274-
]
275-
},
276-
{
277-
"name": "stderr",
278-
"output_type": "stream",
279-
"text": [
280-
"Using fallback default objective scorer: TrueFalseInverterScorer with chat target: OpenAIChatTarget\n"
281-
]
282-
},
283-
{
284-
"name": "stderr",
285-
"output_type": "stream",
286-
"text": [
287-
"TargetRegistry entry 'adversarial_chat' not found. Falling back to default OpenAIChatTarget.\n"
288-
]
289-
},
290-
{
291-
"name": "stderr",
292-
"output_type": "stream",
293-
"text": [
294-
"TargetRegistry entry 'objective_scorer_chat' not found. Falling back to default OpenAIChatTarget.\n"
295-
]
296-
},
297-
{
298-
"name": "stderr",
299-
"output_type": "stream",
300-
"text": [
301-
"Using fallback default objective scorer: TrueFalseInverterScorer with chat target: OpenAIChatTarget\n"
302-
]
303-
},
304194
{
305195
"name": "stdout",
306196
"output_type": "stream",
@@ -309,6 +199,26 @@
309199
"Available Scenarios:\n",
310200
"================================================================================\n",
311201
"\u001b[1m\u001b[36m\n",
202+
" adaptive.text_adaptive\u001b[0m\n",
203+
" Class: TextAdaptive\n",
204+
" Description:\n",
205+
" Adaptive text-attack scenario. Selects techniques per-objective via an\n",
206+
" epsilon-greedy selector over the set of selected strategies.\n",
207+
" ``prompt_sending`` runs as the baseline comparison and is excluded from\n",
208+
" the adaptive technique pool.\n",
209+
" Aggregate Strategies:\n",
210+
" - all, default, single_turn, multi_turn\n",
211+
" Available Strategies (10):\n",
212+
" role_play, many_shot, tap, pair, crescendo_simulated, red_teaming,\n",
213+
" context_compliance, crescendo_movie_director, crescendo_history_lecture,\n",
214+
" crescendo_journalist_interview\n",
215+
" Default Strategy: default\n",
216+
" Default Datasets (7, max 4 per dataset):\n",
217+
" airt_hate, airt_fairness, airt_violence, airt_sexual, airt_harassment,\n",
218+
" airt_misinformation, airt_leakage\n",
219+
" Supported Parameters:\n",
220+
" - max_attempts_per_objective (int) [default: '3']: Max techniques tried per objective. Defaults to 3.\n",
221+
"\u001b[1m\u001b[36m\n",
312222
" airt.cyber\u001b[0m\n",
313223
" Class: Cyber\n",
314224
" Description:\n",
@@ -497,14 +407,18 @@
497407
"\n",
498408
"================================================================================\n",
499409
"\n",
500-
"Total scenarios: 9\n"
410+
"Total scenarios: 10\n"
501411
]
502412
}
503413
],
504414
"source": [
415+
"import logging\n",
416+
"\n",
505417
"from pyrit.backend.services.scenario_service import get_scenario_service\n",
506418
"from pyrit.cli._output import print_scenario_list\n",
507419
"\n",
420+
"logging.getLogger(\"pyrit\").setLevel(logging.ERROR)\n",
421+
"\n",
508422
"response = await get_scenario_service().list_scenarios_async(limit=200) # type: ignore\n",
509423
"print_scenario_list(items=[s.model_dump() for s in response.items])"
510424
]

‎doc/code/scenarios/0_scenarios.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,13 +75,15 @@
7575
# - `name`: Descriptive name for your scenario
7676
# - `version`: Integer version number
7777
# - `strategy_class`: The strategy enum class for this scenario
78-
# - `objective_scorer_identifier`: Identifier dict for the scoring mechanism (optional)
78+
# - `default_strategy`: The default strategy member (typically `YourStrategy.ALL` or `YourStrategy.DEFAULT`)
79+
# - `default_dataset_config`: A `DatasetConfiguration` specifying the scenario's default datasets
80+
# - `objective_scorer`: The scorer used to judge responses
7981
# - `scenario_result_id`: Optional ID to resume an existing scenario (optional)
8082
#
8183
# 5. **Initialization**: Call `await scenario.initialize_async()` to populate atomic attacks:
8284
# - `objective_target`: The target system being tested (required)
8385
# - `scenario_strategies`: List of strategies to execute (optional, defaults to ALL)
84-
# - `max_concurrency`: Number of concurrent operations (default: 1)
86+
# - `max_concurrency`: Number of concurrent operations (default: 4)
8587
# - `max_retries`: Number of retry attempts on failure (default: 0)
8688
# - `memory_labels`: Optional labels for tracking (optional)
8789
# - `include_baseline`: Whether to prepend a baseline attack (defaults to the scenario type's
@@ -157,9 +159,13 @@ def _build_display_group(self, *, technique_name: str, seed_group_name: str) ->
157159
# ## Existing Scenarios
158160

159161
# %%
162+
import logging
163+
160164
from pyrit.backend.services.scenario_service import get_scenario_service
161165
from pyrit.cli._output import print_scenario_list
162166

167+
logging.getLogger("pyrit").setLevel(logging.ERROR)
168+
163169
response = await get_scenario_service().list_scenarios_async(limit=200) # type: ignore
164170
print_scenario_list(items=[s.model_dump() for s in response.items])
165171

0 commit comments

Comments
 (0)