Skip to content

Commit 2fe634b

Browse files
ValbuenaVCVictor ValbuenaCopilot
authored
FIX: Integration Test Fixes (#1907)
Co-authored-by: Victor Valbuena <vvalbuena@microsoft.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 07243dc commit 2fe634b

14 files changed

Lines changed: 349 additions & 432 deletions

‎doc/code/executor/attack/barge_in_attack.ipynb‎

Lines changed: 7 additions & 207 deletions
Large diffs are not rendered by default.

‎doc/code/executor/attack/barge_in_attack.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
# extension: .py
77
# format_name: percent
88
# format_version: '1.3'
9-
# jupytext_version: 1.18.1
9+
# jupytext_version: 1.19.3
1010
# ---
1111

1212
# %% [markdown]
@@ -188,7 +188,7 @@ async def barge_in_source():
188188
if piece.converted_value_data_type == "audio_path":
189189
val = Path(val).name
190190
value_preview = (val[:80] + "...") if len(val) > 80 else val
191-
print(f" {piece._role} {piece.converted_value_data_type}{marker}: {value_preview}")
191+
print(f" {piece.role} {piece.converted_value_data_type}{marker}: {value_preview}")
192192

193193
await ConsoleAttackResultPrinter(width=200).write_async(result=barge_in_result) # type: ignore
194194
await target2.cleanup_target_async() # type: ignore

‎doc/code/scenarios/1_common_scenario_parameters.ipynb‎

Lines changed: 41 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,13 @@
6060
"Skipping target 'azure_foundry_phi4': AZURE_FOUNDRY_PHI4_MODEL is not set. All declared env vars (endpoint, key, model) must be present for this target to register.\n"
6161
]
6262
},
63+
{
64+
"name": "stderr",
65+
"output_type": "stream",
66+
"text": [
67+
"TextAdaptive: _EXCLUDED_TECHNIQUES entries ['prompt_sending'] are not in the current scenario-techniques catalog ['context_compliance', 'crescendo_history_lecture', 'crescendo_journalist_interview', 'crescendo_movie_director', 'crescendo_simulated', 'many_shot', 'pair', 'red_teaming', 'role_play', 'tap']; the exclusion is a no-op for those entries. Remove stale entries or update the catalog.\n"
68+
]
69+
},
6370
{
6471
"name": "stderr",
6572
"output_type": "stream",
@@ -75,9 +82,7 @@
7582
"from pyrit.registry import TargetRegistry\n",
7683
"from pyrit.scenario.scenarios.foundry import FoundryStrategy, RedTeamAgent\n",
7784
"from pyrit.setup import initialize_from_config_async\n",
78-
"from pyrit.setup.initializers.components import ScenarioTechniqueInitializer\n",
7985
"\n",
80-
"await ScenarioTechniqueInitializer().initialize_async() # type: ignore [top-level-await]\n",
8186
"await initialize_from_config_async(config_path=Path(\"../../scanner/pyrit_conf.yaml\")) # type: ignore\n",
8287
"\n",
8388
"objective_target = TargetRegistry.get_registry_singleton().get_instance_by_name(\"openai_chat\")"
@@ -251,7 +256,7 @@
251256
{
252257
"data": {
253258
"application/vnd.jupyter.widget-view+json": {
254-
"model_id": "e386860005e440a98009fc5cd1b55e86",
259+
"model_id": "20dadc12a43444a583579caf2f178403",
255260
"version_major": 2,
256261
"version_minor": 0
257262
},
@@ -314,25 +319,25 @@
314319
"\u001b[1m 📈 Summary\u001b[0m\n",
315320
"\u001b[32m • Total Strategies: 21\u001b[0m\n",
316321
"\u001b[32m • Total Attack Results: 42\u001b[0m\n",
317-
"\u001b[32m • Overall Success Rate: 0%\u001b[0m\n",
322+
"\u001b[32m • Overall Success Rate: 2%\u001b[0m\n",
318323
"\u001b[32m • Unique Objectives: 2\u001b[0m\n",
319324
"\n",
320325
"\u001b[1m\u001b[36m▼ Per-Group Breakdown\u001b[0m\n",
321326
"\u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n",
322327
"\n",
323-
"\u001b[1m 🔸 Group: baseline\u001b[0m\n",
328+
"\u001b[1m 🔸 Group: ansi_attack\u001b[0m\n",
324329
"\u001b[33m • Number of Results: 2\u001b[0m\n",
325330
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
326331
"\n",
327-
"\u001b[1m 🔸 Group: ansi_attack\u001b[0m\n",
332+
"\u001b[1m 🔸 Group: baseline\u001b[0m\n",
328333
"\u001b[33m • Number of Results: 2\u001b[0m\n",
329334
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
330335
"\n",
331-
"\u001b[1m 🔸 Group: ascii_smuggler\u001b[0m\n",
336+
"\u001b[1m 🔸 Group: ascii_art\u001b[0m\n",
332337
"\u001b[33m • Number of Results: 2\u001b[0m\n",
333338
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
334339
"\n",
335-
"\u001b[1m 🔸 Group: ascii_art\u001b[0m\n",
340+
"\u001b[1m 🔸 Group: ascii_smuggler\u001b[0m\n",
336341
"\u001b[33m • Number of Results: 2\u001b[0m\n",
337342
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
338343
"\n",
@@ -344,15 +349,15 @@
344349
"\u001b[33m • Number of Results: 2\u001b[0m\n",
345350
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
346351
"\n",
347-
"\u001b[1m 🔸 Group: binary\u001b[0m\n",
352+
"\u001b[1m 🔸 Group: caesar\u001b[0m\n",
348353
"\u001b[33m • Number of Results: 2\u001b[0m\n",
349354
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
350355
"\n",
351-
"\u001b[1m 🔸 Group: caesar\u001b[0m\n",
356+
"\u001b[1m 🔸 Group: character_space\u001b[0m\n",
352357
"\u001b[33m • Number of Results: 2\u001b[0m\n",
353358
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
354359
"\n",
355-
"\u001b[1m 🔸 Group: character_space\u001b[0m\n",
360+
"\u001b[1m 🔸 Group: binary\u001b[0m\n",
356361
"\u001b[33m • Number of Results: 2\u001b[0m\n",
357362
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
358363
"\n",
@@ -376,19 +381,15 @@
376381
"\u001b[33m • Number of Results: 2\u001b[0m\n",
377382
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
378383
"\n",
379-
"\u001b[1m 🔸 Group: suffix_append\u001b[0m\n",
380-
"\u001b[33m • Number of Results: 2\u001b[0m\n",
381-
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
382-
"\n",
383384
"\u001b[1m 🔸 Group: rot13\u001b[0m\n",
384385
"\u001b[33m • Number of Results: 2\u001b[0m\n",
385-
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
386+
"\u001b[33m • Success Rate: 50%\u001b[0m\n",
386387
"\n",
387-
"\u001b[1m 🔸 Group: string_join\u001b[0m\n",
388+
"\u001b[1m 🔸 Group: suffix_append\u001b[0m\n",
388389
"\u001b[33m • Number of Results: 2\u001b[0m\n",
389390
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
390391
"\n",
391-
"\u001b[1m 🔸 Group: unicode_confusable\u001b[0m\n",
392+
"\u001b[1m 🔸 Group: string_join\u001b[0m\n",
392393
"\u001b[33m • Number of Results: 2\u001b[0m\n",
393394
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
394395
"\n",
@@ -404,6 +405,10 @@
404405
"\u001b[33m • Number of Results: 2\u001b[0m\n",
405406
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
406407
"\n",
408+
"\u001b[1m 🔸 Group: unicode_confusable\u001b[0m\n",
409+
"\u001b[33m • Number of Results: 2\u001b[0m\n",
410+
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
411+
"\n",
407412
"\u001b[36m====================================================================================================\u001b[0m\n",
408413
"\n"
409414
]
@@ -492,37 +497,37 @@
492497
"\u001b[1m 📈 Summary\u001b[0m\n",
493498
"\u001b[32m • Total Strategies: 21\u001b[0m\n",
494499
"\u001b[32m • Total Attack Results: 42\u001b[0m\n",
495-
"\u001b[32m • Overall Success Rate: 0%\u001b[0m\n",
500+
"\u001b[32m • Overall Success Rate: 2%\u001b[0m\n",
496501
"\u001b[32m • Unique Objectives: 2\u001b[0m\n",
497502
"\n",
498503
"\u001b[1m\u001b[36m▼ Per-Group Breakdown\u001b[0m\n",
499504
"\u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n",
500505
"\n",
501-
"\u001b[1m 🔸 Group: baseline\u001b[0m\n",
506+
"\u001b[1m 🔸 Group: rot13\u001b[0m\n",
502507
"\u001b[33m • Number of Results: 2\u001b[0m\n",
503-
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
508+
"\u001b[33m • Success Rate: 50%\u001b[0m\n",
504509
"\n",
505510
"\u001b[1m 🔸 Group: ansi_attack\u001b[0m\n",
506511
"\u001b[33m • Number of Results: 2\u001b[0m\n",
507512
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
508513
"\n",
509-
"\u001b[1m 🔸 Group: ascii_smuggler\u001b[0m\n",
514+
"\u001b[1m 🔸 Group: baseline\u001b[0m\n",
510515
"\u001b[33m • Number of Results: 2\u001b[0m\n",
511516
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
512517
"\n",
513518
"\u001b[1m 🔸 Group: ascii_art\u001b[0m\n",
514519
"\u001b[33m • Number of Results: 2\u001b[0m\n",
515520
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
516521
"\n",
517-
"\u001b[1m 🔸 Group: atbash\u001b[0m\n",
522+
"\u001b[1m 🔸 Group: ascii_smuggler\u001b[0m\n",
518523
"\u001b[33m • Number of Results: 2\u001b[0m\n",
519524
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
520525
"\n",
521-
"\u001b[1m 🔸 Group: base64\u001b[0m\n",
526+
"\u001b[1m 🔸 Group: atbash\u001b[0m\n",
522527
"\u001b[33m • Number of Results: 2\u001b[0m\n",
523528
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
524529
"\n",
525-
"\u001b[1m 🔸 Group: binary\u001b[0m\n",
530+
"\u001b[1m 🔸 Group: base64\u001b[0m\n",
526531
"\u001b[33m • Number of Results: 2\u001b[0m\n",
527532
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
528533
"\n",
@@ -534,6 +539,10 @@
534539
"\u001b[33m • Number of Results: 2\u001b[0m\n",
535540
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
536541
"\n",
542+
"\u001b[1m 🔸 Group: binary\u001b[0m\n",
543+
"\u001b[33m • Number of Results: 2\u001b[0m\n",
544+
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
545+
"\n",
537546
"\u001b[1m 🔸 Group: char_swap\u001b[0m\n",
538547
"\u001b[33m • Number of Results: 2\u001b[0m\n",
539548
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
@@ -558,18 +567,10 @@
558567
"\u001b[33m • Number of Results: 2\u001b[0m\n",
559568
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
560569
"\n",
561-
"\u001b[1m 🔸 Group: rot13\u001b[0m\n",
562-
"\u001b[33m • Number of Results: 2\u001b[0m\n",
563-
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
564-
"\n",
565570
"\u001b[1m 🔸 Group: string_join\u001b[0m\n",
566571
"\u001b[33m • Number of Results: 2\u001b[0m\n",
567572
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
568573
"\n",
569-
"\u001b[1m 🔸 Group: unicode_confusable\u001b[0m\n",
570-
"\u001b[33m • Number of Results: 2\u001b[0m\n",
571-
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
572-
"\n",
573574
"\u001b[1m 🔸 Group: unicode_substitution\u001b[0m\n",
574575
"\u001b[33m • Number of Results: 2\u001b[0m\n",
575576
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
@@ -582,6 +583,10 @@
582583
"\u001b[33m • Number of Results: 2\u001b[0m\n",
583584
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
584585
"\n",
586+
"\u001b[1m 🔸 Group: unicode_confusable\u001b[0m\n",
587+
"\u001b[33m • Number of Results: 2\u001b[0m\n",
588+
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
589+
"\n",
585590
"\u001b[36m====================================================================================================\u001b[0m\n",
586591
"\n"
587592
]
@@ -632,7 +637,7 @@
632637
{
633638
"data": {
634639
"application/vnd.jupyter.widget-view+json": {
635-
"model_id": "9f07786563cc4128ba51a5e47eb53a6f",
640+
"model_id": "db595f84b76f4efa8efcb2453bcb798c",
636641
"version_major": 2,
637642
"version_minor": 0
638643
},
@@ -690,15 +695,15 @@
690695
"\u001b[1m 📈 Summary\u001b[0m\n",
691696
"\u001b[32m • Total Strategies: 2\u001b[0m\n",
692697
"\u001b[32m • Total Attack Results: 4\u001b[0m\n",
693-
"\u001b[33m • Overall Success Rate: 50%\u001b[0m\n",
698+
"\u001b[36m • Overall Success Rate: 25%\u001b[0m\n",
694699
"\u001b[32m • Unique Objectives: 2\u001b[0m\n",
695700
"\n",
696701
"\u001b[1m\u001b[36m▼ Per-Group Breakdown\u001b[0m\n",
697702
"\u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n",
698703
"\n",
699704
"\u001b[1m 🔸 Group: baseline\u001b[0m\n",
700705
"\u001b[33m • Number of Results: 2\u001b[0m\n",
701-
"\u001b[33m • Success Rate: 50%\u001b[0m\n",
706+
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
702707
"\n",
703708
"\u001b[1m 🔸 Group: base64\u001b[0m\n",
704709
"\u001b[33m • Number of Results: 2\u001b[0m\n",

‎doc/code/scenarios/1_common_scenario_parameters.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,7 @@
3232
from pyrit.registry import TargetRegistry
3333
from pyrit.scenario.scenarios.foundry import FoundryStrategy, RedTeamAgent
3434
from pyrit.setup import initialize_from_config_async
35-
from pyrit.setup.initializers.components import ScenarioTechniqueInitializer
3635

37-
await ScenarioTechniqueInitializer().initialize_async() # type: ignore [top-level-await]
3836
await initialize_from_config_async(config_path=Path("../../scanner/pyrit_conf.yaml")) # type: ignore
3937

4038
objective_target = TargetRegistry.get_registry_singleton().get_instance_by_name("openai_chat")

0 commit comments

Comments
 (0)