FIX: Bug in RolePlayAttack (microsoft#1211)

Author: rlundeen2

doc/code/executor/attack/context_compliance_attack.ipynb

@@ -5,7 +5,11 @@
 #       extension: .py
 #       format_name: percent
 #       format_version: '1.3'
-#       jupytext_version: 1.17.3
+#       jupytext_version: 1.18.1
+#   kernelspec:
+#     display_name: pyrit-dev
+#     language: python
+#     name: python3
 # ---
 
 # %% [markdown]
@@ -23,12 +27,12 @@
 #
 # This conversation is sent to the `objective_target`.
 #
-# Before you begin, ensure you have the correct version of PyRIT installed and have secrets configured as described [here](../../../index.md#Installation-Guide.
+# Before you begin, ensure you have the correct version of PyRIT installed and have secrets configured as described [here](../../../setup/1a_install_conda.md).
 #
 # The results and intermediate interactions will be saved to memory according to the environment settings. For details, see the [Memory Configuration Guide](../../memory/0_memory.md).
 
 # %%
-
+import os
 
 from pyrit.executor.attack import (
     AttackAdversarialConfig,
@@ -38,26 +42,34 @@
     ConsoleAttackResultPrinter,
     ContextComplianceAttack,
 )
-from pyrit.prompt_converter import RandomCapitalLettersConverter
+from pyrit.prompt_converter import TranslationConverter
 from pyrit.prompt_normalizer import PromptConverterConfiguration
 from pyrit.prompt_target import OpenAIChatTarget
-from pyrit.score import AzureContentFilterScorer
+from pyrit.score import AzureContentFilterScorer, FloatScaleThresholdScorer
 from pyrit.setup import IN_MEMORY, initialize_pyrit
 
 initialize_pyrit(memory_db_type=IN_MEMORY)
 
 objective_target = OpenAIChatTarget()
-adversarial_chat = OpenAIChatTarget()
+adversarial_chat = OpenAIChatTarget(
+    endpoint=os.environ["AZURE_OPENAI_GPT4O_UNSAFE_ENDPOINT"],
+    api_key=os.environ["AZURE_OPENAI_GPT4O_UNSAFE_CHAT_KEY"],
+)
+
+scoring_config = AttackScoringConfig(
+    objective_scorer=FloatScaleThresholdScorer(scorer=AzureContentFilterScorer(), threshold=0.2),
+)
+
+language_converter = TranslationConverter(converter_target=adversarial_chat, language="German")
 
 attack = ContextComplianceAttack(
     objective_target=objective_target,
     attack_adversarial_config=AttackAdversarialConfig(target=adversarial_chat),
     attack_converter_config=AttackConverterConfig(
-        request_converters=PromptConverterConfiguration.from_converters(
-            converters=[RandomCapitalLettersConverter(percentage=50)]
-        )
+        request_converters=PromptConverterConfiguration.from_converters(converters=[language_converter]),
+        response_converters=PromptConverterConfiguration.from_converters(converters=[language_converter]),
     ),
-    attack_scoring_config=AttackScoringConfig(auxiliary_scorers=[AzureContentFilterScorer()]),
+    attack_scoring_config=scoring_config,
 )
 
 objectives = [

doc/code/executor/attack/context_compliance_attack.py

@@ -5,7 +5,11 @@
 #       extension: .py
 #       format_name: percent
 #       format_version: '1.3'
-#       jupytext_version: 1.17.3
+#       jupytext_version: 1.18.1
+#   kernelspec:
+#     display_name: pyrit-dev
+#     language: python
+#     name: python3
 # ---
 
 # %% [markdown]
@@ -14,7 +18,7 @@
 # This attack prepends some prompts defined in `role_play_definition`, along with an `adversarial_chat` target LLM to generate the first turns to send. Typically these prompts describe a fictional scenario to attempt and elicit harmful responses.
 # Any converters that you provide will be applied to the prompt that has already been converted by the role play definition (using the provided `adversarial_chat` target). You may see better success if you provide a LLM that has no content moderation or other safety mechanisms. Otherwise, it may refuse to convert the prompt as expected.
 #
-# Before you begin, ensure you have the correct version of PyRIT installed and have secrets configured as described [here](../../../index.md#Installation-Guide.
+# Before you begin, ensure you have the correct version of PyRIT installed and have secrets configured as described [here](../../../setup/1a_install_conda.md).
 #
 # The results and intermediate interactions will be saved to memory according to the environment settings. For details, see the [Memory Configuration Guide](../../memory/0_memory.md).
 
@@ -32,7 +36,7 @@
 from pyrit.prompt_converter import CharSwapConverter
 from pyrit.prompt_normalizer import PromptConverterConfiguration
 from pyrit.prompt_target import OpenAIChatTarget
-from pyrit.score import AzureContentFilterScorer
+from pyrit.score import AzureContentFilterScorer, FloatScaleThresholdScorer
 from pyrit.setup import IN_MEMORY, initialize_pyrit
 
 initialize_pyrit(memory_db_type=IN_MEMORY)
@@ -47,7 +51,7 @@
 converter_config = AttackConverterConfig(request_converters=converters)
 
 scoring_config = AttackScoringConfig(
-    auxiliary_scorers=[AzureContentFilterScorer()],
+    objective_scorer=FloatScaleThresholdScorer(scorer=AzureContentFilterScorer(), threshold=0.2),
 )
 
 attack = RolePlayAttack(
@@ -67,4 +71,4 @@
 )
 
 for result in results:
-    await ConsoleAttackResultPrinter().print_conversation_async(result=result, include_auxiliary_scores=True)  # type: ignore
+    await ConsoleAttackResultPrinter().print_result_async(result=result, include_auxiliary_scores=True)  # type: ignore

doc/code/executor/attack/role_play_attack.ipynb

@@ -264,7 +264,6 @@
     }
    ],
    "source": [
-    "\n",
     "from pyrit.cli.frontend_core import FrontendCore, print_scenarios_list\n",
     "\n",
     "print_scenarios_list(context=FrontendCore())"

doc/code/executor/attack/role_play_attack.py

@@ -16,6 +16,8 @@
 from pyrit.models import (
     Message,
     SeedDataset,
+    SeedGroup,
+    SeedPrompt,
 )
 from pyrit.prompt_converter import LLMGenericTextConverter
 from pyrit.prompt_normalizer import PromptConverterConfiguration, PromptNormalizer
@@ -101,7 +103,7 @@ def __init__(
         self._parse_role_play_definition(role_play_definition)
 
         # Create the rephrase converter configuration
-        rephrase_converter = PromptConverterConfiguration.from_converters(
+        self._rephrase_converter = PromptConverterConfiguration.from_converters(
             converters=[
                 LLMGenericTextConverter(
                     converter_target=self._adversarial_chat,
@@ -110,19 +112,25 @@ def __init__(
             ]
         )
 
-        # Prepend the rephrase converter to existing request converters
-        self._request_converters = rephrase_converter + self._request_converters
-
     async def _setup_async(self, *, context: SingleTurnAttackContext) -> None:
         """
-        Sets up the attack by preparing conversation context with role-play start.
+        Sets up the attack by preparing conversation context with role-play start
+        and converting the objective to role-play format.
 
         Args:
             context (SingleTurnAttackContext): The attack context containing attack parameters.
         """
-        # Get role-play conversation start
+        # Get role-play conversation start (turns 0 and 1)
         context.prepended_conversation = await self._get_conversation_start() or []
 
+        # Rephrase the objective using the LLM converter
+        # This converts the user's objective into a role-play scenario
+        rephrased_objective = await self._rephrase_objective_async(objective=context.objective)
+
+        # Set the rephrased objective as the seed_group
+        # This will be used by _get_prompt_group() to send the rephrased content to the target
+        context.seed_group = SeedGroup(seeds=[SeedPrompt(value=rephrased_objective, data_type="text")])
+
         # Call parent setup which handles conversation ID generation, memory labels, etc.
         await super()._setup_async(context=context)
 
@@ -134,12 +142,35 @@ def _validate_context(self, *, context: SingleTurnAttackContext) -> None:
             context (SingleTurnAttackContext): The attack context containing parameters and objective.
 
         Raises:
-            ValueError: If the context is invalid.
+            ValueError: If seed_group or prepended_conversation are provided by the user.
         """
+        if context.seed_group is not None:
+            raise ValueError(
+                "RolePlayAttack does not accept a seed_group parameter. "
+                "The seed group is generated internally by rephrasing the objective."
+            )
         if context.prepended_conversation:
-            raise ValueError("RolePlayAttack does not support prepended conversations.")
+            raise ValueError(
+                "RolePlayAttack does not accept prepended_conversation parameter. "
+                "The conversation start is generated internally from the role-play definition."
+            )
         super()._validate_context(context=context)
 
+    async def _rephrase_objective_async(self, *, objective: str) -> str:
+        """
+        Rephrase the objective into a role-play scenario using the adversarial chat.
+
+        Args:
+            objective (str): The original objective to rephrase.
+
+        Returns:
+            str: The rephrased objective in role-play format.
+        """
+        # Use the LLMGenericTextConverter to rephrase the objective
+        converter = self._rephrase_converter[0].converters[0]
+        result = await converter.convert_async(prompt=objective, input_type="text")
+        return result.output_text
+
     async def _get_conversation_start(self) -> Optional[list[Message]]:
         """
         Get the role-play conversation start messages.