Rename BASELINE_POLICY to BASELINE_ATTACK_POLICY for clarity (microsoft#1763)

Co-authored-by: hannahwestra25 <hannahwestra25@users.noreply.github.com>
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: hannahwestra25

.github/instructions/scenarios.instructions.md

@@ -11,15 +11,15 @@ Scenarios orchestrate multi-attack security testing campaigns. Each scenario gro
 All scenarios inherit from `Scenario` (ABC) and must:
 
 1. **Define `VERSION`** as a class constant (increment on breaking changes)
-2. **Optionally declare `BASELINE_POLICY`** (defaults to `BaselinePolicy.Enabled` — a baseline `PromptSendingAttack` is prepended and callers can opt out per run via `initialize_async(include_baseline=False)`):
-   - `BaselinePolicy.Disabled` — baseline supported but off by default (e.g. `Jailbreak`, where templates dominate the run).
-   - `BaselinePolicy.Forbidden` — baseline is meaningless for this scenario's comparison axis (e.g. `AdversarialBenchmark`, which compares against gold-standard answers). Explicit `include_baseline=True` raises `ValueError`.
+2. **Optionally declare `BASELINE_ATTACK_POLICY`** (defaults to `BaselineAttackPolicy.Enabled` — a baseline `PromptSendingAttack` is prepended and callers can opt out per run via `initialize_async(include_baseline=False)`):
+   - `BaselineAttackPolicy.Disabled` — baseline supported but off by default (e.g. `Jailbreak`, where templates dominate the run).
+   - `BaselineAttackPolicy.Forbidden` — baseline is meaningless for this scenario's comparison axis (e.g. `AdversarialBenchmark`, which compares against gold-standard answers). Explicit `include_baseline=True` raises `ValueError`.
 3. **Implement three abstract methods:**
 
 ```python
 class MyScenario(Scenario):
     VERSION: int = 1
-    BASELINE_POLICY: ClassVar[BaselinePolicy] = BaselinePolicy.Enabled
+    BASELINE_ATTACK_POLICY: ClassVar[BaselineAttackPolicy] = BaselineAttackPolicy.Enabled
 
     @classmethod
     def get_strategy_class(cls) -> type[ScenarioStrategy]:

doc/code/scenarios/0_scenarios.ipynb

@@ -83,7 +83,7 @@
     "   - `max_retries`: Number of retry attempts on failure (default: 0)\n",
     "   - `memory_labels`: Optional labels for tracking (optional)\n",
     "   - `include_baseline`: Whether to prepend a baseline attack (defaults to the scenario type's\n",
-    "     `BASELINE_POLICY`; most scenarios default it on, `Jailbreak` defaults it off)\n",
+    "     `BASELINE_ATTACK_POLICY`; most scenarios default it on, `Jailbreak` defaults it off)\n",
     "\n",
     "### Example Structure\n",
     "\n",
@@ -406,11 +406,11 @@
     "Every scenario can optionally include a **baseline attack** — a `PromptSendingAttack` that sends\n",
     "each objective directly to the target without any converters or multi-turn techniques. This is\n",
     "controlled by the `include_baseline` parameter on `initialize_async`; when omitted, each\n",
-    "scenario falls back to its own `BASELINE_POLICY` class attribute (most scenarios default\n",
+    "scenario falls back to its own `BASELINE_ATTACK_POLICY` class attribute (most scenarios default\n",
     "it on; `Jailbreak` defaults it off). See\n",
     "[Common Scenario Parameters](./1_common_scenario_parameters.ipynb) for a worked example.\n",
     "\n",
-    "Custom scenarios should choose their `BASELINE_POLICY` based on whether an unmodified\n",
+    "Custom scenarios should choose their `BASELINE_ATTACK_POLICY` based on whether an unmodified\n",
     "prompt is a meaningful comparator for the scenario's strategies:\n",
     "\n",
     "- **`Enabled`** — the baseline is prepended by default and the caller can opt out. Use when an\n",

doc/code/scenarios/0_scenarios.py

@@ -85,7 +85,7 @@
 #    - `max_retries`: Number of retry attempts on failure (default: 0)
 #    - `memory_labels`: Optional labels for tracking (optional)
 #    - `include_baseline`: Whether to prepend a baseline attack (defaults to the scenario type's
-#      `BASELINE_POLICY`; most scenarios default it on, `Jailbreak` defaults it off)
+#      `BASELINE_ATTACK_POLICY`; most scenarios default it on, `Jailbreak` defaults it off)
 #
 # ### Example Structure
 #
@@ -176,11 +176,11 @@ def _build_display_group(self, *, technique_name: str, seed_group_name: str) ->
 # Every scenario can optionally include a **baseline attack** — a `PromptSendingAttack` that sends
 # each objective directly to the target without any converters or multi-turn techniques. This is
 # controlled by the `include_baseline` parameter on `initialize_async`; when omitted, each
-# scenario falls back to its own `BASELINE_POLICY` class attribute (most scenarios default
+# scenario falls back to its own `BASELINE_ATTACK_POLICY` class attribute (most scenarios default
 # it on; `Jailbreak` defaults it off). See
 # [Common Scenario Parameters](./1_common_scenario_parameters.ipynb) for a worked example.
 #
-# Custom scenarios should choose their `BASELINE_POLICY` based on whether an unmodified
+# Custom scenarios should choose their `BASELINE_ATTACK_POLICY` based on whether an unmodified
 # prompt is a meaningful comparator for the scenario's strategies:
 #
 # - **`Enabled`** — the baseline is prepended by default and the caller can opt out. Use when an

pyrit/scenario/__init__.py

@@ -21,7 +21,7 @@
     AtomicAttack,
     AttackTechnique,
     AttackTechniqueFactory,
-    BaselinePolicy,
+    BaselineAttackPolicy,
     DatasetConfiguration,
     Scenario,
     ScenarioCompositeStrategy,
@@ -51,7 +51,7 @@
     "AtomicAttack",
     "AttackTechnique",
     "AttackTechniqueFactory",
-    "BaselinePolicy",
+    "BaselineAttackPolicy",
     "DatasetConfiguration",
     "Parameter",
     "Scenario",

pyrit/scenario/core/__init__.py

@@ -8,7 +8,7 @@
 from pyrit.scenario.core.attack_technique import AttackTechnique
 from pyrit.scenario.core.attack_technique_factory import AttackTechniqueFactory, ScorerOverridePolicy
 from pyrit.scenario.core.dataset_configuration import EXPLICIT_SEED_GROUPS_KEY, DatasetConfiguration
-from pyrit.scenario.core.scenario import BaselinePolicy, Scenario
+from pyrit.scenario.core.scenario import BaselineAttackPolicy, Scenario
 from pyrit.scenario.core.scenario_strategy import ScenarioCompositeStrategy, ScenarioStrategy
 from pyrit.scenario.core.scenario_target_defaults import get_default_adversarial_target, get_default_scorer_target
 from pyrit.scenario.core.scenario_techniques import (
@@ -20,7 +20,7 @@
     "AtomicAttack",
     "AttackTechnique",
     "AttackTechniqueFactory",
-    "BaselinePolicy",
+    "BaselineAttackPolicy",
     "DatasetConfiguration",
     "EXPLICIT_SEED_GROUPS_KEY",
     "SCENARIO_TECHNIQUES",

pyrit/scenario/core/scenario.py

@@ -55,13 +55,13 @@
 logger = logging.getLogger(__name__)
 
 
-class BaselinePolicy(Enum):
+class BaselineAttackPolicy(Enum):
     """
     Declares how a scenario type treats the default baseline atomic attack.
 
     The baseline is a plain ``PromptSendingAttack`` that sends each objective unmodified,
     used as a comparison point against the scenario's strategies. Each scenario class
-    declares its policy via ``Scenario.BASELINE_POLICY``; callers can still override
+    declares its policy via ``Scenario.BASELINE_ATTACK_POLICY``; callers can still override
     at runtime via ``initialize_async(include_baseline=...)`` for the ``Enabled`` and
     ``Disabled`` states.
     """
@@ -145,7 +145,7 @@ class Scenario(ABC):
     #: ``initialize_async`` and overridable per run via ``include_baseline`` for the
     #: ``Enabled`` and ``Disabled`` states; ``Forbidden`` is a hard constraint and a
     #: caller-supplied ``include_baseline=True`` raises ``ValueError``.
-    BASELINE_POLICY: ClassVar[BaselinePolicy] = BaselinePolicy.Enabled
+    BASELINE_ATTACK_POLICY: ClassVar[BaselineAttackPolicy] = BaselineAttackPolicy.Enabled
 
     @classmethod
     def _get_additional_scoring_questions(cls) -> Sequence[Path]:
@@ -621,14 +621,14 @@ async def initialize_async(
             include_baseline (bool | None): Whether to prepend a baseline atomic attack that sends
                 all objectives without modifications, allowing comparison between unmodified prompts
                 and the scenario's strategies. If None (the default), the scenario type's
-                ``BASELINE_POLICY`` class attribute decides: ``Enabled`` includes it,
+                ``BASELINE_ATTACK_POLICY`` class attribute decides: ``Enabled`` includes it,
                 ``Disabled`` omits it, and ``Forbidden`` always omits it (and rejects an
-                explicit ``True``). Passing ``True`` to a scenario whose ``BASELINE_POLICY``
+                explicit ``True``). Passing ``True`` to a scenario whose ``BASELINE_ATTACK_POLICY``
                 is ``Forbidden`` raises ``ValueError``.
 
         Raises:
             ValueError: If no objective_target is provided, or if ``include_baseline=True`` is passed
-                to a scenario whose ``BASELINE_POLICY`` is ``Forbidden``.
+                to a scenario whose ``BASELINE_ATTACK_POLICY`` is ``Forbidden``.
         """
         # Validate required parameters
         if objective_target is None:
@@ -657,15 +657,15 @@ async def initialize_async(
         # scenario type never silently inherits a True default; explicit-True on a forbidden
         # type is a hard error rather than a silent ignore. For the Enabled / Disabled states,
         # a None runtime value defers to the policy.
-        if self.BASELINE_POLICY is BaselinePolicy.Forbidden:
+        if self.BASELINE_ATTACK_POLICY is BaselineAttackPolicy.Forbidden:
             if include_baseline is True:
                 raise ValueError(
                     f"{type(self).__name__} does not support a default baseline "
-                    f"(BASELINE_POLICY = Forbidden); pass include_baseline=False or omit the argument."
+                    f"(BASELINE_ATTACK_POLICY = Forbidden); pass include_baseline=False or omit the argument."
                 )
             include_baseline = False
         elif include_baseline is None:
-            include_baseline = self.BASELINE_POLICY is BaselinePolicy.Enabled
+            include_baseline = self.BASELINE_ATTACK_POLICY is BaselineAttackPolicy.Enabled
 
         self._include_baseline = include_baseline
 

pyrit/scenario/scenarios/benchmark/adversarial.py

@@ -15,7 +15,7 @@
 from pyrit.registry.tag_query import TagQuery
 from pyrit.scenario.core.atomic_attack import AtomicAttack
 from pyrit.scenario.core.dataset_configuration import DatasetConfiguration
-from pyrit.scenario.core.scenario import BaselinePolicy, Scenario
+from pyrit.scenario.core.scenario import BaselineAttackPolicy, Scenario
 from pyrit.scenario.core.scenario_techniques import SCENARIO_TECHNIQUES
 
 if TYPE_CHECKING:
@@ -37,7 +37,7 @@ class AdversarialBenchmark(Scenario):
 
     #: AdversarialBenchmark compares attack-success rates across adversarial models; a baseline
     #: attack would be model-independent and contribute no signal to the comparison.
-    BASELINE_POLICY: ClassVar[BaselinePolicy] = BaselinePolicy.Forbidden
+    BASELINE_ATTACK_POLICY: ClassVar[BaselineAttackPolicy] = BaselineAttackPolicy.Forbidden
 
     @classmethod
     def get_strategy_class(cls) -> type[ScenarioStrategy]:

tests/unit/scenario/test_adversarial.py

@@ -22,7 +22,7 @@
 )
 from pyrit.prompt_target import PromptTarget, TargetCapabilities, TargetConfiguration
 from pyrit.registry.object_registries.attack_technique_registry import AttackTechniqueRegistry
-from pyrit.scenario.core import AtomicAttack, BaselinePolicy
+from pyrit.scenario.core import AtomicAttack, BaselineAttackPolicy
 from pyrit.scenario.core.dataset_configuration import DatasetConfiguration
 from pyrit.scenario.core.scenario_techniques import SCENARIO_TECHNIQUES
 from pyrit.scenario.scenarios.benchmark.adversarial import AdversarialBenchmark
@@ -466,7 +466,7 @@ async def test_baseline_excluded(self, mock_objective_target, single_adversarial
             mock_objective_target=mock_objective_target,
             adversarial_models=single_adversarial_model,
         )
-        assert type(scenario).BASELINE_POLICY is BaselinePolicy.Forbidden
+        assert type(scenario).BASELINE_ATTACK_POLICY is BaselineAttackPolicy.Forbidden
         assert not any(a.atomic_attack_name == "baseline" for a in scenario._atomic_attacks)
 
     async def test_baseline_explicit_true_raises(self, mock_objective_target, single_adversarial_model):

tests/unit/scenario/test_baseline_deprecation.py

@@ -16,7 +16,7 @@
 
 from pyrit.identifiers import ComponentIdentifier
 from pyrit.scenario import DatasetConfiguration
-from pyrit.scenario.core import BaselinePolicy, Scenario, ScenarioStrategy
+from pyrit.scenario.core import BaselineAttackPolicy, Scenario, ScenarioStrategy
 from pyrit.score import TrueFalseScorer
 
 _TEST_SCORER_ID = ComponentIdentifier(class_name="MockScorer", class_module="tests.unit.scenarios")
@@ -34,7 +34,7 @@ def get_aggregate_tags(cls) -> set[str]:
 class _LegacyScenario(Scenario):
     """Minimal Scenario stand-in for exercising the deprecated baseline kwargs."""
 
-    BASELINE_POLICY: ClassVar[BaselinePolicy] = BaselinePolicy.Enabled
+    BASELINE_ATTACK_POLICY: ClassVar[BaselineAttackPolicy] = BaselineAttackPolicy.Enabled
 
     def __init__(self, **kwargs):
         kwargs.setdefault("strategy_class", _LegacyStrategy)
@@ -99,7 +99,7 @@ def test_base_kwarg_omitted_emits_no_warning(self):
         assert scenario._legacy_include_baseline is None
 
     async def test_legacy_value_drives_initialize_when_runtime_kwarg_omitted(self, mock_objective_target):
-        """Constructor-time False suppresses the baseline that BASELINE_POLICY=Enabled would add."""
+        """Constructor-time False suppresses the baseline that BASELINE_ATTACK_POLICY=Enabled would add."""
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", DeprecationWarning)
             scenario = _LegacyScenario(include_default_baseline=False)

tests/unit/scenario/test_jailbreak.py

@@ -16,7 +16,7 @@
 from pyrit.identifiers import ComponentIdentifier
 from pyrit.models import SeedGroup, SeedObjective
 from pyrit.prompt_target import PromptTarget
-from pyrit.scenario.core import BaselinePolicy
+from pyrit.scenario.core import BaselineAttackPolicy
 from pyrit.scenario.scenarios.airt.jailbreak import Jailbreak, JailbreakStrategy
 from pyrit.score.true_false.true_false_inverter_scorer import TrueFalseInverterScorer
 
@@ -203,14 +203,14 @@ async def test_init_raises_exception_when_no_datasets_available(self, mock_objec
         with pytest.raises(ValueError, match="DatasetConfiguration has no seed_groups"):
             await scenario.initialize_async(objective_target=mock_objective_target)
 
-    def test_class_inherits_default_baseline_policy(self):
+    def test_class_inherits_default_baseline_attack_policy(self):
         """Jailbreak inherits the base default (Enabled) — baseline included by default."""
-        assert Jailbreak.BASELINE_POLICY is BaselinePolicy.Enabled
+        assert Jailbreak.BASELINE_ATTACK_POLICY is BaselineAttackPolicy.Enabled
 
     async def test_default_initialize_includes_baseline(
         self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups
     ):
-        """initialize_async without include_baseline honors BASELINE_POLICY=Enabled."""
+        """initialize_async without include_baseline honors BASELINE_ATTACK_POLICY=Enabled."""
         with patch.object(Jailbreak, "_resolve_seed_groups", return_value=mock_memory_seed_groups):
             scenario = Jailbreak(objective_scorer=mock_objective_scorer)
             await scenario.initialize_async(objective_target=mock_objective_target)