FEAT: Adversarial Benchmark Scenario Refactor

.env_example

@@ -75,10 +75,24 @@ AZURE_OPENAI_GPT4O_UNSAFE_CHAT_MODEL2="deployment-name"
 AZURE_OPENAI_GPT4O_UNSAFE_CHAT_UNDERLYING_MODEL2=""
 
 # Adversarial chat target (used by scenario attack techniques, e.g. role-play, TAP)
+# Default endpoint goes here; specialized ones below
 ADVERSARIAL_CHAT_ENDPOINT="https://xxxxx.openai.azure.com/openai/v1"
 ADVERSARIAL_CHAT_KEY="xxxxx"
 ADVERSARIAL_CHAT_MODEL="deployment-name"
 
+ADVERSARIAL_CHAT_SINGLETURN_ENDPOINT="https://xxxxxx.westus3.inference.ml.azure.com/score"
+ADVERSARIAL_CHAT_SINGLETURN_KEY="xxxxx"
+ADVERSARIAL_CHAT_SINGLETURN_MODEL="deployment-name"
+
+ADVERSARIAL_CHAT_MULTITURN_ENDPOINT="https://xxxxxx.westus3.inference.ml.azure.com/score"
+ADVERSARIAL_CHAT_MULTITURN_KEY="xxxxx"
+ADVERSARIAL_CHAT_MULTITURN_MODEL="deployment-name"
+
+ADVERSARIAL_CHAT_REASONING_ENDPOINT="https://xxxxxx.westus3.inference.ml.azure.com/score"
+ADVERSARIAL_CHAT_REASONING_KEY="xxxxx"
+ADVERSARIAL_CHAT_REASONING_MODEL="deployment-name"
+
+
 # Objective Scorer chat target (used in scorers in scenarios)
 OBJECTIVE_SCORER_CHAT_ENDPOINT="https://xxxxx.openai.azure.com/openai/v1"
 OBJECTIVE_SCORER_CHAT_KEY="xxxxx"

doc/scanner/benchmark.py

@@ -11,37 +11,186 @@
 # %% [markdown]
 # # Benchmark Scenarios
 #
-# Benchmark scenarios are a subset of scenarios that compare the effectiveness of attacks across an axis that varies within the scenario itself. The axis can be many things; currently, the only benchmark variant is the adversarial benchmark, whose axis of change is the adversarial model used in attacks.
+# Benchmark scenarios compare the effectiveness of attacks across an axis that varies within the
+# scenario itself. The axis can be many things; currently, the only benchmark variant is the
+# adversarial benchmark, whose axis of change is the **adversarial chat helper model** used in
+# attacks.
 
 # %% [markdown]
 # ## Adversarial Benchmark
-# The adversarial benchmarking scenario (`AdversarialBenchmark`) compares the effectiveness of different adversarial models in successfully executing attacks against a target model.
+#
+# `AdversarialBenchmark` holds the objective target and dataset constant and varies the adversarial
+# chat model used to drive multi-turn attacks (and crescendo-style simulated conversations). Useful
+# for evaluating which adversarial helper models produce stronger or weaker attack success rates
+# against the same target.
+#
+# Model fan-out is owned by `BenchmarkInitializer`, which discovers every ADVERSARIAL-tagged target
+# in `TargetRegistry` (populated by `TargetInitializer` from `ADVERSARIAL_CHAT_*` env vars) and
+# registers one fanned variant per `(adversarial-capable technique, target)` pair into
+# `AttackTechniqueRegistry`. The scenario then reads those variants when it builds its strategy
+# enum.
+#
+# ### Prerequisites
+#
+# Set at least one `ADVERSARIAL_CHAT_*` group of env vars (see `.env_example`):
+#
+# ```bash
+# # Default adversarial target (always available when set)
+# ADVERSARIAL_CHAT_ENDPOINT="https://your-endpoint.openai.azure.com/openai/v1"
+# ADVERSARIAL_CHAT_KEY="your-key"
+# ADVERSARIAL_CHAT_MODEL="deployment-name"
+#
+# # Optional turn-style variants — auto-discovered by TargetInitializer when set
+# ADVERSARIAL_CHAT_SINGLETURN_ENDPOINT="..."
+# ADVERSARIAL_CHAT_SINGLETURN_KEY="..."
+# ADVERSARIAL_CHAT_SINGLETURN_MODEL="..."
+# # ADVERSARIAL_CHAT_MULTITURN_* and ADVERSARIAL_CHAT_REASONING_* follow the same pattern
+# ```
+#
+# If no adversarial-tagged target is registered, `BenchmarkInitializer.initialize_async` raises
+# `ValueError` naming the env vars to set.
+#
+# ### CLI quickstart
+#
+# ```bash
+# pyrit_scan benchmark.adversarial \
+#   --initializers target load_default_datasets benchmark \
+#   --target openai_chat \
+#   --max-dataset-size 4
+# ```
+#
+# **Available strategies (depend on registered adversarial targets):** `all`, `light`, `single_turn`,
+# `multi_turn`, plus one concrete member per fanned variant named
+# `f"{source_technique}__{target_name}"` (e.g. `red_teaming__adversarial_chat`,
+# `tap__adversarial_chat_singleturn`).
+
+# %% [markdown]
+# ## Setup
+#
+# `BenchmarkInitializer` runs after `TargetInitializer` so adversarial-tagged targets are present
+# in the registry before the fan-out logic queries it. The initializer chain is executed in list
+# order.
 
 # %%
 from pyrit.output import output_scenario_async
 from pyrit.prompt_target import OpenAIChatTarget
+from pyrit.scenario import DatasetConfiguration
 from pyrit.scenario.scenarios.benchmark import AdversarialBenchmark
 from pyrit.setup import IN_MEMORY, initialize_pyrit_async
-from pyrit.setup.initializers import LoadDefaultDatasets
+from pyrit.setup.initializers import (
+    BenchmarkInitializer,
+    LoadDefaultDatasets,
+    ScorerInitializer,
+    TargetInitializer,
+)
+
+await initialize_pyrit_async(  # type: ignore
+    memory_db_type=IN_MEMORY,
+    initializers=[
+        TargetInitializer(),
+        ScorerInitializer(),
+        LoadDefaultDatasets(),
+        BenchmarkInitializer(),
+    ],
+)
 
-await initialize_pyrit_async(memory_db_type=IN_MEMORY, initializers=[LoadDefaultDatasets()])  # type: ignore
+objective_target = OpenAIChatTarget()
 
-# Pass any number of adversarial PromptTarget instances (with chat-target
-# capabilities — multi-turn and editable history) as a list; AdversarialBenchmark
-# infers a label for each from its identifier and runs every benchmark-friendly
-# attack technique against the objective target with each adversarial model.
-adversarial_model = OpenAIChatTarget()
+# %% [markdown]
+# ## Run the benchmark
+#
+# Instantiate with no model arguments — adversarial models come from the registry via
+# `BenchmarkInitializer`. The default strategy (`light`) runs the benchmark-friendly subset of
+# fanned variants for a quick comparison.
 
-benchmark_scenario = AdversarialBenchmark(adversarial_models=[adversarial_model])
+# %%
+dataset_config = DatasetConfiguration(dataset_names=["harmbench"], max_dataset_size=4)
 
-await benchmark_scenario.initialize_async(  # type: ignore
-    objective_target=OpenAIChatTarget(), max_concurrency=2
+scenario = AdversarialBenchmark()
+await scenario.initialize_async(  # type: ignore
+    objective_target=objective_target,
+    dataset_config=dataset_config,
 )
 
-baseline_result = await benchmark_scenario.run_async()  # type: ignore
+baseline_result = await scenario.run_async()  # type: ignore
 
-# Resume handle: re-run with `AdversarialBenchmark(..., scenario_result_id=<this id>)` to pick
-# up where this run left off (constructor args must match the original run).
+# Save this id to resume the run later via AdversarialBenchmark(scenario_result_id=...).
 print(f"Scenario result id: {baseline_result.id}")
 
 await output_scenario_async(baseline_result)
+
+# %% [markdown]
+# ## Cross-run caching
+#
+# Re-run the benchmark with `skip_cached=True` and atomic attacks that completed (`SUCCESS` or
+# `FAILURE` outcome) in any prior `COMPLETED` run of the same scenario name + version are skipped.
+# `ERROR` and `UNDETERMINED` outcomes always retry, so transient failures don't poison the cache.
+#
+# The cache key is `(atomic_attack_name, technique_eval_hash)` — two atomic attacks that share a
+# name but use different technique configurations (e.g. different scorer) don't cross-pollinate.
+#
+# Useful for:
+# * Resuming a long-running benchmark after a crash or `Ctrl-C`.
+# * Incrementally adding new adversarial targets without re-running the existing ones — the new
+#   fanned variants have new names, so they don't match any cached entry and execute fresh.
+
+# %%
+scenario_cached = AdversarialBenchmark(skip_cached=True)
+await scenario_cached.initialize_async(  # type: ignore
+    objective_target=objective_target,
+    dataset_config=dataset_config,
+)
+
+cached_result = await scenario_cached.run_async()  # type: ignore
+
+await output_scenario_async(cached_result)
+
+# %% [markdown]
+# ## Narrowing the fan-out
+#
+# Limit the benchmark to a specific subset of registered adversarial targets via
+# `BenchmarkInitializer`'s `target_names` parameter. Settable from `.pyrit_conf` (see next section),
+# or programmatically:
+#
+# ```python
+# narrow_init = BenchmarkInitializer()
+# narrow_init.set_params_from_args(args={"target_names": ["adversarial_chat_singleturn"]})
+# await narrow_init.initialize_async()
+# ```
+#
+# Unknown names raise `ValueError` listing both the unknowns and the discovered set, so typos fail
+# loudly rather than silently expanding the run.
+
+# %% [markdown]
+# ## Bootstrapping from `.pyrit_conf`
+#
+# For production / repeated runs, declare the full initializer chain in `.pyrit_conf`. Initializers
+# run in list order; `target` must precede `benchmark` so adversarial-tagged targets are present
+# when `BenchmarkInitializer` queries the registry.
+#
+# ```yaml
+# memory_db_type: duckdb
+# initializers:
+#   - name: target
+#   - name: scorer
+#   - name: load_default_datasets
+#   - name: benchmark
+#     args:
+#       target_names:
+#         - adversarial_chat_singleturn
+#         - adversarial_chat_reasoning
+# scenario:
+#   name: benchmark.adversarial
+# ```
+#
+# Then run `pyrit_scan --config-file .pyrit_conf` — the scenario picks up the initializer-registered
+# fan-out automatically.
+
+# %% [markdown]
+# ## Scorer flexibility (forward-looking)
+#
+# `AdversarialBenchmark.__init__`'s `objective_scorer` parameter is typed as the broad `Scorer`
+# base class. The current runtime contract is still `TrueFalseScorer` — passing a non-true/false
+# scorer raises `TypeError` with a pointer to the planned scoring follow-up. The annotation is
+# widened ahead of runtime support so callers coding against the signature today see the eventual
+# contract, and the follow-up that drops the guard won't require a signature change.

pyrit/registry/object_registries/target_registry.py

@@ -18,6 +18,8 @@
 
 if TYPE_CHECKING:
     from pyrit.prompt_target import PromptTarget
+    from pyrit.registry.object_registries.base_instance_registry import RegistryEntry
+    from pyrit.registry.tag_query import TagQuery
 
 logger = logging.getLogger(__name__)
 
@@ -74,3 +76,30 @@ def get_instance_by_name(self, name: str) -> Optional[PromptTarget]:
             The target instance, or None if not found.
         """
         return self.get(name)
+
+    def get_by_tag_query(self, *, query: TagQuery) -> list[RegistryEntry[PromptTarget]]:
+        """
+        Get all entries whose tag keys satisfy ``query``.
+
+        ``TagQuery`` operates on a tag set, so this method matches against
+        ``entry.tags.keys()`` and ignores tag values. For value-aware
+        single-tag lookups use ``get_by_tag(*, tag, value)`` on the base
+        class.
+
+        Composite queries compose with ``&`` and ``|`` operators, e.g.
+        ``TagQuery.all("adversarial") & TagQuery.any_of("singleturn", "multiturn")``.
+
+        Args:
+            query: The tag predicate to evaluate against each entry.
+
+        Returns:
+            List of matching ``RegistryEntry`` objects sorted by registry name.
+        """
+        results: list[RegistryEntry[PromptTarget]] = []
+        # Note: this erases insertion order, but respects the base_instance_registry pattern
+        # (get_by_tag).
+        for name in sorted(self._registry_items.keys()):
+            entry = self._registry_items[name]
+            if query.matches(set(entry.tags.keys())):
+                results.append(entry)
+        return results