Merge remote-tracking branch 'origin/main' into riedgar-ms/selfask-jsonschema-01

Author: riedgar-ms

.github/copilot-instructions.md

@@ -28,4 +28,13 @@ Do NOT leave comments about:
 
 Aim for fewer, higher-signal comments. A review with 2-3 important comments is better than 15 trivial ones.
 
-Follow `.github/instructions/style-guide.instructions.md` for style guidelines. And look in `.github/instructions/` for specific instructions on the different components.
+## Instruction Files
+
+BEFORE editing or code-reviewing any file, you MUST read the `.github/instructions/` files whose `applyTo` patterns match the files you are about to edit. For example:
+- Editing/code-reviewing `pyrit/**/*.py` → read `style-guide.instructions.md` and `user-custom.instructions.md`
+- Editing/code-reviewing `pyrit/scenario/**` → also read `scenarios.instructions.md`
+- Editing/code-reviewing `pyrit/prompt_converter/**` → also read `converters.instructions.md`
+- Editing/code-reviewing `tests/**` → also read `test.instructions.md`
+- Editing/code-reviewing `doc/**/*.py` or `doc/**/*.ipynb` → also read `docs.instructions.md`
+
+Follow every rule in the applicable instruction files. Do not skip this step.

.pyrit_conf_example

@@ -24,9 +24,10 @@ memory_db_type: sqlite
 # Available initializers:
 # - simple: Basic OpenAI configuration (requires OPENAI_CHAT_* env vars)
 # - airt: AI Red Team setup with Azure OpenAI (requires AZURE_OPENAI_* env vars)
+# - targets: Registers available prompt targets into the TargetRegistry
+# - scorers: Registers pre-configured scorers into the ScorerRegistry
 # - load_default_datasets: Loads default datasets for all registered scenarios
 # - objective_list: Sets default objectives for scenarios
-# - openai_objective_target: Sets up OpenAI target for scenarios
 #
 # Each initializer can be specified as:
 # - A simple string (name only)
@@ -38,13 +39,14 @@ memory_db_type: sqlite
 # Example:
 # initializers:
 #   - simple
-#   - name: target
+#   - name: targets
 #     args:
 #       tags:
 #         - default
 #         - scorer
 initializers:
   - name: simple
+  - name: load_default_datasets
   - name: scorers
   - name: targets
     args:

doc/code/scenarios/1_configuring_scenarios.ipynb

@@ -36,8 +36,9 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Found default environment files: ['./.pyrit/.env']\n",
-      "Loaded environment file: ./.pyrit/.env\n"
+      "Found default environment files: ['./.pyrit/.env', './.pyrit/.env.local']\n",
+      "Loaded environment file: ./.pyrit/.env\n",
+      "Loaded environment file: ./.pyrit/.env.local\n"
      ]
     }
    ],
@@ -74,47 +75,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "\r",
-      "Loading datasets - this can take a few minutes:   0%|                                      | 0/46 [00:00<?, ?dataset/s]"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\r",
-      "Loading datasets - this can take a few minutes:   2%|▋                             | 1/46 [00:00<00:20,  2.20dataset/s]"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\r",
-      "Loading datasets - this can take a few minutes:  43%|████████████▌                | 20/46 [00:00<00:00, 45.31dataset/s]"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\r",
-      "Loading datasets - this can take a few minutes:  67%|███████████████████▌         | 31/46 [00:00<00:00, 39.43dataset/s]"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\r",
-      "Loading datasets - this can take a few minutes: 100%|█████████████████████████████| 46/46 [00:00<00:00, 51.19dataset/s]"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\n"
+      "Loading datasets - this can take a few minutes: 100%|██████████| 58/58 [00:00<00:00, 69.09dataset/s]\n"
      ]
     }
    ],
@@ -222,7 +183,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "fe242b7b0aaa476187fb5643e41cdd94",
+       "model_id": "811109e1b31543f3835ee76ed5c708bd",
        "version_major": 2,
        "version_minor": 0
       },
@@ -268,7 +229,7 @@
       "\u001b[1m  📋 Scenario Details\u001b[0m\n",
       "\u001b[36m    • Name: RedTeamAgent\u001b[0m\n",
       "\u001b[36m    • Scenario Version: 1\u001b[0m\n",
-      "\u001b[36m    • PyRIT Version: 0.10.1.dev0\u001b[0m\n",
+      "\u001b[36m    • PyRIT Version: 0.11.1.dev0\u001b[0m\n",
       "\u001b[36m    • Description:\u001b[0m\n",
       "\u001b[36m        RedTeamAgent is a preconfigured scenario that automatically generates multiple AtomicAttack instances based on\u001b[0m\n",
       "\u001b[36m        the specified attack strategies. It supports both single-turn attacks (with various converters) and multi-turn\u001b[0m\n",
@@ -280,58 +241,49 @@
       "\u001b[1m  🎯 Target Information\u001b[0m\n",
       "\u001b[36m    • Target Type: OpenAIChatTarget\u001b[0m\n",
       "\u001b[36m    • Target Model: gpt-4o\u001b[0m\n",
-      "\u001b[36m    • Target Endpoint: https://pyrit-japan-test.openai.azure.com/openai/v1\u001b[0m\n",
+      "\u001b[36m    • Target Endpoint: https://pyrit-dev.openai.azure.com/openai/v1\u001b[0m\n",
       "\n",
       "\u001b[1m  📊 Scorer Information\u001b[0m\n",
       "\u001b[37m    ▸ Scorer Identifier\u001b[0m\n",
-      "\u001b[36m      • Scorer Type: TrueFalseCompositeScorer\u001b[0m\n",
-      "\u001b[36m      • Score Aggregator: AND_\u001b[0m\n",
-      "\u001b[36m        └─ Composite of 2 scorer(s):\u001b[0m\n",
-      "\u001b[36m            • Scorer Type: FloatScaleThresholdScorer\u001b[0m\n",
-      "\u001b[36m            • Score Aggregator: OR_\u001b[0m\n",
-      "\u001b[36m            • threshold: 0.5\u001b[0m\n",
-      "\u001b[36m            • float_scale_aggregator: MAX_\u001b[0m\n",
-      "\u001b[36m              └─ Composite of 1 scorer(s):\u001b[0m\n",
-      "\u001b[36m                  • Scorer Type: AzureContentFilterScorer\u001b[0m\n",
-      "\u001b[36m                  • score_categories: ['Hate', 'SelfHarm', 'Sexual', 'Violence']\u001b[0m\n",
-      "\u001b[36m            • Scorer Type: TrueFalseInverterScorer\u001b[0m\n",
-      "\u001b[36m            • Score Aggregator: OR_\u001b[0m\n",
-      "\u001b[36m              └─ Composite of 1 scorer(s):\u001b[0m\n",
-      "\u001b[36m                  • Scorer Type: SelfAskRefusalScorer\u001b[0m\n",
-      "\u001b[36m                  • Target Model: gpt-4o\u001b[0m\n",
-      "\u001b[36m                  • Temperature: 0.9\u001b[0m\n",
-      "\u001b[36m                  • Score Aggregator: OR_\u001b[0m\n",
+      "\u001b[36m      • Scorer Type: TrueFalseInverterScorer\u001b[0m\n",
+      "\u001b[36m      • scorer_type: true_false\u001b[0m\n",
+      "\u001b[36m      • score_aggregator: OR_\u001b[0m\n",
+      "\u001b[36m        └─ Composite of 1 scorer(s):\u001b[0m\n",
+      "\u001b[36m            • Scorer Type: SelfAskRefusalScorer\u001b[0m\n",
+      "\u001b[36m            • scorer_type: true_false\u001b[0m\n",
+      "\u001b[36m            • score_aggregator: OR_\u001b[0m\n",
+      "\u001b[36m            • model_name: gpt-4o\u001b[0m\n",
       "\n",
       "\u001b[37m    ▸ Performance Metrics\u001b[0m\n",
-      "\u001b[31m      • Accuracy: 54.05%\u001b[0m\n",
-      "\u001b[36m      • Accuracy Std Error: ±0.0410\u001b[0m\n",
-      "\u001b[31m      • F1 Score: 0.2273\u001b[0m\n",
-      "\u001b[36m      • Precision: 0.7143\u001b[0m\n",
-      "\u001b[31m      • Recall: 0.1351\u001b[0m\n",
-      "\u001b[36m      • Average Score Time: 0.76s\u001b[0m\n",
+      "\u001b[36m      • Accuracy: 84.84%\u001b[0m\n",
+      "\u001b[36m      • Accuracy Std Error: ±0.0185\u001b[0m\n",
+      "\u001b[36m      • F1 Score: 0.8606\u001b[0m\n",
+      "\u001b[36m      • Precision: 0.7928\u001b[0m\n",
+      "\u001b[32m      • Recall: 0.9412\u001b[0m\n",
+      "\u001b[36m      • Average Score Time: 1.27s\u001b[0m\n",
       "\n",
       "\u001b[1m\u001b[36m▼ Overall Statistics\u001b[0m\n",
       "\u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n",
       "\u001b[1m  📈 Summary\u001b[0m\n",
       "\u001b[32m    • Total Strategies: 4\u001b[0m\n",
       "\u001b[32m    • Total Attack Results: 8\u001b[0m\n",
-      "\u001b[32m    • Overall Success Rate: 0%\u001b[0m\n",
+      "\u001b[36m    • Overall Success Rate: 25%\u001b[0m\n",
       "\u001b[32m    • Unique Objectives: 4\u001b[0m\n",
       "\n",
       "\u001b[1m\u001b[36m▼ Per-Strategy Breakdown\u001b[0m\n",
       "\u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n",
       "\n",
       "\u001b[1m  🔸 Strategy: baseline\u001b[0m\n",
       "\u001b[33m    • Number of Results: 2\u001b[0m\n",
-      "\u001b[32m    • Success Rate: 0%\u001b[0m\n",
+      "\u001b[33m    • Success Rate: 50%\u001b[0m\n",
       "\n",
       "\u001b[1m  🔸 Strategy: base64\u001b[0m\n",
       "\u001b[33m    • Number of Results: 2\u001b[0m\n",
       "\u001b[32m    • Success Rate: 0%\u001b[0m\n",
       "\n",
       "\u001b[1m  🔸 Strategy: binary\u001b[0m\n",
       "\u001b[33m    • Number of Results: 2\u001b[0m\n",
-      "\u001b[32m    • Success Rate: 0%\u001b[0m\n",
+      "\u001b[33m    • Success Rate: 50%\u001b[0m\n",
       "\n",
       "\u001b[1m  🔸 Strategy: ComposedStrategy(caesar, char_swap)\u001b[0m\n",
       "\u001b[33m    • Number of Results: 2\u001b[0m\n",

pyrit/cli/_banner.py

@@ -296,7 +296,7 @@ def add(line: str, role: ColorRole, segments: Optional[list[tuple[int, int, Colo
     quick_start = [
         "Quick Start:",
         "  pyrit> list-scenarios",
-        "  pyrit> run foundry --initializers openai_objective_target load_default_datasets",
+        "  pyrit> run foundry --target my_target --initializers targets load_default_datasets",
     ]
     for qs in quick_start:
         full_line = _box_line("  " + qs)

pyrit/cli/_cli_args.py

@@ -286,6 +286,9 @@ def parse_memory_labels(json_string: str) -> dict[str, str]:
     "Creates a new dataset config; fetches all items unless --max-dataset-size is also specified",
     "max_dataset_size": "Maximum number of items to use from the dataset (must be >= 1). "
     "Limits new datasets if --dataset-names provided, otherwise overrides scenario's default limit",
+    "target": "Name of a registered target from the TargetRegistry to use as the objective target. "
+    "Targets are registered by initializers (e.g., 'targets' initializer). "
+    "Use --list-targets to see available target names after initializers have run",
 }
 
 
@@ -372,15 +375,14 @@ def parse_run_arguments(*, args_string: str) -> dict[str, Any]:
         "scenario_name": parts[0],
         "initializers": None,
         "initialization_scripts": None,
-        "env_files": None,
         "scenario_strategies": None,
         "max_concurrency": None,
         "max_retries": None,
         "memory_labels": None,
-        "database": None,
         "log_level": None,
         "dataset_names": None,
         "max_dataset_size": None,
+        "target": None,
     }
 
     i = 1
@@ -399,13 +401,6 @@ def parse_run_arguments(*, args_string: str) -> dict[str, Any]:
             while i < len(parts) and not parts[i].startswith("--"):
                 result["initialization_scripts"].append(parts[i])
                 i += 1
-        elif parts[i] == "--env-files":
-            # Collect env file paths until next flag
-            result["env_files"] = []
-            i += 1
-            while i < len(parts) and not parts[i].startswith("--"):
-                result["env_files"].append(parts[i])
-                i += 1
         elif parts[i] in ("--strategies", "-s"):
             # Collect strategies until next flag
             result["scenario_strategies"] = []
@@ -431,12 +426,6 @@ def parse_run_arguments(*, args_string: str) -> dict[str, Any]:
                 raise ValueError("--memory-labels requires a value")
             result["memory_labels"] = parse_memory_labels(parts[i])
             i += 1
-        elif parts[i] == "--database":
-            i += 1
-            if i >= len(parts):
-                raise ValueError("--database requires a value")
-            result["database"] = validate_database(database=parts[i])
-            i += 1
         elif parts[i] == "--log-level":
             i += 1
             if i >= len(parts):
@@ -456,6 +445,12 @@ def parse_run_arguments(*, args_string: str) -> dict[str, Any]:
                 raise ValueError("--max-dataset-size requires a value")
             result["max_dataset_size"] = validate_integer(parts[i], name="--max-dataset-size", min_value=1)
             i += 1
+        elif parts[i] == "--target":
+            i += 1
+            if i >= len(parts):
+                raise ValueError("--target requires a value")
+            result["target"] = parts[i]
+            i += 1
         else:
             raise ValueError(f"Unknown argument: {parts[i]}")
 
@@ -470,24 +465,12 @@ def parse_run_arguments(*, args_string: str) -> dict[str, Any]:
 def add_common_arguments(parser: argparse.ArgumentParser) -> None:
     """Add arguments shared between pyrit_shell and pyrit_scan."""
     parser.add_argument("--config-file", type=Path, help=ARG_HELP["config_file"])
-    parser.add_argument(
-        "--database",
-        type=validate_database_argparse,
-        default=None,
-        help=f"Database type to use ({IN_MEMORY}, {SQLITE}, {AZURE_SQL}). Defaults to config or {SQLITE}.",
-    )
     parser.add_argument(
         "--log-level",
         type=validate_log_level_argparse,
         default=logging.WARNING,
         help="Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) (default: WARNING)",
     )
-    parser.add_argument(
-        "--env-files",
-        type=str,
-        nargs="+",
-        help=ARG_HELP["env_files"],
-    )
 
 
 # Module-level logger (stdlib only — no heavy deps)