[BREAKING] FIX: CLI Updates (microsoft#1545)

Author: jsong468

.pyrit_conf_example

@@ -24,8 +24,8 @@ memory_db_type: sqlite
 # Available initializers:
 # - simple: Basic OpenAI configuration (requires OPENAI_CHAT_* env vars)
 # - airt: AI Red Team setup with Azure OpenAI (requires AZURE_OPENAI_* env vars)
-# - targets: Registers available prompt targets into the TargetRegistry
-# - scorers: Registers pre-configured scorers into the ScorerRegistry
+# - target: Registers available prompt targets into the TargetRegistry
+# - scorer: Registers pre-configured scorers into the ScorerRegistry
 # - load_default_datasets: Loads default datasets for all registered scenarios
 # - objective_list: Sets default objectives for scenarios
 #
@@ -39,16 +39,16 @@ memory_db_type: sqlite
 # Example:
 # initializers:
 #   - simple
-#   - name: targets
+#   - name: target
 #     args:
 #       tags:
 #         - default
 #         - scorer
 initializers:
   - name: simple
   - name: load_default_datasets
-  - name: scorers
-  - name: targets
+  - name: scorer
+  - name: target
     args:
       tags:
         - default

build_scripts/env_local_integration_test

@@ -17,7 +17,7 @@ OPENAI_TTS_KEY=${OPENAI_TTS_KEY2}
 AZURE_SQL_DB_CONNECTION_STRING=${AZURE_SQL_DB_CONNECTION_STRING_TEST}
 AZURE_STORAGE_ACCOUNT_DB_DATA_CONTAINER_URL=${AZURE_STORAGE_ACCOUNT_DB_DATA_CONTAINER_URL_TEST}
 
-# E2E scenario test variables (used by openai_objective_target initializer)
+# E2E scenario test variables (used by target initializer)
 DEFAULT_OPENAI_FRONTEND_ENDPOINT=${AZURE_OPENAI_INTEGRATION_TEST_ENDPOINT}
 DEFAULT_OPENAI_FRONTEND_KEY=${AZURE_OPENAI_INTEGRATION_TEST_KEY}
 DEFAULT_OPENAI_FRONTEND_MODEL=${AZURE_OPENAI_INTEGRATION_TEST_MODEL}

doc/code/front_end/1_pyrit_scan.ipynb

@@ -5,11 +5,7 @@
 #       extension: .py
 #       format_name: percent
 #       format_version: '1.3'
-#       jupytext_version: 1.17.3
-#   kernelspec:
-#     display_name: pyrit-dev
-#     language: python
-#     name: python3
+#       jupytext_version: 1.19.1
 # ---
 
 # %% [markdown]
@@ -47,7 +43,7 @@
 #
 # PyRITInitializers are how you can configure the CLI scanner. PyRIT includes several built-in initializers you can use with the `--initializers` flag.
 #
-# The `--list-initializers` command shows all available initializers. Initializers are referenced by their filename (e.g., `objective_target`, `objective_list`, `simple`) regardless of which subdirectory they're in.
+# The `--list-initializers` command shows all available initializers. Initializers are referenced by their filename (e.g., `target`, `objective_list`, `simple`) regardless of which subdirectory they're in.
 #
 # List the available initializers using the --list-initializers flag.
 
@@ -60,13 +56,13 @@
 # You need a single scenario to run, you need two things:
 #
 # 1. A Scenario. Many are defined in `pyrit.scenario.scenarios`. But you can also define your own in initialization_scripts.
-# 2. Initializers (which can be supplied via `--initializers` or `--initialization-scripts`). Scenarios often don't need many arguments, but they can be configured in different ways. And at the very least, most need an `objective_target` (the thing you're running a scan against).
+# 2. Initializers (which can be supplied via `--initializers` or `--initialization-scripts` or `initializers` section of config file (see [here](../../getting_started/pyrit_conf.md))). Scenarios often don't need many arguments, but they can be configured in different ways. And at the very least, most need an `objective_target` (the thing you're running a scan against) which you can configure by using the `--target` flag if your initializer registers targets (e.g. `target` initializer)
 # 3. Scenario Strategies (optional). These are supplied by the `--scenario-strategies` flag and tell the scenario what to test, but they are always optional. Also note you can obtain these by running `--list-scenarios`
 #
 # Basic usage will look something like:
 #
 # ```shell
-# pyrit_scan <scenario> --initializers <initializer1> <initializer2> --scenario-strategies <strategy1> <strategy2>
+# pyrit_scan <scenario> --target <target_name> --initializers <initializer1> <initializer2> --scenario-strategies <strategy1> <strategy2>
 # ```
 #
 # You can also override scenario parameters directly from the CLI:
@@ -78,29 +74,29 @@
 # Or concretely:
 #
 # ```shell
-# !pyrit_scan foundry.red_team_agent --initializers simple openai_objective_target --scenario-strategies base64
+# !pyrit_scan foundry.red_team_agent --target openai_chat --initializers load_default_datasets target --scenario-strategies base64
 # ```
 #
-# Example with a basic configuration that runs the Foundry scenario against the objective target defined in `openai_objective_target` (which just is an OpenAIChatTarget with `DEFAULT_OPENAI_FRONTEND_ENDPOINT` and `DEFAULT_OPENAI_FRONTEND_KEY`).
+# Example with a basic configuration that runs the Foundry scenario against the objective target defined in the `target` initializer.
 
 # %%
-# !pyrit_scan foundry.red_team_agent --initializers openai_objective_target --strategies base64
+# !pyrit_scan foundry.red_team_agent --target openai_chat --initializers load_default_datasets target --strategies base64
 
 # %% [markdown]
 # Or with all options and multiple initializers and multiple strategies:
 #
 # ```shell
-# pyrit_scan foundry.red_team_agent --database InMemory --initializers simple objective_target objective_list --scenario-strategies easy crescendo
+# pyrit_scan foundry.red_team_agent --target openai_chat --initializers load_default_datasets target --strategies easy crescendo
 # ```
 #
 # You can also override scenario execution parameters:
 #
 # ```shell
 # # Override concurrency and retry settings
-# pyrit_scan foundry.red_team_agent --initializers simple objective_target --max-concurrency 10 --max-retries 3
+# pyrit_scan foundry.red_team_agent --target openai_chat --initializers load_default_datasets target --max-concurrency 10 --max-retries 3
 #
 # # Add custom memory labels for tracking (must be valid JSON)
-# pyrit_scan foundry.red_team_agent --initializers simple objective_target --memory-labels '{"experiment": "test1", "version": "v2", "researcher": "alice"}'
+# pyrit_scan foundry.red_team_agent --target openai_chat --initializers load_default_datasets target --memory-labels '{"experiment": "test1", "version": "v2", "researcher": "alice"}'
 # ```
 #
 # Available CLI parameter overrides:
@@ -175,6 +171,7 @@ async def _get_atomic_attacks_async(self):
 await initialize_pyrit_async(memory_db_type="InMemory")  # type: ignore
 MyCustomScenario()
 
+
 # %% [markdown]
 # Then discover and run it:
 #

doc/code/front_end/1_pyrit_scan.py

@@ -13,27 +13,29 @@ pyrit_shell
 With startup options:
 
 ```bash
-# Set default database for all runs
-pyrit_shell --database InMemory
+# Load configuration file (if not provided, defaults to ~/.pyrit/.pyrit_conf if it exists)
+# to set database preference, initializers, labels, env_file, and more.
+pyrit_shell --config-file ./.pyrit_conf
 
 # Set default log level
 pyrit_shell --log-level DEBUG
 
 # Load initializers at startup
-pyrit_shell --initializers openai_objective_target load_default_datasets
+pyrit_shell --initializers load_default_datasets
 
 # Load custom initialization scripts
 pyrit_shell --initialization-scripts ./my_config.py
 ```
 
 ## Available Commands
 
-Once in the shell, you have access to:
+Once starting the shell, you will see the list of commands you have access to. Some of them are shown below:
 
 | Command | Description |
 |---------|-------------|
 | `list-scenarios` | List all available scenarios |
 | `list-initializers` | List all available initializers |
+| `list-targets` | List all available targets from the registry |
 | `run <scenario> [options]` | Run a scenario with optional parameters |
 | `scenario-history` | List all previous scenario runs in this session |
 | `print-scenario [N]` | Print detailed results for scenario run(s) |
@@ -48,32 +50,32 @@ The `run` command executes scenarios with the same options as `pyrit_scan`:
 ### Basic Usage
 
 ```bash
-pyrit> run foundry.red_team_agent --initializers openai_objective_target load_default_datasets
+pyrit> run foundry.red_team_agent --target my_target --initializers target load_default_datasets
 ```
 
 ### With Strategies
 
 ```bash
-pyrit> run garak.encoding --initializers openai_objective_target load_default_datasets --strategies base64 rot13
+pyrit> run garak.encoding --target my_target --initializers target load_default_datasets --strategies base64 rot13
 
-pyrit> run foundry.red_team_agent --initializers openai_objective_target load_default_datasets -s jailbreak crescendo
+pyrit> run foundry.red_team_agent --target my_target --initializers target load_default_datasets -s jailbreak crescendo
 ```
 
 ### With Runtime Parameters
 
 ```bash
 # Set concurrency and retries
-pyrit> run foundry.red_team_agent --initializers openai_objective_target load_default_datasets --max-concurrency 10 --max-retries 3
+pyrit> run foundry.red_team_agent --target my_target --initializers target load_default_datasets --max-concurrency 10 --max-retries 3
 
 # Add memory labels for tracking
-pyrit> run garak.encoding --initializers openai_objective_target load_default_datasets --memory-labels '{"experiment":"test1","version":"v2"}'
+pyrit> run garak.encoding --target my_target --initializers target load_default_datasets --memory-labels '{"experiment":"test1","version":"v2"}'
 ```
 
 ### Override Defaults Per-Run
 
 ```bash
-# Override database and log level for this run only
-pyrit> run garak.encoding --initializers openai_objective_target load_default_datasets --database InMemory --log-level DEBUG
+# Override log level for this run only
+pyrit> run garak.encoding --target my_target --initializers target load_default_datasets --log-level DEBUG
 ```
 
 ### Run Command Options
@@ -85,7 +87,6 @@ pyrit> run garak.encoding --initializers openai_objective_target load_default_da
 --max-concurrency <N>           Maximum concurrent operations
 --max-retries <N>               Maximum retry attempts
 --memory-labels <JSON>          JSON string of labels
---database <type>               Override default database (InMemory, SQLite, AzureSQL)
 --log-level <level>             Override default log level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
 ```
 
@@ -114,9 +115,9 @@ pyrit> scenario-history
 
 Scenario Run History:
 ================================================================================
-1) foundry.red_team_agent --initializers openai_objective_target load_default_datasets --strategies base64
-2) garak.encoding --initializers openai_objective_target load_default_datasets --strategies rot13
-3) foundry.red_team_agent --initializers openai_objective_target load_default_datasets -s jailbreak
+1) foundry.red_team_agent --initializers target load_default_datasets --strategies base64
+2) garak.encoding --initializers target load_default_datasets --strategies rot13
+3) foundry.red_team_agent --initializers target load_default_datasets -s jailbreak
 ================================================================================
 
 Total runs: 3
@@ -130,7 +131,7 @@ The shell excels at interactive testing workflows:
 
 ```bash
 # Start shell with defaults
-pyrit_shell --database InMemory --initializers openai_objective_target load_default_datasets
+pyrit_shell --initializers target load_default_datasets
 
 # Quick exploration
 pyrit> list-scenarios
@@ -161,7 +162,7 @@ pyrit> print-scenario 2
 
 2. **Use short strategy aliases** with `-s`:
    ```bash
-   pyrit> run foundry.red_team_agent --initializers openai_objective_target load_default_datasets -s base64 rot13
+   pyrit> run foundry.red_team_agent --initializers target load_default_datasets -s base64 rot13
    ```
 
 3. **Review history regularly** to track what you've tested:

doc/code/front_end/2_pyrit_shell.md

@@ -20,14 +20,12 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Available scenarios: ['airt.content_harms', 'airt.cyber', 'airt.scam', 'foundry.red_team_agent', 'garak.encoding']...\n",
+      "Available scenarios: ['airt.content_harms', 'airt.cyber', 'airt.jailbreak', 'airt.leakage', 'airt.psychosocial']...\n",
       "\n",
-      "airt.content_harms:\n",
-      "  Class: ContentHarms\n",
+      "ContentHarms:\n",
       "  Description: Content Harms Scenario implementation for PyRIT. This scenario contains various ...\n",
       "\n",
-      "airt.cyber:\n",
-      "  Class: Cyber\n",
+      "Cyber:\n",
       "  Description: Cyber scenario implementation for PyRIT. This scenario tests how willing models ...\n"
      ]
     }
@@ -74,8 +72,6 @@
     }
    ],
    "source": [
-    "# Get a scenario class\n",
-    "\n",
     "scenario_class = registry.get_class(\"garak.encoding\")\n",
     "\n",
     "print(f\"Got class: {scenario_class}\")\n",
@@ -102,56 +98,15 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Found default environment files: ['./.pyrit/.env', './.pyrit/.env.local']\n",
-      "Loaded environment file: ./.pyrit/.env\n",
-      "Loaded environment file: ./.pyrit/.env.local\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\r\n",
-      "Loading datasets - this can take a few minutes:   0%|                                      | 0/45 [00:00<?, ?dataset/s]"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\r\n",
-      "Loading datasets - this can take a few minutes:   2%|▋                             | 1/45 [00:00<00:16,  2.70dataset/s]"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\r\n",
-      "Loading datasets - this can take a few minutes:  38%|██████████▉                  | 17/45 [00:00<00:00, 46.49dataset/s]"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\r\n",
-      "Loading datasets - this can take a few minutes:  69%|███████████████████▉         | 31/45 [00:00<00:00, 56.82dataset/s]"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\r\n",
-      "Loading datasets - this can take a few minutes: 100%|█████████████████████████████| 45/45 [00:00<00:00, 66.94dataset/s]"
+      "Found default environment files: ['./.pyrit/.env']\n",
+      "Loaded environment file: ./.pyrit/.env\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "\n"
+      "Loading datasets - this can take a few minutes: 100%|██████████| 58/58 [00:00<00:00, 68.45dataset/s]\n"
      ]
     },
     {
@@ -178,7 +133,7 @@
     "await scenario.initialize_async(objective_target=target)  # type: ignore\n",
     "\n",
     "# Option 2: Use create_instance() shortcut\n",
-    "# scenario = registry.create_instance(\"encoding\", objective_target=my_target, ...)\n",
+    "# scenario = registry.create_instance(\"garak.encoding\", objective_target=my_target, ...)\n",
     "\n",
     "print(\"Scenarios can be instantiated with your target and parameters\")"
    ]
@@ -205,10 +160,10 @@
      "text": [
       "'garak.encoding' registered: True\n",
       "'nonexistent' registered: False\n",
-      "Total scenarios: 5\n",
+      "Total scenarios: 8\n",
       "  - airt.content_harms\n",
       "  - airt.cyber\n",
-      "  - airt.scam\n"
+      "  - airt.jailbreak\n"
      ]
     }
    ],
@@ -245,13 +200,13 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Available initializers: ['airt', 'load_default_datasets', 'objective_list', 'openai_objective_target', 'simple']...\n",
+      "Available initializers: ['airt', 'load_default_datasets', 'scenario_objective_list', 'scorer', 'simple']...\n",
       "\n",
-      "airt:\n",
+      "AIRT Default Configuration:\n",
       "  Class: AIRTInitializer\n",
       "  Description: AI Red Team setup with Azure OpenAI converters, composite harm/objective scorers...\n",
       "\n",
-      "load_default_datasets:\n",
+      "Default Dataset Loader for Scenarios:\n",
       "  Class: LoadDefaultDatasets\n",
       "  Description: This configuration uses the DatasetLoader to load default datasets into memory.\n",
       "...\n"
@@ -286,7 +241,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.13.5"
+   "version": "3.11.9"
   }
  },
  "nbformat": 4,