FIX: unblock main CI — Test GUI (PyPI), Crescendo parser, CoCoNot empty prompt (#1862)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: romanlutz

docker/Dockerfile

@@ -46,11 +46,20 @@ COPY --chown=vscode:vscode frontend/ /app/frontend/
 COPY --chown=vscode:vscode build_scripts/ /app/build_scripts/
 COPY --chown=vscode:vscode doc/ /app/doc/
 
-# Install PyRIT and create build info (combined to ensure dependencies are available)
+# Install PyRIT and create build info (combined to ensure dependencies are available).
+# For PYRIT_SOURCE=pypi we also delete the local pyrit/ + packaging files copied
+# above so they don't shadow the installed wheel: WORKDIR is /app, so otherwise
+# `python -m pyrit.*` would import the local source — which is how the
+# missing-alembic crash on Test GUI (PyPI) happens (local source uses alembic
+# but PyPI <=0.13.0 doesn't depend on it). The rm mirrors the COPY block above
+# (lines 43-47) one-to-one, except /app/doc which is intentionally retained
+# because the later RUN block copies it into /app/notebooks/ for Jupyter mode.
 # Note: We use 'uv pip' because the devcontainer creates venv with uv (no pip by default)
 RUN if [ "$PYRIT_SOURCE" = "pypi" ]; then \
         echo "Installing PyRIT from PyPI version: $PYRIT_VERSION"; \
         uv pip install --python /opt/venv/bin/python pyrit[speech,opencv,fairness_bias,fastapi,playwright]==$PYRIT_VERSION; \
+        echo "Removing local source so the installed PyPI package isn't shadowed"; \
+        rm -rf /app/pyrit /app/frontend /app/build_scripts /app/pyproject.toml /app/MANIFEST.in /app/README.md /app/LICENSE; \
     elif [ "$PYRIT_SOURCE" = "local" ]; then \
         echo "Installing PyRIT from local source"; \
         uv pip install --python /opt/venv/bin/python -e .[speech,opencv,fairness_bias,fastapi,playwright]; \

docker/start.sh

@@ -80,7 +80,23 @@ elif [ "$PYRIT_MODE" = "gui" ]; then
         fi
     } >"$RUNTIME_CONFIG"
 
-    exec python -m pyrit.backend.pyrit_backend \
+    # Pick the launcher module. PR #1753 moved the launcher from
+    # ``pyrit.cli.pyrit_backend`` to ``pyrit.backend.pyrit_backend``. The PyPI
+    # docker_build CI job pins to whatever's currently published (0.13.0 at
+    # time of writing), which still uses the old path, so fall back to it when
+    # the new module isn't present. Once a release containing the new layout
+    # ships, this fallback is dead code and can be removed.
+    if python -c "import pyrit.backend.pyrit_backend" >/dev/null 2>&1; then
+        BACKEND_MODULE="pyrit.backend.pyrit_backend"
+    elif python -c "import pyrit.cli.pyrit_backend" >/dev/null 2>&1; then
+        echo "Using legacy pyrit.cli.pyrit_backend launcher (PyRIT <= 0.13.0)"
+        BACKEND_MODULE="pyrit.cli.pyrit_backend"
+    else
+        echo "ERROR: cannot find pyrit backend launcher module" >&2
+        exit 1
+    fi
+
+    exec python -m "$BACKEND_MODULE" \
         --host 0.0.0.0 \
         --port 8000 \
         --config-file "$RUNTIME_CONFIG"

pyrit/datasets/seed_datasets/remote/coconot_dataset.py

@@ -151,6 +151,16 @@ async def fetch_dataset_async(self, *, cache: bool = True) -> SeedDataset:
                 category = row.get("category")
                 if wanted_categories is not None and category not in wanted_categories:
                     continue
+                # The upstream HF dataset contains a small number of rows with an
+                # empty ``prompt`` (observed in original.train under the wildchats
+                # subcategory). SeedObjective enforces value != "" downstream, so
+                # skip them here to keep the loader resilient to upstream drift.
+                if not (row.get("prompt") or "").strip():
+                    logger.warning(
+                        f"Skipping CoCoNot row with empty prompt "
+                        f"(id={row.get('id')!r}, category={category!r}, split={split!r})"
+                    )
+                    continue
                 seeds.append(self._row_to_seed(row=row, split=split, source_url=source_url))
 
         if not seeds:

pyrit/executor/attack/multi_turn/crescendo.py

@@ -5,6 +5,7 @@
 
 import json
 import logging
+import re
 from dataclasses import dataclass
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Optional, Union, cast
@@ -568,6 +569,12 @@ def _parse_adversarial_response(self, response_text: str) -> str:
         """
         Parse and validate the JSON response from the adversarial chat.
 
+        camelCase keys are normalized to snake_case before validation. The
+        Crescendo system prompts specify a snake_case JSON schema, but some
+        backends drift to camelCase (``generatedQuestion`` instead of
+        ``generated_question``); accepting both prevents the attack from
+        burning all its retries on a casing mismatch.
+
         Args:
             response_text (str): The response text to parse.
 
@@ -582,25 +589,41 @@ def _parse_adversarial_response(self, response_text: str) -> str:
         try:
             parsed_output = json.loads(response_text)
 
-            # Check for required keys
-            missing_keys = expected_keys - set(parsed_output.keys())
+            normalized_output = {self._camel_to_snake(key): value for key, value in parsed_output.items()}
+
+            missing_keys = expected_keys - set(normalized_output.keys())
             if missing_keys:
                 raise InvalidJsonException(
                     message=f"Missing required keys {missing_keys} in JSON response: {response_text}"
                 )
 
-            # Check for unexpected keys
-            extra_keys = set(parsed_output.keys()) - expected_keys
+            extra_keys = set(normalized_output.keys()) - expected_keys
             if extra_keys:
                 raise InvalidJsonException(
                     message=f"Unexpected keys {extra_keys} found in JSON response: {response_text}"
                 )
 
-            return str(parsed_output["generated_question"])
+            return str(normalized_output["generated_question"])
 
         except json.JSONDecodeError as e:
             raise InvalidJsonException(message=f"Invalid JSON encountered: {response_text}") from e
 
+    @staticmethod
+    def _camel_to_snake(name: str) -> str:
+        """
+        Convert a ``camelCase`` or ``PascalCase`` identifier to ``snake_case``.
+
+        Existing snake_case identifiers are returned unchanged.
+
+        Args:
+            name (str): The identifier to convert.
+
+        Returns:
+            str: The snake_case form of ``name``.
+        """
+        intermediate = re.sub(r"(.)([A-Z][a-z]+)", r"\1_\2", name)
+        return re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", intermediate).lower()
+
     async def _send_prompt_to_objective_target_async(
         self,
         *,

tests/unit/datasets/test_coconot_dataset.py

@@ -213,6 +213,53 @@ def test_invalid_split_raises(self) -> None:
         with pytest.raises(ValueError, match="Expected CoCoNotSplit"):
             _CoCoNotRefusalDataset(splits=[CoCoNotCategory.SAFETY])  # type: ignore[ty:invalid-argument-type]
 
+    async def test_rows_with_empty_prompts_are_skipped(self) -> None:
+        """Upstream rows with empty/whitespace ``prompt`` are dropped, not turned into empty seeds.
+
+        Regression test for the end_to_end ``test_fetch_dataset[_CoCoNotRefusalDataset]``
+        failure, where an empty-prompt row in ``original.train`` (wildchats subcategory)
+        produced a SeedObjective with ``value=""`` and tripped the loader-wide
+        ``seed.value`` invariant.
+        """
+        loader = _CoCoNotRefusalDataset(splits=[CoCoNotSplit.TRAIN])
+        rows_with_empty = [
+            {
+                "id": "ok",
+                "prompt": "real prompt",
+                "response": "",
+                "category": "Indeterminate requests",
+                "subcategory": "fine",
+            },
+            {
+                "id": "empty",
+                "prompt": "",
+                "response": "",
+                "category": "Indeterminate requests",
+                "subcategory": "wildchats",
+            },
+            {
+                "id": "whitespace",
+                "prompt": "   ",
+                "response": "",
+                "category": "Indeterminate requests",
+                "subcategory": "wildchats",
+            },
+            {
+                "id": "missing",
+                "response": "",
+                "category": "Indeterminate requests",
+                "subcategory": "wildchats",
+            },
+        ]
+        with patch.object(loader, "_fetch_from_huggingface", new=AsyncMock(return_value=rows_with_empty)):
+            dataset = await loader.fetch_dataset_async()
+
+        assert len(dataset.seeds) == 1
+        kept = dataset.seeds[0]
+        assert kept.value == "real prompt"
+        assert kept.metadata is not None
+        assert kept.metadata["id"] == "ok"
+
 
 class TestCoCoNotContrastDataset:
     """Tests for the CoCoNot contrast (over-refusal) sibling (`contrast.test`)."""

tests/unit/executor/attack/multi_turn/test_crescendo.py

@@ -931,6 +931,73 @@ async def test_parse_adversarial_response_with_various_inputs(
             result = attack._parse_adversarial_response(response_json)
             assert isinstance(result, str)
 
+    @pytest.mark.parametrize(
+        "raw,expected",
+        [
+            ("generated_question", "generated_question"),
+            ("generatedQuestion", "generated_question"),
+            ("GeneratedQuestion", "generated_question"),
+            ("rationaleBehindJailbreak", "rationale_behind_jailbreak"),
+            ("lastResponseSummary", "last_response_summary"),
+            ("", ""),
+        ],
+    )
+    def test_camel_to_snake_handles_common_cases(self, raw: str, expected: str) -> None:
+        """``_camel_to_snake`` normalizes camelCase / PascalCase and leaves snake_case alone."""
+        assert CrescendoAttack._camel_to_snake(raw) == expected
+
+    def test_parse_adversarial_response_accepts_camel_case_keys(
+        self,
+        mock_objective_target: MagicMock,
+        mock_adversarial_chat: MagicMock,
+    ) -> None:
+        """camelCase keys are normalized to snake_case so well-formed JSON with the wrong casing still parses.
+
+        Regression test for the Azure DevOps Integration Tests failure on
+        ``4_sequential_attack.ipynb``, where the adversarial model returned
+        ``generatedQuestion`` / ``rationaleBehindJailbreak`` /
+        ``lastResponseSummary`` for three retries straight and the strict
+        snake_case-only parser tore down the run.
+        """
+        attack = CrescendoTestHelper.create_attack(
+            objective_target=mock_objective_target,
+            adversarial_chat=mock_adversarial_chat,
+        )
+        camel_case_response = (
+            '{"generatedQuestion": "Attack question", '
+            '"lastResponseSummary": "Summary text", '
+            '"rationaleBehindJailbreak": "Why this works"}'
+        )
+
+        result = attack._parse_adversarial_response(camel_case_response)
+
+        assert result == "Attack question"
+
+    def test_parse_adversarial_response_mixed_casing_still_validates_extras(
+        self,
+        mock_objective_target: MagicMock,
+        mock_adversarial_chat: MagicMock,
+    ) -> None:
+        """Extra keys remain rejected even after camelCase normalization.
+
+        ``unexpectedKey`` normalizes to ``unexpected_key`` (still not in the
+        expected set), so the strict extra-key check continues to fire — we
+        only loosen casing, not the schema.
+        """
+        attack = CrescendoTestHelper.create_attack(
+            objective_target=mock_objective_target,
+            adversarial_chat=mock_adversarial_chat,
+        )
+        response_with_extra = (
+            '{"generatedQuestion": "Attack", '
+            '"lastResponseSummary": "Summary", '
+            '"rationaleBehindJailbreak": "Rationale", '
+            '"unexpectedKey": "value"}'
+        )
+
+        with pytest.raises(InvalidJsonException, match="Unexpected keys"):
+            attack._parse_adversarial_response(response_with_extra)
+
     async def test_custom_message_is_sent_to_target(
         self,
         mock_objective_target: MagicMock,