TEST: stop GCG unit tests from hitting HuggingFace (#1886)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: romanlutz

…iliary_attacks/gcg/test_attack_wiring.py …ks/test_gcg_attack_wiring_integration.pytests/unit/auxiliary_attacks/gcg/test_attack_wiring.py renamed to tests/integration/auxiliary_attacks/test_gcg_attack_wiring_integration.py tests/unit/auxiliary_attacks/gcg/test_attack_wiring.py renamed to tests/integration/auxiliary_attacks/test_gcg_attack_wiring_integration.py

@@ -1,12 +1,28 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT license.
 
-"""Tests that exercise the full attack class wiring without mocking manager classes.
-
-These tests catch kwarg mismatches between IndividualPromptAttack/ProgressiveMultiPromptAttack
-and MultiPromptAttack.__init__(), and template compatibility issues in _update_ids().
+"""Integration tests for GCG attack-class wiring.
+
+These tests construct real ``IndividualPromptAttack`` / ``ProgressiveMultiPromptAttack``
+instances and exercise their wiring all the way down through
+``GCGAttackPrompt._update_ids``, which calls ``tokenizer.apply_chat_template`` and
+walks character positions via ``char_to_token``. They live here (not under
+``tests/unit/``) because they require a real HuggingFace tokenizer (gpt2) to
+back the chat-template pipeline — mocking that out would defeat the test's
+purpose, which is to catch kwarg-mismatch and template-compatibility bugs that
+mocked tests miss.
+
+Network dependency: the gpt2 tokenizer is fetched from HuggingFace on first
+use (and then cached). This is why these tests run only in the integration tier
+(``make integration-test``), not in the PR-time unit-test matrix.
+
+Requires: torch, transformers (GCG optional deps).
+Skipped via importorskip when deps are not installed.
 """
 
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
 from unittest.mock import MagicMock, patch
 
 import pytest
@@ -22,29 +38,48 @@
     reason="GCG optional dependencies not installed",
 )
 
+generator_mod = pytest.importorskip(
+    "pyrit.auxiliary_attacks.gcg.generator",
+    reason="GCG optional dependencies (torch, transformers, etc.) not installed",
+)
+
+from pyrit.auxiliary_attacks.gcg.config import (  # noqa: E402
+    GCGAlgorithmConfig,
+    GCGModelConfig,
+    GCGOutputConfig,
+    GCGStrategyConfig,
+)
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
 IndividualPromptAttack = attack_manager_mod.IndividualPromptAttack
 ProgressiveMultiPromptAttack = attack_manager_mod.ProgressiveMultiPromptAttack
 MultiPromptAttack = attack_manager_mod.MultiPromptAttack
 GCGAttackPrompt = gcg_attack_mod.GCGAttackPrompt
 GCGPromptManager = gcg_attack_mod.GCGPromptManager
 GCGMultiPromptAttack = gcg_attack_mod.GCGMultiPromptAttack
+GCGGenerator = generator_mod.GCGGenerator
+GCGContext = generator_mod.GCGContext
 
 MANAGERS = {
     "AP": GCGAttackPrompt,
     "PM": GCGPromptManager,
     "MPA": GCGMultiPromptAttack,
 }
 
+_LLAMA_2 = "meta-llama/Llama-2-7b-chat-hf"
+
 
-def _make_mock_worker() -> MagicMock:
-    """Create a mock worker whose tokenizer can stand in for a real chat tokenizer.
+def _make_mock_worker_with_real_tokenizer() -> MagicMock:
+    """Worker mock backed by a real gpt2 tokenizer.
 
     The wiring tests construct real ``GCGAttackPrompt`` instances which call
     ``tokenizer.apply_chat_template`` and then walk character positions in the
     rendered prompt. We need a real string + a tokenizer that can answer
-    ``char_to_token`` queries on it, so we back the mock with a real
-    distilgpt2 tokenizer (the smallest available transformers tokenizer that
-    ships with all the methods we touch).
+    ``char_to_token`` queries on it, so we back the mock with the smallest
+    workable real HF tokenizer (gpt2) plus an explicit llama-2-style chat
+    template (gpt2 ships without one).
     """
     from transformers import AutoTokenizer
 
@@ -67,9 +102,9 @@ def _make_mock_worker() -> MagicMock:
 
 
 class TestAttackClassWiring:
-    """Tests that verify attack classes can be constructed with real manager classes.
+    """Verify attack classes can be constructed and run with real manager classes.
 
-    These catch kwarg mismatches that mocked tests miss.
+    Catches kwarg mismatches that mocked tests miss.
     """
 
     def test_individual_attack_creates_mpa_without_error(self) -> None:
@@ -78,7 +113,7 @@ def test_individual_attack_creates_mpa_without_error(self) -> None:
         This catches the mpa_kwargs bug where dead kwargs (deterministic, lr, etc.)
         were passed to MultiPromptAttack.__init__() which didn't accept them.
         """
-        worker = _make_mock_worker()
+        worker = _make_mock_worker_with_real_tokenizer()
 
         # Create IndividualPromptAttack with the real GCG manager classes
         attack = IndividualPromptAttack(
@@ -114,7 +149,7 @@ def test_individual_attack_creates_mpa_without_error(self) -> None:
 
     def test_progressive_attack_creates_mpa_without_error(self) -> None:
         """ProgressiveMultiPromptAttack.run() should create MultiPromptAttack without TypeError."""
-        worker = _make_mock_worker()
+        worker = _make_mock_worker_with_real_tokenizer()
 
         attack = ProgressiveMultiPromptAttack(
             goals=["test goal"],
@@ -145,3 +180,54 @@ def test_progressive_attack_creates_mpa_without_error(self) -> None:
                 verbose=False,
                 filter_cand=True,
             )
+
+
+class TestCreateAttackWiring:
+    """Construct real attack classes via :meth:`GCGGenerator._create_attack` to catch kwarg mismatches."""
+
+    def test_transfer_false_returns_individual(self, tmp_path: Path) -> None:
+        gen = GCGGenerator(
+            models=[GCGModelConfig(name=_LLAMA_2)],
+            algorithm=GCGAlgorithmConfig(n_steps=5, batch_size=64, control_init="! ! !"),
+            output=GCGOutputConfig(result_prefix=str(tmp_path / "gcg")),
+        )
+        worker = _make_mock_worker_with_real_tokenizer()
+        context = GCGContext(goals=["g"], targets=["t"])
+        params = gen._to_attack_params(context=context)
+
+        attack = gen._create_attack(
+            params=params,
+            managers=MANAGERS,
+            train_goals=["g"],
+            train_targets=["t"],
+            test_goals=[],
+            test_targets=[],
+            workers=[worker],
+            test_workers=[],
+            logfile_path=str(tmp_path / "log.json"),
+        )
+        assert isinstance(attack, IndividualPromptAttack)
+
+    def test_transfer_true_returns_progressive(self, tmp_path: Path) -> None:
+        gen = GCGGenerator(
+            models=[GCGModelConfig(name=_LLAMA_2)],
+            algorithm=GCGAlgorithmConfig(n_steps=5, batch_size=64, control_init="! ! !"),
+            strategy=GCGStrategyConfig(transfer=True, progressive_goals=True, progressive_models=True),
+            output=GCGOutputConfig(result_prefix=str(tmp_path / "gcg")),
+        )
+        worker = _make_mock_worker_with_real_tokenizer()
+        context = GCGContext(goals=["g"], targets=["t"])
+        params = gen._to_attack_params(context=context)
+
+        attack = gen._create_attack(
+            params=params,
+            managers=MANAGERS,
+            train_goals=["g"],
+            train_targets=["t"],
+            test_goals=[],
+            test_targets=[],
+            workers=[worker],
+            test_workers=[],
+            logfile_path=str(tmp_path / "log.json"),
+        )
+        assert isinstance(attack, ProgressiveMultiPromptAttack)

tests/unit/auxiliary_attacks/gcg/test_gcg_core.py

@@ -443,29 +443,39 @@ def char_to_token(pos: int) -> int | None:
 
     def test_end_tok_returns_len_toks_when_target_is_at_prompt_end(self) -> None:
         """If the target sits at the very end of the rendered prompt,
-        char_to_token(end_pos) returns None — end_tok must clamp to len(toks)."""
-        from transformers import AutoTokenizer
-
-        tokenizer = AutoTokenizer.from_pretrained("gpt2")
-        tokenizer.pad_token = tokenizer.eos_token
-        tokenizer.chat_template = (
-            "{%- for m in messages -%}"
-            "{%- if m['role'] == 'user' -%}"
-            "[INST] {{ m['content'] }} [/INST]"
-            "{%- elif m['role'] == 'assistant' -%}"
-            " {{ m['content'] }}"
-            "{%- endif -%}"
-            "{%- endfor -%}"
-        )
+        char_to_token(end_pos) returns None — end_tok must clamp to len(toks)
+        (line 201 in attack_manager.py)."""
+        # Fully-mocked tokenizer so we can deterministically force char_to_token to
+        # return None at the position just past the target. Mirrors the pattern used
+        # by the two adjacent tests above.
+        prompt_text = "[INST] hello !! [/INST] world"
+        toks = list(range(10))
+        target_end_pos = len(prompt_text)  # one past the final char of "world"
+
+        def char_to_token(pos: int) -> int | None:
+            # Position at/after end-of-prompt has no token → triggers the
+            # `return len(toks)` fallback in end_tok.
+            if pos >= target_end_pos:
+                return None
+            # Everything else maps to a valid token index that preserves ordering.
+            return min(pos // 3, len(toks) - 1)
+
+        encoding = MagicMock()
+        encoding.input_ids = toks
+        encoding.char_to_token.side_effect = char_to_token
+
+        tokenizer = MagicMock()
+        tokenizer.apply_chat_template.return_value = prompt_text
+        tokenizer.return_value = encoding
 
         prompt = AttackPrompt(
             goal="hello",
-            target="world",  # this sits at end of rendered prompt with no trailing tokens
+            target="world",  # sits at end of prompt_text; target end has no token
             tokenizer=tokenizer,
-            control_init="! ! !",
+            control_init="!!",
         )
-        # _target_slice.stop should be len(toks), not None or NoneType arithmetic
-        assert isinstance(prompt._target_slice.stop, int)
+        # end_tok(target_end_pos) saw None from char_to_token → clamped to len(toks).
+        assert prompt._target_slice.stop == len(toks)
         assert prompt._target_slice.stop > prompt._target_slice.start
 
 

tests/unit/auxiliary_attacks/gcg/test_generator.py

@@ -238,57 +238,6 @@ def test_augmentation_modifies_at_least_some_targets(self) -> None:
         assert num_changed > 0
 
 
-class TestCreateAttackWiring:
-    """Construct real attack classes via _create_attack to catch kwarg mismatches."""
-
-    def test_transfer_false_returns_individual(self, tmp_path: Path) -> None:
-        from pyrit.auxiliary_attacks.gcg.attack.base.attack_manager import IndividualPromptAttack
-
-        gen = _make_generator(output_dir=tmp_path, n_steps=5, batch_size=64, control_init="! ! !")
-        worker = _make_mock_worker_with_real_tokenizer()
-        context = GCGContext(goals=["g"], targets=["t"])
-        params = gen._to_attack_params(context=context)
-
-        attack = gen._create_attack(
-            params=params,
-            managers=_real_managers(),
-            train_goals=["g"],
-            train_targets=["t"],
-            test_goals=[],
-            test_targets=[],
-            workers=[worker],
-            test_workers=[],
-            logfile_path=str(tmp_path / "log.json"),
-        )
-        assert isinstance(attack, IndividualPromptAttack)
-
-    def test_transfer_true_returns_progressive(self, tmp_path: Path) -> None:
-        from pyrit.auxiliary_attacks.gcg.attack.base.attack_manager import ProgressiveMultiPromptAttack
-
-        gen = GCGGenerator(
-            models=[GCGModelConfig(name=_LLAMA_2)],
-            algorithm=GCGAlgorithmConfig(n_steps=5, batch_size=64, control_init="! ! !"),
-            strategy=GCGStrategyConfig(transfer=True, progressive_goals=True, progressive_models=True),
-            output=GCGOutputConfig(result_prefix=str(tmp_path / "gcg")),
-        )
-        worker = _make_mock_worker_with_real_tokenizer()
-        context = GCGContext(goals=["g"], targets=["t"])
-        params = gen._to_attack_params(context=context)
-
-        attack = gen._create_attack(
-            params=params,
-            managers=_real_managers(),
-            train_goals=["g"],
-            train_targets=["t"],
-            test_goals=[],
-            test_targets=[],
-            workers=[worker],
-            test_workers=[],
-            logfile_path=str(tmp_path / "log.json"),
-        )
-        assert isinstance(attack, ProgressiveMultiPromptAttack)
-
-
 class TestReadResult:
     def test_reads_final_suffix_and_loss(self, tmp_path: Path) -> None:
         log_path = tmp_path / "result.json"
@@ -323,34 +272,3 @@ def test_empty_controls_returns_nan_loss(self, tmp_path: Path) -> None:
         result = GCGGenerator._read_result(logfile_path=str(log_path), memory_labels={})
         assert result.final_suffix == ""
         assert math.isnan(result.final_loss)
-
-
-def _make_mock_worker_with_real_tokenizer() -> MagicMock:
-    """Worker mock backed by a real GPT-2 tokenizer (the smallest workable for chat templates)."""
-    from transformers import AutoTokenizer
-
-    tokenizer = AutoTokenizer.from_pretrained("gpt2")
-    tokenizer.pad_token = tokenizer.eos_token
-    tokenizer.chat_template = (
-        "{%- for m in messages -%}"
-        "{%- if m['role'] == 'user' -%}"
-        "[INST] {{ m['content'] }} [/INST] "
-        "{%- elif m['role'] == 'assistant' -%}"
-        "{{ m['content'] }}"
-        "{%- endif -%}"
-        "{%- endfor -%}"
-    )
-    worker = MagicMock()
-    worker.model.name_or_path = "test-model"
-    worker.tokenizer = tokenizer
-    return worker
-
-
-def _real_managers() -> dict:
-    from pyrit.auxiliary_attacks.gcg.attack.gcg.gcg_attack import (
-        GCGAttackPrompt,
-        GCGMultiPromptAttack,
-        GCGPromptManager,
-    )
-
-    return {"AP": GCGAttackPrompt, "PM": GCGPromptManager, "MPA": GCGMultiPromptAttack}