Validate expected_goal for goal_integrity assertions

Rejects goal_integrity assertions during scenario validation when expected_goal is missing, blank, or not a string.

Adds regression tests covering valid, missing, blank, non-string, and unrelated assertion cases.

Closes #37.

Author: mertsatilmaz

src/agent_harness/scenario.py

@@ -124,10 +124,19 @@ def validate_scenario_data(data: Any) -> Scenario:
                 f"assertions[{index}].type must be a non-empty string"
             )
 
+        assertion_type = assertion_type.strip()
+        if assertion_type == "goal_integrity":
+            expected_goal = assertion.get("expected_goal")
+            if not isinstance(expected_goal, str) or not expected_goal.strip():
+                raise ScenarioValidationError(
+                    f"assertions[{index}].expected_goal must be a non-empty string "
+                    "for goal_integrity assertions"
+                )
+
     return Scenario(
         id=scenario_id,
         title=title,
         category=category,
         severity=severity,
         raw=data,
-    )
+    )

tests/test_scenarios.py

@@ -4,7 +4,27 @@
 
 from pathlib import Path
 
-from agent_harness.scenario import load_scenario
+import pytest
+
+from agent_harness.scenario import (
+    ScenarioValidationError,
+    load_scenario,
+    validate_scenario_data,
+)
+
+
+def _minimal_scenario(assertions):
+    return {
+        "id": "goal-hijack-basic",
+        "title": "Goal hijack basic",
+        "category": "goal_hijack",
+        "severity": "medium",
+        "target": {},
+        "input": {},
+        "expected": {},
+        "assertions": assertions,
+    }
+
 
 def test_all_bundled_scenarios_validate():
     scenario_paths = sorted(Path("scenarios").rglob("*.yaml"))
@@ -15,5 +35,49 @@ def test_all_bundled_scenarios_validate():
         load_scenario(scenario_path)
 
 
+def test_goal_integrity_assertion_validates_with_expected_goal():
+    scenario = validate_scenario_data(
+        _minimal_scenario(
+            [{"type": "goal_integrity", "expected_goal": "summarize_document"}]
+        )
+    )
+
+    assert scenario.id == "goal-hijack-basic"
+
+
+def test_goal_integrity_assertion_requires_expected_goal():
+    with pytest.raises(
+        ScenarioValidationError,
+        match="expected_goal must be a non-empty string",
+    ):
+        validate_scenario_data(_minimal_scenario([{"type": "goal_integrity"}]))
+
+
+def test_goal_integrity_assertion_rejects_blank_expected_goal():
+    with pytest.raises(
+        ScenarioValidationError,
+        match="expected_goal must be a non-empty string",
+    ):
+        validate_scenario_data(
+            _minimal_scenario([{"type": "goal_integrity", "expected_goal": "   "}])
+        )
+
+
+def test_goal_integrity_assertion_rejects_non_string_expected_goal():
+    with pytest.raises(
+        ScenarioValidationError,
+        match="expected_goal must be a non-empty string",
+    ):
+        validate_scenario_data(
+            _minimal_scenario([{"type": "goal_integrity", "expected_goal": 123}])
+        )
+
+
+def test_other_assertion_types_do_not_require_expected_goal():
+    scenario = validate_scenario_data(
+        _minimal_scenario([{"type": "no_denied_tool_call"}])
+    )
+
+    assert scenario.id == "goal-hijack-basic"