Skip to content

Commit 2727e79

Browse files
committed
test: update mock responses to use correct Violated Categories key
Existing tests provided mock NemoGuard JSON responses with the wrong key Safety Categories. Now that the parser correctly reads Violated Categories, update all mock response fixtures to match what the real model emits. The intentional regression tests in test_content_safety_output_parsers.py that verify Safety Categories no longer extracts data are left unchanged.
1 parent 343f52a commit 2727e79

5 files changed

Lines changed: 11 additions & 11 deletions

File tree

tests/guardrails/test_content_safety_iorails_actions.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,15 +43,15 @@
4343
UNSAFE_JSON = json.dumps(
4444
{
4545
"User Safety": "unsafe",
46-
"Safety Categories": "S1: Violence, S3: Criminal Planning/Confessions",
46+
"Violated Categories": "S1: Violence, S3: Criminal Planning/Confessions",
4747
}
4848
)
4949
SAFE_OUTPUT_JSON = json.dumps({"User Safety": "safe", "Response Safety": "safe"})
5050
UNSAFE_OUTPUT_JSON = json.dumps(
5151
{
5252
"User Safety": "safe",
5353
"Response Safety": "unsafe",
54-
"Safety Categories": "S17: Malware",
54+
"Violated Categories": "S17: Malware",
5555
}
5656
)
5757

tests/guardrails/test_data.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@
5353
<END CONVERSATION>
5454
5555
Provide your safety assessment for the above conversation in the below output JSON format:
56-
{"User Safety": safe or unsafe rating for the user message, "Response Safety": safe or unsafe rating for the agent response. Omit if no agent response present. "Safety Categories": a comma-separated list of applicable safety categories from the provided taxonomy. Omit if all safe.}
56+
{"User Safety": safe or unsafe rating for the user message, "Response Safety": safe or unsafe rating for the agent response. Omit if no agent response present. "Violated Categories": a comma-separated list of applicable safety categories from the provided taxonomy. Omit if all safe.}
5757
5858
Do not include anything other than the output JSON in your response.
5959
Output JSON:
@@ -97,7 +97,7 @@
9797
<END CONVERSATION>
9898
9999
Provide your safety assessment for the above conversation in the below output JSON format:
100-
{"User Safety": safe or unsafe rating for the user message, "Response Safety": safe or unsafe rating for the agent response. Omit if no agent response present. "Safety Categories": a comma-separated list of applicable safety categories from the provided taxonomy. Omit if all safe.}
100+
{"User Safety": safe or unsafe rating for the user message, "Response Safety": safe or unsafe rating for the agent response. Omit if no agent response present. "Violated Categories": a comma-separated list of applicable safety categories from the provided taxonomy. Omit if all safe.}
101101
102102
Do not include anything other than the output JSON in your response.
103103
Output JSON:

tests/guardrails/test_iorails_telemetry.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -372,7 +372,7 @@ async def capture_then_pass(messages):
372372

373373
SAFE_INPUT_JSON = json.dumps({"User Safety": "safe"})
374374
SAFE_OUTPUT_JSON = json.dumps({"User Safety": "safe", "Response Safety": "safe"})
375-
UNSAFE_INPUT_JSON = json.dumps({"User Safety": "unsafe", "Safety Categories": "S1: Violence"})
375+
UNSAFE_INPUT_JSON = json.dumps({"User Safety": "unsafe", "Violated Categories": "S1: Violence"})
376376

377377

378378
def _stub_deep_pipeline(iorails, main_llm_response="Hello", input_safe=True):

tests/guardrails/test_rails_manager.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,13 +41,13 @@
4141
)
4242

4343
SAFE_INPUT_JSON = json.dumps({"User Safety": "safe"})
44-
UNSAFE_INPUT_JSON = json.dumps({"User Safety": "unsafe", "Safety Categories": "S1: Violence"})
44+
UNSAFE_INPUT_JSON = json.dumps({"User Safety": "unsafe", "Violated Categories": "S1: Violence"})
4545
SAFE_OUTPUT_JSON = json.dumps({"User Safety": "safe", "Response Safety": "safe"})
4646
UNSAFE_OUTPUT_JSON = json.dumps(
4747
{
4848
"User Safety": "safe",
4949
"Response Safety": "unsafe",
50-
"Safety Categories": "S17: Malware",
50+
"Violated Categories": "S17: Malware",
5151
}
5252
)
5353
MESSAGES = [{"role": "user", "content": "hello"}]

tests/test_content_safety_integration.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,7 @@ async def test_content_safety_input_with_nemoguard_parser_unsafe_with_categories
161161
self,
162162
):
163163
"""Test input action with real nemoguard_parse_prompt_safety parser - unsafe with categories."""
164-
json_response = '{"User Safety": "unsafe", "Safety Categories": "S1, S8, S10"}'
164+
json_response = '{"User Safety": "unsafe", "Violated Categories": "S1, S8, S10"}'
165165
parsed_result = nemoguard_parse_prompt_safety(json_response)
166166
llms, mock_task_manager = _create_mock_setup([json_response], parsed_result)
167167
context = _create_input_context("Potentially harmful content")
@@ -181,7 +181,7 @@ async def test_content_safety_input_with_nemoguard_parser_unsafe_with_categories
181181
[
182182
('{"Response Safety": "safe"}', True, []),
183183
(
184-
'{"Response Safety": "unsafe", "Safety Categories": "Violence, Hate Speech"}',
184+
'{"Response Safety": "unsafe", "Violated Categories": "Violence, Hate Speech"}',
185185
False,
186186
["Violence", "Hate Speech"],
187187
),
@@ -251,13 +251,13 @@ def test_iterable_unpacking_with_is_content_safe_outputs(self, response, expecte
251251
[
252252
('{"User Safety": "safe"}', True, []),
253253
(
254-
'{"User Safety": "unsafe", "Safety Categories": "S1, S8"}',
254+
'{"User Safety": "unsafe", "Violated Categories": "S1, S8"}',
255255
False,
256256
["S1", "S8"],
257257
),
258258
('{"Response Safety": "safe"}', True, []),
259259
(
260-
'{"Response Safety": "unsafe", "Safety Categories": "Violence, Hate"}',
260+
'{"Response Safety": "unsafe", "Violated Categories": "Violence, Hate"}',
261261
False,
262262
["Violence", "Hate"],
263263
),

0 commit comments

Comments
 (0)