Skip to content

Commit 4d217a3

Browse files
PouyanpiPouyanpi
authored
refactor(library): make moderation outcomes flow-independent (#2183)
Signed-off-by: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com>
1 parent 84645b7 commit 4d217a3

8 files changed

Lines changed: 394 additions & 242 deletions

File tree

‎nemoguardrails/library/activefence/actions.py‎

Lines changed: 51 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -25,26 +25,54 @@
2525

2626
log = logging.getLogger(__name__)
2727

28-
ACTIVEFENCE_DETAILED_THRESHOLDS = {
29-
"abusive_or_harmful.harassment_or_bullying": 0.8,
30-
"abusive_or_harmful.profanity": 0.7,
31-
"abusive_or_harmful.hate_speech": 0.6,
32-
"abusive_or_harmful.child_grooming": 0.4,
33-
"abusive_or_harmful.general_violence": 0.7,
34-
"self_harm.general": 0.8,
35-
"adult_content.general": 0.3,
36-
"privacy_violation.pii": 0.8,
28+
ACTIVEFENCE_DETAILED_RULES = {
29+
"abusive_or_harmful.harassment_or_bullying": (
30+
0.8,
31+
"ActiveFence moderation triggered. The harassment or bullying risk score exceeded the threshold.",
32+
),
33+
"abusive_or_harmful.profanity": (
34+
0.7,
35+
"ActiveFence moderation triggered. The profanity risk score exceeded the threshold.",
36+
),
37+
"abusive_or_harmful.hate_speech": (
38+
0.6,
39+
"ActiveFence moderation triggered. The hate speech risk score exceeded the threshold.",
40+
),
41+
"abusive_or_harmful.child_grooming": (
42+
0.4,
43+
"ActiveFence moderation triggered. The child grooming risk score exceeded the threshold.",
44+
),
45+
"abusive_or_harmful.general_violence": (
46+
0.7,
47+
"ActiveFence moderation triggered. The general violence risk score exceeded the threshold.",
48+
),
49+
"self_harm.general": (
50+
0.8,
51+
"ActiveFence moderation triggered. The self harm risk score exceeded the threshold.",
52+
),
53+
"adult_content.general": (
54+
0.3,
55+
"ActiveFence moderation triggered. The adult content risk score exceeded the threshold.",
56+
),
57+
"privacy_violation.pii": (
58+
0.8,
59+
"ActiveFence moderation triggered. The privacy violation risk score exceeded the threshold.",
60+
),
3761
}
3862

3963

4064
def _activefence_simple_blocked(max_risk_score: float) -> bool:
4165
return max_risk_score > 0.7
4266

4367

44-
def _activefence_detailed_blocked(violations: dict[str, float]) -> bool:
45-
return any(
46-
violations.get(violation_type, 0) > threshold
47-
for violation_type, threshold in ACTIVEFENCE_DETAILED_THRESHOLDS.items()
68+
def _activefence_triggered_violation(violations: dict[str, float]) -> Optional[str]:
69+
return next(
70+
(
71+
violation_type
72+
for violation_type, (threshold, _) in ACTIVEFENCE_DETAILED_RULES.items()
73+
if violations.get(violation_type, 0) > threshold
74+
),
75+
None,
4876
)
4977

5078

@@ -53,19 +81,21 @@ def _activefence_outcome(
5381
violations: dict[str, float],
5482
threshold_mode: Literal["simple", "detailed"] = "simple",
5583
) -> RailOutcome:
84+
triggered_violation = _activefence_triggered_violation(violations) if threshold_mode == "detailed" else None
5685
metadata = {
5786
"max_risk_score": max_risk_score,
5887
"violations": violations,
5988
"threshold_mode": threshold_mode,
89+
"triggered_violation": triggered_violation,
6090
}
61-
blocked = (
62-
_activefence_detailed_blocked(violations)
63-
if threshold_mode == "detailed"
64-
else _activefence_simple_blocked(max_risk_score)
65-
)
66-
67-
if blocked:
68-
return RailOutcome.block(metadata=metadata)
91+
if triggered_violation is not None:
92+
reason = ACTIVEFENCE_DETAILED_RULES[triggered_violation][1]
93+
return RailOutcome.block(reason=reason, metadata=metadata)
94+
if threshold_mode != "detailed" and _activefence_simple_blocked(max_risk_score):
95+
return RailOutcome.block(
96+
reason="ActiveFence moderation triggered. The maximum risk score exceeded the threshold.",
97+
metadata=metadata,
98+
)
6999
return RailOutcome.allow(metadata=metadata)
70100

71101

‎nemoguardrails/library/activefence/flows.co‎

Lines changed: 49 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ flow activefence moderation on input
1919

2020
if $result.is_blocked
2121
if $system.config.enable_rails_exceptions
22-
send ActiveFenceModerationRailException(message="ActiveFence moderation triggered. The maximum risk score exceeded the threshold.")
22+
send ActiveFenceModerationRailException(message=$result.reason)
2323
else
2424
bot refuse to respond
2525
abort
@@ -30,7 +30,7 @@ flow activefence moderation on output
3030

3131
if $result.is_blocked
3232
if $system.config.enable_rails_exceptions
33-
send ActiveFenceModerationRailException(message="ActiveFence moderation triggered. The maximum risk score exceeded the threshold.")
33+
send ActiveFenceModerationRailException(message=$result.reason)
3434
else
3535
bot refuse to respond
3636
abort
@@ -39,60 +39,54 @@ flow activefence moderation on input detailed
3939
"""Guardrail based on individual risk scores."""
4040
$result = await CallActivefenceApiAction(text=$user_message, threshold_mode="detailed")
4141

42-
if $result.metadata["violations"].get("abusive_or_harmful.harassment_or_bullying", 0) > 0.8
43-
if $system.config.enable_rails_exceptions
44-
send ActiveFenceHarassmentRailException(message="ActiveFence moderation triggered. The harassment or bullying risk score exceeded the threshold.")
45-
else
46-
bot inform cannot engage in abusive_or_harmful behavior
47-
abort
48-
49-
if $result.metadata["violations"].get("abusive_or_harmful.profanity", 0) > 0.7
50-
if $system.config.enable_rails_exceptions
51-
send ActiveFenceProfanityRailException(message="ActiveFence moderation triggered. The profanity risk score exceeded the threshold.")
52-
else
53-
bot inform cannot engage in abusive_or_harmful behavior
54-
abort
55-
56-
if $result.metadata["violations"].get("abusive_or_harmful.hate_speech", 0) > 0.6
57-
if $system.config.enable_rails_exceptions
58-
send ActiveFenceHateSpeechRailException(message="ActiveFence moderation triggered. The hate speech risk score exceeded the threshold.")
59-
else
60-
bot inform cannot engage in abusive_or_harmful behavior
61-
abort
62-
63-
if $result.metadata["violations"].get("abusive_or_harmful.child_grooming", 0) > 0.4
64-
if $system.config.enable_rails_exceptions
65-
send ActiveFenceChildGroomingRailException(message="ActiveFence moderation triggered. The child grooming risk score exceeded the threshold.")
66-
else
67-
bot inform cannot engage in abusive_or_harmful behavior
68-
abort
69-
70-
if $result.metadata["violations"].get("abusive_or_harmful.general_violence", 0) > 0.7
71-
if $system.config.enable_rails_exceptions
72-
send ActiveFenceGeneralViolenceRailException(message="ActiveFence moderation triggered. The general violence risk score exceeded the threshold.")
73-
else
74-
bot inform cannot engage in abusive_or_harmful behavior
75-
abort
76-
77-
if $result.metadata["violations"].get("self_harm.general", 0) > 0.8
78-
if $system.config.enable_rails_exceptions
79-
send ActiveFenceSelfHarmRailException(message="ActiveFence moderation triggered. The self harm risk score exceeded the threshold.")
80-
else
81-
bot inform cannot engage in self harm behavior
82-
abort
83-
84-
if $result.metadata["violations"].get("adult_content.general", 0) > 0.3
85-
if $system.config.enable_rails_exceptions
86-
send ActiveFenceAdultContentRailException(message="ActiveFence moderation triggered. The adult content risk score exceeded the threshold.")
87-
else
88-
bot inform cannot engage with inappropriate content
89-
abort
90-
91-
if $result.metadata["violations"].get("privacy_violation.pii", 0) > 0.8
92-
if $system.config.enable_rails_exceptions
93-
send ActiveFencePrivacyViolationRailException(message="ActiveFence moderation triggered. The privacy violation risk score exceeded the threshold.")
42+
if $result.is_blocked
43+
$triggered_violation = $result.metadata.get("triggered_violation")
44+
45+
if $triggered_violation == "abusive_or_harmful.harassment_or_bullying"
46+
if $system.config.enable_rails_exceptions
47+
send ActiveFenceHarassmentRailException(message=$result.reason)
48+
else
49+
bot inform cannot engage in abusive_or_harmful behavior
50+
else if $triggered_violation == "abusive_or_harmful.profanity"
51+
if $system.config.enable_rails_exceptions
52+
send ActiveFenceProfanityRailException(message=$result.reason)
53+
else
54+
bot inform cannot engage in abusive_or_harmful behavior
55+
else if $triggered_violation == "abusive_or_harmful.hate_speech"
56+
if $system.config.enable_rails_exceptions
57+
send ActiveFenceHateSpeechRailException(message=$result.reason)
58+
else
59+
bot inform cannot engage in abusive_or_harmful behavior
60+
else if $triggered_violation == "abusive_or_harmful.child_grooming"
61+
if $system.config.enable_rails_exceptions
62+
send ActiveFenceChildGroomingRailException(message=$result.reason)
63+
else
64+
bot inform cannot engage in abusive_or_harmful behavior
65+
else if $triggered_violation == "abusive_or_harmful.general_violence"
66+
if $system.config.enable_rails_exceptions
67+
send ActiveFenceGeneralViolenceRailException(message=$result.reason)
68+
else
69+
bot inform cannot engage in abusive_or_harmful behavior
70+
else if $triggered_violation == "self_harm.general"
71+
if $system.config.enable_rails_exceptions
72+
send ActiveFenceSelfHarmRailException(message=$result.reason)
73+
else
74+
bot inform cannot engage in self harm behavior
75+
else if $triggered_violation == "adult_content.general"
76+
if $system.config.enable_rails_exceptions
77+
send ActiveFenceAdultContentRailException(message=$result.reason)
78+
else
79+
bot inform cannot engage with inappropriate content
80+
else if $triggered_violation == "privacy_violation.pii"
81+
if $system.config.enable_rails_exceptions
82+
send ActiveFencePrivacyViolationRailException(message=$result.reason)
83+
else
84+
bot inform cannot engage with sensitive content
9485
else
95-
bot inform cannot engage with sensitive content
86+
if $system.config.enable_rails_exceptions
87+
send ActiveFenceModerationRailException(message=$result.reason)
88+
else
89+
bot refuse to respond
9690
abort
9791

9892
flow bot inform cannot engage in abusive_or_harmful behavior

‎nemoguardrails/library/activefence/flows.v1.co‎

Lines changed: 49 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ define subflow activefence moderation on input
1919

2020
if $result.is_blocked
2121
if $config.enable_rails_exceptions
22-
create event ActiveFenceModerationRailException(message="ActiveFence moderation triggered. The maximum risk score exceeded the threshold.")
22+
create event ActiveFenceModerationRailException(message=$result.reason)
2323
else
2424
bot refuse to respond
2525
stop
@@ -30,7 +30,7 @@ define subflow activefence moderation on input
3030

3131
if $result.is_blocked
3232
if $config.enable_rails_exceptions
33-
create event ActiveFenceModerationRailException(message="ActiveFence moderation triggered. The maximum risk score exceeded the threshold.")
33+
create event ActiveFenceModerationRailException(message=$result.reason)
3434
else
3535
bot refuse to respond
3636
stop
@@ -40,60 +40,54 @@ define subflow activefence moderation on input detailed
4040
"""Guardrail based on individual risk scores."""
4141
$result = execute call_activefence_api(text=$user_message, threshold_mode="detailed")
4242

43-
if $result.metadata["violations"].get("abusive_or_harmful.harassment_or_bullying", 0) > 0.8
44-
if $config.enable_rails_exceptions
45-
create event ActiveFenceHarassmentRailException(message="ActiveFence moderation triggered. The harassment or bullying risk score exceeded the threshold.")
46-
else
47-
bot inform cannot engage in abusive or harmful behavior
48-
stop
49-
50-
if $result.metadata["violations"].get("abusive_or_harmful.profanity", 0) > 0.7
51-
if $config.enable_rails_exceptions
52-
create event ActiveFenceProfanityRailException(message="ActiveFence moderation triggered. The profanity risk score exceeded the threshold.")
53-
else
54-
bot inform cannot engage in abusive or harmful behavior
55-
stop
56-
57-
if $result.metadata["violations"].get("abusive_or_harmful.hate_speech", 0) > 0.6
58-
if $config.enable_rails_exceptions
59-
create event ActiveFenceHateSpeechRailException(message="ActiveFence moderation triggered. The hate speech risk score exceeded the threshold.")
60-
else
61-
bot inform cannot engage in abusive or harmful behavior
62-
stop
63-
64-
if $result.metadata["violations"].get("abusive_or_harmful.child_grooming", 0) > 0.4
65-
if $config.enable_rails_exceptions
66-
create event ActiveFenceChildGroomingRailException(message="ActiveFence moderation triggered. The child grooming risk score exceeded the threshold.")
67-
else
68-
bot inform cannot engage in abusive or harmful behavior
69-
stop
70-
71-
if $result.metadata["violations"].get("abusive_or_harmful.general_violence", 0) > 0.7
72-
if $config.enable_rails_exceptions
73-
create event ActiveFenceGeneralViolenceRailException(message="ActiveFence moderation triggered. The general violence risk score exceeded the threshold.")
74-
else
75-
bot inform cannot engage in abusive or harmful behavior
76-
stop
77-
78-
if $result.metadata["violations"].get("self_harm.general", 0) > 0.8
79-
if $config.enable_rails_exceptions
80-
create event ActiveFenceSelfHarmRailException(message="ActiveFence moderation triggered. The self harm risk score exceeded the threshold.")
81-
else
82-
bot inform cannot engage in self harm behavior
83-
stop
84-
85-
if $result.metadata["violations"].get("adult_content.general", 0) > 0.3
86-
if $config.enable_rails_exceptions
87-
create event ActiveFenceAdultContentRailException(message="ActiveFence moderation triggered. The adult content risk score exceeded the threshold.")
88-
else
89-
bot inform cannot engage with inappropriate content
90-
stop
91-
92-
if $result.metadata["violations"].get("privacy_violation.pii", 0) > 0.8
93-
if $config.enable_rails_exceptions
94-
create event ActiveFencePrivacyViolationRailException(message="ActiveFence moderation triggered. The privacy violation risk score exceeded the threshold.")
43+
if $result.is_blocked
44+
$triggered_violation = $result.metadata.get("triggered_violation")
45+
46+
if $triggered_violation == "abusive_or_harmful.harassment_or_bullying"
47+
if $config.enable_rails_exceptions
48+
create event ActiveFenceHarassmentRailException(message=$result.reason)
49+
else
50+
bot inform cannot engage in abusive or harmful behavior
51+
else if $triggered_violation == "abusive_or_harmful.profanity"
52+
if $config.enable_rails_exceptions
53+
create event ActiveFenceProfanityRailException(message=$result.reason)
54+
else
55+
bot inform cannot engage in abusive or harmful behavior
56+
else if $triggered_violation == "abusive_or_harmful.hate_speech"
57+
if $config.enable_rails_exceptions
58+
create event ActiveFenceHateSpeechRailException(message=$result.reason)
59+
else
60+
bot inform cannot engage in abusive or harmful behavior
61+
else if $triggered_violation == "abusive_or_harmful.child_grooming"
62+
if $config.enable_rails_exceptions
63+
create event ActiveFenceChildGroomingRailException(message=$result.reason)
64+
else
65+
bot inform cannot engage in abusive or harmful behavior
66+
else if $triggered_violation == "abusive_or_harmful.general_violence"
67+
if $config.enable_rails_exceptions
68+
create event ActiveFenceGeneralViolenceRailException(message=$result.reason)
69+
else
70+
bot inform cannot engage in abusive or harmful behavior
71+
else if $triggered_violation == "self_harm.general"
72+
if $config.enable_rails_exceptions
73+
create event ActiveFenceSelfHarmRailException(message=$result.reason)
74+
else
75+
bot inform cannot engage in self harm behavior
76+
else if $triggered_violation == "adult_content.general"
77+
if $config.enable_rails_exceptions
78+
create event ActiveFenceAdultContentRailException(message=$result.reason)
79+
else
80+
bot inform cannot engage with inappropriate content
81+
else if $triggered_violation == "privacy_violation.pii"
82+
if $config.enable_rails_exceptions
83+
create event ActiveFencePrivacyViolationRailException(message=$result.reason)
84+
else
85+
bot inform cannot engage with sensitive content
9586
else
96-
bot inform cannot engage with sensitive content
87+
if $config.enable_rails_exceptions
88+
create event ActiveFenceModerationRailException(message=$result.reason)
89+
else
90+
bot refuse to respond
9791
stop
9892

9993
define bot inform cannot engage in abusive or harmful behavior

‎nemoguardrails/library/gcp_moderate_text/actions.py‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -45,27 +45,35 @@ def _gcp_text_simple_blocked(max_risk_score: float) -> bool:
4545
return max_risk_score > 0.8
4646

4747

48-
def _gcp_text_detailed_blocked(violations: dict[str, float]) -> bool:
49-
return any(violations.get(name, 0) > threshold for name, threshold in GCP_TEXT_DETAILED_THRESHOLDS.items())
48+
def _gcp_text_triggered_violation(violations: dict[str, float]) -> Optional[str]:
49+
return next(
50+
(name for name, threshold in GCP_TEXT_DETAILED_THRESHOLDS.items() if violations.get(name, 0) > threshold),
51+
None,
52+
)
5053

5154

5255
def _gcp_text_moderation_outcome(
5356
max_risk_score: float,
5457
violations: dict[str, float],
5558
threshold_mode: Literal["simple", "detailed"] = "simple",
5659
) -> RailOutcome:
60+
triggered_violation = _gcp_text_triggered_violation(violations) if threshold_mode == "detailed" else None
5761
metadata = {
5862
"max_risk_score": max_risk_score,
5963
"violations": violations,
6064
"threshold_mode": threshold_mode,
65+
"triggered_violation": triggered_violation,
6166
}
62-
blocked = (
63-
_gcp_text_detailed_blocked(violations)
64-
if threshold_mode == "detailed"
65-
else _gcp_text_simple_blocked(max_risk_score)
66-
)
67-
if blocked:
68-
return RailOutcome.block(metadata=metadata)
67+
if triggered_violation is not None:
68+
return RailOutcome.block(
69+
reason=f"GCP text moderation triggered. {triggered_violation} exceeded the threshold.",
70+
metadata=metadata,
71+
)
72+
if threshold_mode != "detailed" and _gcp_text_simple_blocked(max_risk_score):
73+
return RailOutcome.block(
74+
reason="GCP text moderation triggered. The maximum risk score exceeded the threshold.",
75+
metadata=metadata,
76+
)
6977
return RailOutcome.allow(metadata=metadata)
7078

7179

0 commit comments

Comments
 (0)