feat: (wip) update protect support

Author: mikebranc

evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna2/config.py

@@ -5,20 +5,24 @@
 from agent_control_evaluators import EvaluatorConfig
 from pydantic import Field, model_validator
 
-# Supported Luna-2 metrics
+# Supported Luna-2 metrics — names must match the Galileo Protect API exactly.
+# Use the Protect API / golden-demo code as the source of truth:
+#   output PII  → "pii"         input PII  → "input_pii"
+#   output tox  → "toxicity"    input tox  → "input_toxicity"
 Luna2Metric = Literal[
     "input_toxicity",
-    "output_toxicity",
+    "toxicity",           # output toxicity (API name, not "output_toxicity")
     "input_sexism",
     "output_sexism",
     "prompt_injection",
-    "pii_detection",
+    "pii",                # output PII (API name, replaces "pii_detection")
+    "input_pii",          # input PII
     "hallucination",
     "tone",
 ]
 
-# Supported operators
-Luna2Operator = Literal["gt", "lt", "gte", "lte", "eq", "contains", "any"]
+# Supported operators — "not_empty" matches categorical PII/injection results
+Luna2Operator = Literal["gt", "lt", "gte", "lte", "eq", "contains", "any", "not_empty"]
 
 
 class Luna2EvaluatorConfig(EvaluatorConfig):
@@ -113,7 +117,8 @@ def validate_stage_config(self) -> "Luna2EvaluatorConfig":
                 raise ValueError("'metric' is required for local stage")
             if not self.operator:
                 raise ValueError("'operator' is required for local stage")
-            if self.target_value is None:
+            # not_empty / not_null operators don't need a comparison value
+            if self.target_value is None and self.operator not in ("not_empty", "not_null"):
                 raise ValueError("'target_value' is required for local stage")
         elif self.stage_type == "central":
             if not self.stage_name:

evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna2/evaluator.py

@@ -172,6 +172,20 @@ def _get_numeric_target_value(self) -> float | int | str | None:
     async def _evaluate_local_stage(self, data: Any) -> EvaluatorResult:
         """Evaluate using a local stage (runtime rulesets).
 
+        We use PASSTHROUGH action so Protect computes the metric and returns
+        metric_results without making a block decision itself — agent-control
+        owns that decision via the control's action.decision field.
+
+        Numeric operators (gt, lt, gte, lte, eq): Protect evaluates the rule
+        server-side and returns status="triggered" when the condition is met,
+        so _parse_response picks it up directly.
+
+        Categorical operators (not_empty, any): the Protect local-stage rule
+        engine does not support these operators and always returns
+        status="not_triggered", even when the metric value is non-empty.
+        _parse_response falls back to _evaluate_metric_results which evaluates
+        the condition client-side from the raw metric_results dict.
+
         Args:
             data: The data to evaluate.
 
@@ -187,7 +201,8 @@ async def _evaluate_local_stage(self, data: Any) -> EvaluatorResult:
             target_value=self._get_numeric_target_value() or 0,
         )
 
-        # Create proper Ruleset with PassthroughAction
+        # PASSTHROUGH: Protect scores the content and returns metric_results,
+        # but does not block — agent-control's deny action handles that.
         ruleset = Ruleset(
             rules=[rule],
             action=PassthroughAction(type="PASSTHROUGH"),
@@ -204,6 +219,7 @@ async def _evaluate_local_stage(self, data: Any) -> EvaluatorResult:
                 payload=payload,
                 prioritized_rulesets=[ruleset],
                 project_name=self.config.galileo_project,
+                stage_name=self.config.stage_name,
                 timeout=self.get_timeout_seconds(),
                 metadata=self.config.metadata or {},
             )
@@ -279,10 +295,20 @@ def _prepare_payload(self, data: Any) -> Payload:
         is_output_metric = "output" in metric
 
         if is_output_metric:
-            return Payload(input="", output=data_str)
+            payload = Payload(input="", output=data_str)
         else:
             # Default to input for central stages or input metrics
-            return Payload(input=data_str, output="")
+            payload = Payload(input=data_str, output="")
+
+        logger.debug(
+            "[Luna2] _prepare_payload: metric=%r payload_field_config=%r "
+            "→ input=%d chars, output=%d chars",
+            self.config.metric,
+            self.config.payload_field,
+            len(payload.input),
+            len(payload.output),
+        )
+        return payload
 
     def _parse_response(self, response: ProtectResponse | None) -> EvaluatorResult:
         """Parse Galileo Protect response into EvaluatorResult.
@@ -304,16 +330,34 @@ def _parse_response(self, response: ProtectResponse | None) -> EvaluatorResult:
         status = response.status.lower() if response.status else "unknown"
         triggered = status == "triggered"
 
+        # Numeric operators (gt/lt/etc.) are evaluated server-side by Protect and
+        # return status="triggered" correctly even with PASSTHROUGH.
+        # Categorical operators (not_empty, any) are NOT supported by Protect's
+        # local-stage rule engine — it always returns status="not_triggered" for
+        # them regardless of the metric value.  Fall back to client-side evaluation
+        # from metric_results for those cases.
+        if not triggered and response.metric_results:
+            triggered = self._evaluate_metric_results(response.metric_results)
+
+        logger.info(
+            "[Luna2] response: status=%r triggered=%s metric_results=%s",
+            status,
+            triggered,
+            response.metric_results,
+        )
+
         # Extract trace metadata
         trace_id = response.trace_metadata.id if response.trace_metadata else None
         execution_time = response.trace_metadata.execution_time if response.trace_metadata else None
         received_at = response.trace_metadata.received_at if response.trace_metadata else None
         response_at = response.trace_metadata.response_at if response.trace_metadata else None
 
+        message = self._build_message(triggered, status, response.metric_results)
+
         return EvaluatorResult(
             matched=triggered,
             confidence=1.0 if triggered else 0.0,
-            message=response.text or f"Luna-2 check: {status}",
+            message=message,
             metadata={
                 "status": status,
                 "metric": self.config.metric or "unknown",
@@ -324,6 +368,69 @@ def _parse_response(self, response: ProtectResponse | None) -> EvaluatorResult:
             },
         )
 
+    def _build_message(self, triggered: bool, status: str, metric_results: dict) -> str:
+        """Build a human-readable message from the evaluation result."""
+        metric = self.config.metric or "unknown"
+
+        if not triggered:
+            return f"Luna-2 {metric} check passed"
+
+        result = (metric_results or {}).get(metric, {})
+        value = result.get("value")
+
+        if isinstance(value, list) and value:
+            categories = ", ".join(str(v).replace("_", " ") for v in value)
+            return f"PII detected: {categories}"
+        if isinstance(value, (int, float)):
+            return f"{metric} score {value:.2f} exceeded threshold"
+
+        return f"Luna-2 {metric} check triggered"
+
+    def _evaluate_metric_results(self, metric_results: dict) -> bool:
+        """Evaluate the configured operator/target against raw metric_results.
+
+        Used when the Protect API returns PASSTHROUGH (no server-side trigger)
+        but we still need to decide whether the rule condition is met.
+
+        Args:
+            metric_results: The metric_results dict from the Protect API response.
+
+        Returns:
+            True if the rule condition is satisfied.
+        """
+        metric = self.config.metric
+        if not metric or metric not in metric_results:
+            return False
+
+        result = metric_results[metric]
+        if result.get("status") != "SUCCESS":
+            return False
+
+        value = result.get("value")
+        operator = self.config.operator
+        target = self.config.target_value
+
+        if operator in ("not_empty", "not_null"):
+            return bool(value)
+        if operator in ("empty", "is_null"):
+            return not bool(value)
+        if operator == "any" and isinstance(value, list):
+            return target in value if target is not None else bool(value)
+        if operator == "contains":
+            return target in value if (value and target is not None) else False
+        if isinstance(value, (int, float)) and target is not None:
+            try:
+                t = float(target)
+                if operator == "gt":  return value > t
+                if operator == "gte": return value >= t
+                if operator == "lt":  return value < t
+                if operator == "lte": return value <= t
+                if operator == "eq":  return value == t
+            except (TypeError, ValueError):
+                pass
+
+        return False
+
     def _handle_error(self, error: Exception) -> EvaluatorResult:
         """Handle errors from Luna-2 evaluation.