pr feedback

Signed-off-by: Peter Jausovec <peter.jausovec@solo.io>

Author: peterj

evaluators/contains/contains.py

@@ -4,7 +4,7 @@
 otherwise 0.0.
 
 Config:
-  substring (str): Required for scoring; if omitted, the evaluator is a no-op (1.0).
+  substring (str): Required. If omitted, returns NOT_EVALUATED.
 
 Usage in eval_config.yaml:
     config:
@@ -13,37 +13,39 @@
 
 from __future__ import annotations
 
-from agentevals_evaluator_sdk import EvalInput, EvalResult, evaluator
+from agentevals_evaluator_sdk import EvalInput, EvalResult, EvalStatus, evaluator
 
 
 @evaluator
 def contains(input: EvalInput) -> EvalResult:
-    needle = (input.config.get("substring") or "").strip()
-    if not needle:
+    substring = (input.config.get("substring") or "").strip()
+    n = len(input.invocations)
+    if not substring:
         return EvalResult(
-            score=1.0,
-            per_invocation_scores=[1.0] * len(input.invocations),
-            details={"note": "no substring configured; skipping check"},
+            score=0.0,
+            status=EvalStatus.NOT_EVALUATED,
+            per_invocation_scores=[None] * n,
+            details={"reason": "missing config: substring"},
         )
 
     case_insensitive = bool(input.config.get("case_insensitive", False))
-    haystack_fn = str.lower if case_insensitive else lambda s: s
-    needle_cmp = haystack_fn(needle)
+    normalize = str.lower if case_insensitive else lambda s: s
+    substring_cmp = normalize(substring)
 
     scores: list[float] = []
     issues: list[str] = []
 
     for inv in input.invocations:
-        text = (inv.final_response or "")
+        response_text = inv.final_response or ""
         if case_insensitive:
-            ok = needle_cmp in haystack_fn(text)
+            ok = substring_cmp in normalize(response_text)
         else:
-            ok = needle in text
+            ok = substring in response_text
         if ok:
             scores.append(1.0)
         else:
             scores.append(0.0)
-            issues.append(f"{inv.invocation_id}: response does not contain {needle!r}")
+            issues.append(f"{inv.invocation_id}: response does not contain {substring!r}")
 
     overall = sum(scores) / len(scores) if scores else 0.0
     return EvalResult(

evaluators/contains/evaluator.yaml

@@ -2,5 +2,5 @@ name: contains
 description: Scores whether each final response contains a configured substring (case-sensitive or case-insensitive)
 language: python
 entrypoint: contains.py
-tags: [string, contains,]
+tags: [string, contains]
 author: agentevals-dev

evaluators/equals/equals.py

@@ -1,7 +1,7 @@
 """Exact string match evaluator.
 
 Config:
-  expected (str): If omitted, no-op (1.0).
+  expected (str): Required. If omitted, returns NOT_EVALUATED.
   case_insensitive (bool, default True): Compare normalized strings.
   strip (bool, default True): Strip whitespace before compare.
 
@@ -12,20 +12,22 @@
 
 from __future__ import annotations
 
-from agentevals_evaluator_sdk import EvalInput, EvalResult, evaluator
+from agentevals_evaluator_sdk import EvalInput, EvalResult, EvalStatus, evaluator
 
 
 @evaluator
 def equals(input: EvalInput) -> EvalResult:
     expected = input.config.get("expected")
     if expected is None:
+        n = len(input.invocations)
         return EvalResult(
-            score=1.0,
-            per_invocation_scores=[1.0] * len(input.invocations),
-            details={"note": "no expected string configured; skipping check"},
+            score=0.0,
+            status=EvalStatus.NOT_EVALUATED,
+            per_invocation_scores=[None] * n,
+            details={"reason": "missing config: expected"},
         )
 
-    case_insensitive = bool(input.config.get("case_insensitive", True))
+    case_insensitive = bool(input.config.get("case_insensitive", False))
     strip = bool(input.config.get("strip", True))
 
     def norm(s: str) -> str:

evaluators/is_json/is_json.py

@@ -3,12 +3,11 @@
 Tries to parse final_response as JSON. Optionally extracts fenced ```json ... ``` blocks.
 
 Config:
-  require_json (bool, default False): If False, evaluator is a no-op (1.0).
   extract_markdown_fence (bool, default True): Strip ```json fences if present.
 
 Usage:
     config:
-      require_json: true
+      extract_markdown_fence: true
 """
 
 from __future__ import annotations
@@ -32,13 +31,6 @@ def _parse_json_payload(text: str, extract_fence: bool) -> object:
 
 @evaluator
 def is_json(input: EvalInput) -> EvalResult:
-    if not input.config.get("require_json"):
-        return EvalResult(
-            score=1.0,
-            per_invocation_scores=[1.0] * len(input.invocations),
-            details={"note": "require_json not set; skipping check"},
-        )
-
     extract_fence = bool(input.config.get("extract_markdown_fence", True))
 
     scores: list[float] = []

evaluators/levenshtein_ratio/levenshtein_ratio.py

@@ -3,7 +3,7 @@
 Score for an invocation is 1.0 - (edit_distance / max(len(a), len(b), 1)), clamped to [0, 1].
 
 Config:
-  expected (str): If omitted, no-op (1.0).
+  expected (str): Required. If omitted, returns NOT_EVALUATED.
   case_insensitive (bool, default False): Compare lowercased strings.
 
 Usage:
@@ -13,7 +13,7 @@
 
 from __future__ import annotations
 
-from agentevals_evaluator_sdk import EvalInput, EvalResult, evaluator
+from agentevals_evaluator_sdk import EvalInput, EvalResult, EvalStatus, evaluator
 
 
 def _levenshtein(a: str, b: str) -> int:
@@ -38,10 +38,12 @@ def _levenshtein(a: str, b: str) -> int:
 def levenshtein_ratio(input: EvalInput) -> EvalResult:
     expected = input.config.get("expected")
     if expected is None:
+        n = len(input.invocations)
         return EvalResult(
-            score=1.0,
-            per_invocation_scores=[1.0] * len(input.invocations),
-            details={"note": "no expected string configured; skipping check"},
+            score=0.0,
+            status=EvalStatus.NOT_EVALUATED,
+            per_invocation_scores=[None] * n,
+            details={"reason": "missing config: expected"},
         )
 
     case_insensitive = bool(input.config.get("case_insensitive", False))

evaluators/regex_match/regex_match.py

@@ -1,7 +1,7 @@
 """Regex on final response evaluator.
 
 Config:
-  pattern (str): If omitted, no-op (1.0).
+  pattern (str): Required. If omitted or invalid, returns NOT_EVALUATED.
   flags (str, optional): "IGNORECASE" | "MULTILINE" | "DOTALL" — combined with |.
 
 Usage:
@@ -14,7 +14,7 @@
 
 import re
 
-from agentevals_evaluator_sdk import EvalInput, EvalResult, evaluator
+from agentevals_evaluator_sdk import EvalInput, EvalResult, EvalStatus, evaluator
 
 _FLAG_MAP = {
     "IGNORECASE": re.IGNORECASE,
@@ -26,11 +26,13 @@
 @evaluator
 def regex_match(input: EvalInput) -> EvalResult:
     pattern = input.config.get("pattern")
+    n = len(input.invocations)
     if not pattern:
         return EvalResult(
-            score=1.0,
-            per_invocation_scores=[1.0] * len(input.invocations),
-            details={"note": "no pattern configured; skipping check"},
+            score=0.0,
+            status=EvalStatus.NOT_EVALUATED,
+            per_invocation_scores=[None] * n,
+            details={"reason": "missing config: pattern"},
         )
 
     flag_names = input.config.get("flags")
@@ -51,8 +53,9 @@ def regex_match(input: EvalInput) -> EvalResult:
     except re.error as exc:
         return EvalResult(
             score=0.0,
-            per_invocation_scores=[0.0] * len(input.invocations),
-            details={"error": f"invalid regex: {exc}"},
+            status=EvalStatus.NOT_EVALUATED,
+            per_invocation_scores=[None] * n,
+            details={"reason": "invalid regex pattern", "error": str(exc)},
         )
 
     scores: list[float] = []

evaluators/tool_sequence_match/tool_sequence_match.py

@@ -3,7 +3,7 @@
 Compares the ordered list of tool names in each invocation to config.
 
 Config:
-  expected_tool_names (list[str]): If omitted or empty, no-op (1.0).
+  expected_tool_names (list[str]): Required non-empty. Otherwise returns NOT_EVALUATED.
   require_order (bool, default True): If False, compares multisets (same counts per name).
 
 Usage:
@@ -16,17 +16,26 @@
 
 from collections import Counter
 
-from agentevals_evaluator_sdk import EvalInput, EvalResult, evaluator
+from agentevals_evaluator_sdk import EvalInput, EvalResult, EvalStatus, evaluator
 
 
 @evaluator
 def tool_sequence_match(input: EvalInput) -> EvalResult:
     expected = input.config.get("expected_tool_names")
+    n = len(input.invocations)
+    if expected is None or not isinstance(expected, list):
+        return EvalResult(
+            score=0.0,
+            status=EvalStatus.NOT_EVALUATED,
+            per_invocation_scores=[None] * n,
+            details={"reason": "missing or invalid config: expected_tool_names (need a list of names)"},
+        )
     if not expected:
         return EvalResult(
-            score=1.0,
-            per_invocation_scores=[1.0] * len(input.invocations),
-            details={"note": "no expected_tool_names configured; skipping check"},
+            score=0.0,
+            status=EvalStatus.NOT_EVALUATED,
+            per_invocation_scores=[None] * n,
+            details={"reason": "missing or empty config: expected_tool_names"},
         )
 
     want = [str(x) for x in expected]

scripts/validate_evaluator.py

@@ -158,7 +158,7 @@ def validate_smoke_run(evaluator_dir: Path, manifest: dict) -> bool:
     if not stdout:
         stderr_preview = result.stderr.strip()[:500]
         _fail(
-            f"Evaluator produced no output on stdout"
+            "Evaluator produced no output on stdout"
             + (f"\n  stderr: {stderr_preview}" if stderr_preview else "")
         )
         return False
@@ -208,6 +208,13 @@ def validate_smoke_run(evaluator_dir: Path, manifest: dict) -> bool:
                 f"got {type(per_inv).__name__}"
             )
             return False
+        for i, x in enumerate(per_inv):
+            if x is not None and not isinstance(x, (int, float)):
+                _fail(
+                    f"'per_invocation_scores[{i}]' must be a number or null, "
+                    f"got {type(x).__name__}"
+                )
+                return False
 
     # Full Pydantic validation via the SDK if available
     try: