add a couple of simple evaluators

Signed-off-by: Peter Jausovec <peter.jausovec@solo.io>

Author: peterj

.github/workflows/validate-evaluators.yaml

@@ -20,9 +20,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pip install pyyaml
-          # TODO: switch to `pip install agentevals-grader-sdk` once published to PyPI
-          pip install "agentevals-grader-sdk @ git+https://github.com/agentevals-dev/agentevals.git#subdirectory=packages/grader-sdk-py"
+          pip install pyyaml agentevals-evaluator-sdk
 
       - name: Discover and validate all evaluators
         run: |

.gitignore

@@ -0,0 +1 @@
+.venv/

README.md

@@ -107,13 +107,13 @@ author: your-github-username
 Run the validation script to catch issues before submitting:
 
 ```bash
-pip install agentevals-grader-sdk pyyaml
+pip install pyyaml agentevals-evaluator-sdk
 python scripts/validate_evaluator.py evaluators/my_evaluator
 ```
 
 This checks:
 - **Manifest schema** -- required fields, entrypoint exists, name matches directory
-- **Syntax and imports** -- compiles cleanly, uses `@grader` decorator
+- **Syntax and imports** -- compiles cleanly, uses `@evaluator` decorator
 - **Smoke run** -- runs the evaluator with synthetic input and validates the `EvalResult` output (correct types for `score`, `details`, `status`, etc.)
 
 You can also test with a full eval run:

evaluators/contains/contains.py

@@ -0,0 +1,57 @@
+"""Substring containment evaluator.
+
+Scores each invocation 1.0 if final_response contains the configured substring,
+otherwise 0.0.
+
+Config:
+  substring (str): Required for scoring; if omitted, the evaluator is a no-op (1.0).
+
+Usage in eval_config.yaml:
+    config:
+      substring: "expected phrase"
+"""
+
+from __future__ import annotations
+
+from agentevals_evaluator_sdk import EvalInput, EvalResult, evaluator
+
+
+@evaluator
+def contains(input: EvalInput) -> EvalResult:
+    needle = (input.config.get("substring") or "").strip()
+    if not needle:
+        return EvalResult(
+            score=1.0,
+            per_invocation_scores=[1.0] * len(input.invocations),
+            details={"note": "no substring configured; skipping check"},
+        )
+
+    case_insensitive = bool(input.config.get("case_insensitive", False))
+    haystack_fn = str.lower if case_insensitive else lambda s: s
+    needle_cmp = haystack_fn(needle)
+
+    scores: list[float] = []
+    issues: list[str] = []
+
+    for inv in input.invocations:
+        text = (inv.final_response or "")
+        if case_insensitive:
+            ok = needle_cmp in haystack_fn(text)
+        else:
+            ok = needle in text
+        if ok:
+            scores.append(1.0)
+        else:
+            scores.append(0.0)
+            issues.append(f"{inv.invocation_id}: response does not contain {needle!r}")
+
+    overall = sum(scores) / len(scores) if scores else 0.0
+    return EvalResult(
+        score=overall,
+        per_invocation_scores=scores,
+        details={"issues": issues} if issues else None,
+    )
+
+
+if __name__ == "__main__":
+    contains.run()

evaluators/contains/evaluator.yaml

@@ -0,0 +1,6 @@
+name: contains
+description: Scores whether each final response contains a configured substring (case-sensitive or case-insensitive)
+language: python
+entrypoint: contains.py
+tags: [string, contains,]
+author: agentevals-dev

evaluators/equals/equals.py

@@ -0,0 +1,58 @@
+"""Exact string match evaluator.
+
+Config:
+  expected (str): If omitted, no-op (1.0).
+  case_insensitive (bool, default True): Compare normalized strings.
+  strip (bool, default True): Strip whitespace before compare.
+
+Usage:
+    config:
+      expected: "4"
+"""
+
+from __future__ import annotations
+
+from agentevals_evaluator_sdk import EvalInput, EvalResult, evaluator
+
+
+@evaluator
+def equals(input: EvalInput) -> EvalResult:
+    expected = input.config.get("expected")
+    if expected is None:
+        return EvalResult(
+            score=1.0,
+            per_invocation_scores=[1.0] * len(input.invocations),
+            details={"note": "no expected string configured; skipping check"},
+        )
+
+    case_insensitive = bool(input.config.get("case_insensitive", True))
+    strip = bool(input.config.get("strip", True))
+
+    def norm(s: str) -> str:
+        t = s.strip() if strip else s
+        return t.lower() if case_insensitive else t
+
+    exp = norm(str(expected))
+    scores: list[float] = []
+    issues: list[str] = []
+
+    for inv in input.invocations:
+        got = norm(inv.final_response or "")
+        if got == exp:
+            scores.append(1.0)
+        else:
+            scores.append(0.0)
+            issues.append(
+                f"{inv.invocation_id}: expected {expected!r}, got {inv.final_response!r}"
+            )
+
+    overall = sum(scores) / len(scores) if scores else 0.0
+    return EvalResult(
+        score=overall,
+        per_invocation_scores=scores,
+        details={"issues": issues} if issues else None,
+    )
+
+
+if __name__ == "__main__":
+    equals.run()

evaluators/equals/evaluator.yaml

@@ -0,0 +1,6 @@
+name: equals
+description: Scores whether each final response exactly matches a configured expected string
+language: python
+entrypoint: equals.py
+tags: [string, equals]
+author: agentevals-dev

evaluators/is_json/evaluator.yaml

@@ -0,0 +1,6 @@
+name: is_json
+description: Scores whether each final response parses as JSON (optional markdown code fence extraction)
+language: python
+entrypoint: is_json.py
+tags: [json, structured]
+author: agentevals-dev

evaluators/is_json/is_json.py

@@ -0,0 +1,64 @@
+"""JSON parse check evaluator.
+
+Tries to parse final_response as JSON. Optionally extracts fenced ```json ... ``` blocks.
+
+Config:
+  require_json (bool, default False): If False, evaluator is a no-op (1.0).
+  extract_markdown_fence (bool, default True): Strip ```json fences if present.
+
+Usage:
+    config:
+      require_json: true
+"""
+
+from __future__ import annotations
+
+import json
+import re
+
+from agentevals_evaluator_sdk import EvalInput, EvalResult, evaluator
+
+_FENCE = re.compile(r"^```(?:json)?\s*\n?(.*?)\n?```\s*$", re.DOTALL | re.IGNORECASE)
+
+
+def _parse_json_payload(text: str, extract_fence: bool) -> object:
+    raw = (text or "").strip()
+    if extract_fence:
+        m = _FENCE.match(raw)
+        if m:
+            raw = m.group(1).strip()
+    return json.loads(raw)
+
+
+@evaluator
+def is_json(input: EvalInput) -> EvalResult:
+    if not input.config.get("require_json"):
+        return EvalResult(
+            score=1.0,
+            per_invocation_scores=[1.0] * len(input.invocations),
+            details={"note": "require_json not set; skipping check"},
+        )
+
+    extract_fence = bool(input.config.get("extract_markdown_fence", True))
+
+    scores: list[float] = []
+    issues: list[str] = []
+
+    for inv in input.invocations:
+        try:
+            _parse_json_payload(inv.final_response or "", extract_fence)
+            scores.append(1.0)
+        except (json.JSONDecodeError, TypeError, ValueError) as exc:
+            scores.append(0.0)
+            issues.append(f"{inv.invocation_id}: not valid JSON ({exc})")
+
+    overall = sum(scores) / len(scores) if scores else 0.0
+    return EvalResult(
+        score=overall,
+        per_invocation_scores=scores,
+        details={"issues": issues} if issues else None,
+    )
+
+
+if __name__ == "__main__":
+    is_json.run()

evaluators/levenshtein_ratio/evaluator.yaml

@@ -0,0 +1,6 @@
+name: levenshtein_ratio
+description: Scores similarity of each response to a reference string using normalized Levenshtein distance
+language: python
+entrypoint: levenshtein_ratio.py
+tags: [string, levenshtein]
+author: agentevals-dev