feat: add eval-schemas, eval-fixtures, eval-notebook checks

README.md

@@ -28,6 +28,7 @@ Evaluators and the supporting datasets are built in collaboration with leading l
 | :------------------------ | :--------------------------------------------------------------- |
 | [`evals`](./evals/)       | Evaluators code and prompts                                      |
 | [`datasets`](./datasets/) | Expert annotated datasets used to create and validate evaluators |
+| [`scripts`](./scripts/)   | Repo check harness — run `python scripts/check.py --fix` before committing (see [scripts/README](./scripts/README.md)) |
 | [`LICENSE`](./LICENSE.md) | Open source license details                                      |
 
 Check out the [Evaluators docs](https://docs.learningcommons.org/evaluators) for complete setup instructions and usage examples.

scripts/README.md

@@ -37,25 +37,20 @@ fails the PR (it never auto-fixes) — so fixing locally saves a round-trip.
 |-------|--------------|
 | `strip-notebooks` | Clears outputs, execution counts, and cell metadata from `.ipynb` files |
 | `eval-config` | Validates each evaluator `config.json` against the shared schema (`evals/_schemas/config.schema.json`, referenced by the config's own `$schema`), plus cross-file rules: referenced files exist, prompt `sha256` matches, placeholders ⇄ template `{vars}` are in sync, system prompts carry no placeholders, and no obsolete `format_instructions` survive |
+| `eval-schemas` | Meta-validates each `input_schema.json` / `output_schema.json` is a well-formed JSON Schema document |
+| `eval-fixtures` | Validates `fixtures.json` against the shared `evals/_schemas/fixtures.schema.json`, and binds each case's `input`/`expected` to that evaluator's own input/output schema |
+| `eval-notebook` | Where an evaluator ships an example notebook, confirms it loads the config + prompt files from disk rather than hardcoding them |
 
 ## Coming next
 
 This is the seed of a broader self-service harness. Planned additions:
 
-- **`eval-schemas`** — meta-validate each `input_schema.json` / `output_schema.json`
-  is a well-formed JSON Schema document.
-- **`eval-fixtures`** — validate `fixtures.json` against the shared
-  `evals/_schemas/fixtures.schema.json`, and bind each case's `input`/`expected`
-  to that evaluator's own input/output schema.
-- **`eval-notebook`** — where an evaluator ships an example notebook, confirm it
-  loads the config + prompt files from disk rather than hardcoding them.
 - **delegation to the SDK suites** (`sdks/typescript` lint/type/test,
   `sdks/python` Makefile) so one local command covers everything. CI keeps the
   per-SDK workflows separate for parallelism and path-filtered signal.
 
-> **TODO** (once there are more checks): surface this from the root `README.md`
-> / `CONTRIBUTING.md` so newcomers discover it — e.g. "Before committing, run
-> `python scripts/check.py --fix` (see `scripts/README.md`)."
+> **TODO**: the root `README.md` links here for discoverability; give this a
+> proper home in a `CONTRIBUTING.md` once one exists.
 
 ## Adding a check
 

scripts/checks/__init__.py

@@ -1,9 +1,15 @@
 """Check registry. Add new checks here to wire them into the harness + CI."""
 
 from .eval_config import EvalConfig
+from .eval_fixtures import EvalFixtures
+from .eval_notebook import EvalNotebook
+from .eval_schemas import EvalSchemas
 from .strip_notebooks import StripNotebooks
 
 ALL_CHECKS = [
     StripNotebooks(),
     EvalConfig(),
+    EvalSchemas(),
+    EvalFixtures(),
+    EvalNotebook(),
 ]

scripts/checks/base.py

@@ -60,9 +60,9 @@ def evaluator_configs() -> list[str]:
     return tracked_files("evals/**/config.json")
 
 
-def shared_schema_path(spec_version: str, name: str) -> str:
-    """Path to a version-pinned shared schema, e.g. evals/_schemas/1.0/<name>."""
-    return os.path.join(SCHEMA_ROOT, spec_version, name)
+def shared_schema_path(name: str) -> str:
+    """Path to a shared schema, e.g. evals/_schemas/fixtures.schema.json."""
+    return os.path.join(SCHEMA_ROOT, name)
 
 
 def load_json(path: str):

scripts/checks/eval_fixtures.py

@@ -0,0 +1,134 @@
+"""Validate each evaluator's fixtures.json.
+
+Two layers, mirroring eval-config:
+  - shared structure: every case is {id, input, expected} per the shared
+    evals/_schemas/fixtures.schema.json.
+  - per-evaluator binding: each case's `input` must satisfy that evaluator's own
+    input_schema.json, and each field in `expected` must satisfy the matching
+    property in its output_schema.json. (`expected` is a subset of the full
+    output — just the label(s) — so we validate present fields, not requireds.)
+"""
+
+from __future__ import annotations
+
+import json
+import os
+
+from jsonschema import Draft202012Validator
+from jsonschema.exceptions import SchemaError
+
+from .base import Check, Result, Violation, evaluator_configs, load_json, shared_schema_path
+
+_FIXTURES_SCHEMA = "fixtures.schema.json"
+
+
+class EvalFixtures(Check):
+    name = "eval-fixtures"
+    description = "Validate fixtures.json structure + bind each case's input/expected to the evaluator's schemas"
+
+    def run(self, fix: bool) -> Result:
+        result = Result(self.name)
+        shared = Draft202012Validator(load_json(shared_schema_path(_FIXTURES_SCHEMA)))
+        for cfg_path in evaluator_configs():
+            self._check_fixtures(cfg_path, shared, result)
+        return result
+
+    def _check_fixtures(self, cfg_path: str, shared: Draft202012Validator, result: Result) -> None:
+        base = os.path.dirname(cfg_path)
+        try:
+            config = load_json(cfg_path)
+        except (OSError, json.JSONDecodeError):
+            return  # eval-config reports unreadable configs
+
+        def fail(msg: str) -> None:
+            result.violations.append(Violation(cfg_path, msg))
+
+        rel = config.get("fixtures", {}).get("path")
+        if not rel:
+            return
+        path = os.path.join(base, rel)
+        if not os.path.exists(path):
+            return  # eval-config reports the missing file
+        try:
+            fixtures = load_json(path)
+        except json.JSONDecodeError as e:
+            fail(f"{rel}: invalid JSON: {e}")
+            return
+
+        # Layer 1: shared structure.
+        structurally_ok = True
+        for err in sorted(shared.iter_errors(fixtures), key=lambda e: list(e.path)):
+            structurally_ok = False
+            fail(f"{rel}: {self._loc(err)}: {err.message}")
+        if not structurally_ok:
+            return  # per-case binding below assumes well-formed cases
+
+        # Layer 2: bind input/expected to this evaluator's own schemas.
+        input_validator = self._validator_for(config, base, "input_schema")
+        expected_validator = self._expected_validator(config, base)
+        for case in fixtures:
+            cid = case.get("id", "?")
+            if input_validator:
+                for err in input_validator.iter_errors(case.get("input", {})):
+                    fail(f"{rel}[{cid}].input: {self._loc(err)}: {err.message}")
+            if expected_validator:
+                for err in expected_validator.iter_errors(case.get("expected", {})):
+                    fail(f"{rel}[{cid}].expected: {self._loc(err)}: {err.message}")
+
+    def _validator_for(self, config: dict, base: str, key: str):
+        ref = config.get(key, {})
+        if not (isinstance(ref, dict) and "$ref" in ref):
+            return None
+        path = os.path.join(base, ref["$ref"])
+        doc = self._load_or_none(path)
+        return self._safe_validator(doc) if doc is not None else None
+
+    def _expected_validator(self, config: dict, base: str):
+        """A partial validator: each expected field must match its output property.
+
+        Built from output_schema.properties (+ $defs for $ref resolution) with no
+        `required`, so a fixture carrying only the label validates, and an unknown
+        expected field is rejected.
+        """
+        ref = config.get("output_schema", {})
+        if not (isinstance(ref, dict) and "$ref" in ref):
+            return None
+        path = os.path.join(base, ref["$ref"])
+        out = self._load_or_none(path)
+        if out is None:
+            return None
+        schema = {
+            "type": "object",
+            "additionalProperties": False,
+            "properties": out.get("properties", {}),
+        }
+        if "$defs" in out:
+            schema["$defs"] = out["$defs"]
+        return self._safe_validator(schema)
+
+    @staticmethod
+    def _load_or_none(path: str):
+        """Load a JSON file, or None if missing/unreadable/invalid.
+
+        A malformed input/output schema is reported by eval-schemas; here we just
+        skip binding rather than crash on it.
+        """
+        if not os.path.exists(path):
+            return None
+        try:
+            return load_json(path)
+        except (OSError, json.JSONDecodeError):
+            return None
+
+    @staticmethod
+    def _safe_validator(schema: dict):
+        """Build a validator only if `schema` is itself a valid JSON Schema."""
+        try:
+            Draft202012Validator.check_schema(schema)
+        except SchemaError:
+            return None
+        return Draft202012Validator(schema)
+
+    @staticmethod
+    def _loc(err) -> str:
+        return "/".join(str(p) for p in err.path) or "(root)"

scripts/checks/eval_notebook.py

@@ -0,0 +1,79 @@
+"""Check that an evaluator's example notebook reads from its config, not copies.
+
+Where an evaluator ships a notebook alongside its config, the notebook should
+load the canonical assets from disk (config.json + the prompt files declared in
+it) rather than hardcoding prompts/schema inline — so the notebook can't drift
+from the source of truth. Evaluators without a notebook (e.g. config-only ones)
+are skipped.
+
+This is a lightweight heuristic: it confirms the notebook's code references
+config.json and loads the prompt files (by the `source_path` loader pattern, or
+by naming each prompt file). It does not execute the notebook.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+
+from .base import Check, Result, Violation, evaluator_configs, load_json, tracked_files
+
+
+class EvalNotebook(Check):
+    name = "eval-notebook"
+    description = "Where an evaluator ships a notebook, confirm it loads config + prompts from disk"
+
+    def run(self, fix: bool) -> Result:
+        result = Result(self.name)
+        for cfg_path in evaluator_configs():
+            self._check_notebook(cfg_path, result)
+        return result
+
+    def _check_notebook(self, cfg_path: str, result: Result) -> None:
+        base = os.path.dirname(cfg_path)
+        # Git-tracked only, so local/untracked notebooks and checkpoints don't trip the check.
+        notebooks = tracked_files(os.path.join(base, "*.ipynb"))
+        if not notebooks:
+            return  # config-only evaluator — nothing to check
+
+        try:
+            config = load_json(cfg_path)
+        except (OSError, json.JSONDecodeError):
+            return  # eval-config reports unreadable configs
+        prompt_files = [
+            m["source_path"]
+            for m in config.get("steps", [{}])[0].get("prompt", {}).get("messages", [])
+            if m.get("source_path")
+        ]
+
+        for nb_path in notebooks:
+            src = self._code_source(nb_path)
+            if src is None:
+                result.violations.append(Violation(nb_path, "invalid notebook JSON"))
+                continue
+
+            if "config.json" not in src:
+                result.violations.append(
+                    Violation(nb_path, "does not load config.json — should read the config, not hardcode it")
+                )
+            # Either the dynamic loader pattern, or every prompt file named explicitly.
+            loads_prompts = "source_path" in src or (
+                prompt_files and all(pf in src for pf in prompt_files)
+            )
+            if not loads_prompts:
+                result.violations.append(
+                    Violation(nb_path, "does not load the prompt files from disk (no source_path loader nor prompt filenames)")
+                )
+
+    @staticmethod
+    def _code_source(nb_path: str) -> str | None:
+        """Concatenated source of a notebook's code cells, or None if unreadable."""
+        try:
+            nb = load_json(nb_path)
+        except (OSError, json.JSONDecodeError):
+            return None
+        return "\n".join(
+            "".join(c.get("source", []))
+            for c in nb.get("cells", [])
+            if c.get("cell_type") == "code"
+        )

scripts/checks/eval_schemas.py

@@ -0,0 +1,51 @@
+"""Meta-validate each evaluator's input_schema.json / output_schema.json.
+
+These files *are* JSON Schemas (they describe an evaluator's input and output),
+so the check is: is each one a well-formed JSON Schema document? We run the
+Draft 2020-12 meta-schema against them. Existence of the $ref is already covered
+by eval-config; here we only judge validity of the schema documents themselves.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+
+from jsonschema import Draft202012Validator
+from jsonschema.exceptions import SchemaError
+
+from .base import Check, Result, Violation, evaluator_configs, load_json
+
+
+class EvalSchemas(Check):
+    name = "eval-schemas"
+    description = "Meta-validate each evaluator's input_schema.json / output_schema.json"
+
+    def run(self, fix: bool) -> Result:
+        result = Result(self.name)
+        for cfg_path in evaluator_configs():
+            base = os.path.dirname(cfg_path)
+            try:
+                config = load_json(cfg_path)
+            except (OSError, json.JSONDecodeError):
+                continue  # eval-config reports unreadable configs
+            for key in ("input_schema", "output_schema"):
+                self._check_schema_file(config.get(key, {}), base, result)
+        return result
+
+    def _check_schema_file(self, ref: dict, base: str, result: Result) -> None:
+        if not (isinstance(ref, dict) and "$ref" in ref):
+            return
+        rel = ref["$ref"]
+        path = os.path.join(base, rel)
+        if not os.path.exists(path):
+            return  # eval-config already flags missing $ref targets
+        try:
+            doc = load_json(path)
+        except json.JSONDecodeError as e:
+            result.violations.append(Violation(path, f"invalid JSON: {e}"))
+            return
+        try:
+            Draft202012Validator.check_schema(doc)
+        except SchemaError as e:
+            result.violations.append(Violation(path, f"not a valid JSON Schema: {e.message}"))