From 15f9943c1153da27b4167bac60d60f4dd4e56ba7 Mon Sep 17 00:00:00 2001 From: Adnan Rashid Hussain Date: Mon, 29 Jun 2026 12:24:50 -0700 Subject: [PATCH 1/3] feat: add eval-schemas, eval-fixtures, eval-notebook checks --- scripts/README.md | 12 ++-- scripts/checks/__init__.py | 6 ++ scripts/checks/base.py | 6 +- scripts/checks/eval_fixtures.py | 122 ++++++++++++++++++++++++++++++++ scripts/checks/eval_notebook.py | 73 +++++++++++++++++++ scripts/checks/eval_schemas.py | 51 +++++++++++++ 6 files changed, 260 insertions(+), 10 deletions(-) create mode 100644 scripts/checks/eval_fixtures.py create mode 100644 scripts/checks/eval_notebook.py create mode 100644 scripts/checks/eval_schemas.py diff --git a/scripts/README.md b/scripts/README.md index bc8c17a..5cde5f0 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -37,18 +37,16 @@ fails the PR (it never auto-fixes) — so fixing locally saves a round-trip. |-------|--------------| | `strip-notebooks` | Clears outputs, execution counts, and cell metadata from `.ipynb` files | | `eval-config` | Validates each evaluator `config.json` against the shared schema (`evals/_schemas/config.schema.json`, referenced by the config's own `$schema`), plus cross-file rules: referenced files exist, prompt `sha256` matches, placeholders ⇄ template `{vars}` are in sync, system prompts carry no placeholders, and no obsolete `format_instructions` survive | +| `eval-schemas` | Meta-validates each `input_schema.json` / `output_schema.json` is a well-formed JSON Schema document | +| `eval-fixtures` | Validates `fixtures.json` against the shared `evals/_schemas/fixtures.schema.json`, and binds each case's `input`/`expected` to that evaluator's own input/output schema | +| `eval-notebook` | Where an evaluator ships an example notebook, confirms it loads the config + prompt files from disk rather than hardcoding them | ## Coming next This is the seed of a broader self-service harness. Planned additions: -- **`eval-schemas`** — meta-validate each `input_schema.json` / `output_schema.json` - is a well-formed JSON Schema document. -- **`eval-fixtures`** — validate `fixtures.json` against the shared - `evals/_schemas/fixtures.schema.json`, and bind each case's `input`/`expected` - to that evaluator's own input/output schema. -- **`eval-notebook`** — where an evaluator ships an example notebook, confirm it - loads the config + prompt files from disk rather than hardcoding them. +- **model-snapshot pinning** — require `model.name` to reference a pinned + snapshot (provider-specific), so evaluators don't float across model updates. - **delegation to the SDK suites** (`sdks/typescript` lint/type/test, `sdks/python` Makefile) so one local command covers everything. CI keeps the per-SDK workflows separate for parallelism and path-filtered signal. diff --git a/scripts/checks/__init__.py b/scripts/checks/__init__.py index 521f657..1049a6b 100644 --- a/scripts/checks/__init__.py +++ b/scripts/checks/__init__.py @@ -1,9 +1,15 @@ """Check registry. Add new checks here to wire them into the harness + CI.""" from .eval_config import EvalConfig +from .eval_fixtures import EvalFixtures +from .eval_notebook import EvalNotebook +from .eval_schemas import EvalSchemas from .strip_notebooks import StripNotebooks ALL_CHECKS = [ StripNotebooks(), EvalConfig(), + EvalSchemas(), + EvalFixtures(), + EvalNotebook(), ] diff --git a/scripts/checks/base.py b/scripts/checks/base.py index 22e78a6..7fbf163 100644 --- a/scripts/checks/base.py +++ b/scripts/checks/base.py @@ -60,9 +60,9 @@ def evaluator_configs() -> list[str]: return tracked_files("evals/**/config.json") -def shared_schema_path(spec_version: str, name: str) -> str: - """Path to a version-pinned shared schema, e.g. evals/_schemas/1.0/.""" - return os.path.join(SCHEMA_ROOT, spec_version, name) +def shared_schema_path(name: str) -> str: + """Path to a shared schema, e.g. evals/_schemas/fixtures.schema.json.""" + return os.path.join(SCHEMA_ROOT, name) def load_json(path: str): diff --git a/scripts/checks/eval_fixtures.py b/scripts/checks/eval_fixtures.py new file mode 100644 index 0000000..fd9397f --- /dev/null +++ b/scripts/checks/eval_fixtures.py @@ -0,0 +1,122 @@ +"""Validate each evaluator's fixtures.json. + +Two layers, mirroring eval-config: + - shared structure: every case is {id, input, expected} per the shared + evals/_schemas/fixtures.schema.json. + - per-evaluator binding: each case's `input` must satisfy that evaluator's own + input_schema.json, and each field in `expected` must satisfy the matching + property in its output_schema.json. (`expected` is a subset of the full + output — just the label(s) — so we validate present fields, not requireds.) +""" + +from __future__ import annotations + +import json +import os + +from jsonschema import Draft202012Validator +from jsonschema.exceptions import SchemaError + +from .base import Check, Result, Violation, evaluator_configs, load_json, shared_schema_path + +_FIXTURES_SCHEMA = "fixtures.schema.json" + + +class EvalFixtures(Check): + name = "eval-fixtures" + description = "Validate fixtures.json structure + bind each case's input/expected to the evaluator's schemas" + + def run(self, fix: bool) -> Result: + result = Result(self.name) + shared = Draft202012Validator(load_json(shared_schema_path(_FIXTURES_SCHEMA))) + for cfg_path in evaluator_configs(): + self._check_fixtures(cfg_path, shared, result) + return result + + def _check_fixtures(self, cfg_path: str, shared: Draft202012Validator, result: Result) -> None: + base = os.path.dirname(cfg_path) + config = load_json(cfg_path) + + def fail(msg: str) -> None: + result.violations.append(Violation(cfg_path, msg)) + + rel = config.get("fixtures", {}).get("path") + if not rel: + return + path = os.path.join(base, rel) + if not os.path.exists(path): + return # eval-config reports the missing file + try: + fixtures = load_json(path) + except json.JSONDecodeError as e: + fail(f"{rel}: invalid JSON: {e}") + return + + # Layer 1: shared structure. + structurally_ok = True + for err in sorted(shared.iter_errors(fixtures), key=lambda e: list(e.path)): + structurally_ok = False + fail(f"{rel}: {self._loc(err)}: {err.message}") + if not structurally_ok: + return # per-case binding below assumes well-formed cases + + # Layer 2: bind input/expected to this evaluator's own schemas. + input_validator = self._validator_for(config, base, "input_schema") + expected_validator = self._expected_validator(config, base) + for case in fixtures: + cid = case.get("id", "?") + if input_validator: + for err in input_validator.iter_errors(case.get("input", {})): + fail(f"{rel}[{cid}].input: {self._loc(err)}: {err.message}") + if expected_validator: + for err in expected_validator.iter_errors(case.get("expected", {})): + fail(f"{rel}[{cid}].expected: {self._loc(err)}: {err.message}") + + def _validator_for(self, config: dict, base: str, key: str): + ref = config.get(key, {}) + if not (isinstance(ref, dict) and "$ref" in ref): + return None + path = os.path.join(base, ref["$ref"]) + if not os.path.exists(path): + return None + return self._safe_validator(load_json(path)) + + def _expected_validator(self, config: dict, base: str): + """A partial validator: each expected field must match its output property. + + Built from output_schema.properties (+ $defs for $ref resolution) with no + `required`, so a fixture carrying only the label validates, and an unknown + expected field is rejected. + """ + ref = config.get("output_schema", {}) + if not (isinstance(ref, dict) and "$ref" in ref): + return None + path = os.path.join(base, ref["$ref"]) + if not os.path.exists(path): + return None + out = load_json(path) + schema = { + "type": "object", + "additionalProperties": False, + "properties": out.get("properties", {}), + } + if "$defs" in out: + schema["$defs"] = out["$defs"] + return self._safe_validator(schema) + + @staticmethod + def _safe_validator(schema: dict): + """Build a validator only if `schema` is itself a valid JSON Schema. + + A malformed input/output schema is reported by eval-schemas; here we just + skip binding rather than crash on it. + """ + try: + Draft202012Validator.check_schema(schema) + except SchemaError: + return None + return Draft202012Validator(schema) + + @staticmethod + def _loc(err) -> str: + return "/".join(str(p) for p in err.path) or "(root)" diff --git a/scripts/checks/eval_notebook.py b/scripts/checks/eval_notebook.py new file mode 100644 index 0000000..125aef3 --- /dev/null +++ b/scripts/checks/eval_notebook.py @@ -0,0 +1,73 @@ +"""Check that an evaluator's example notebook reads from its config, not copies. + +Where an evaluator ships a notebook alongside its config, the notebook should +load the canonical assets from disk (config.json + the prompt files declared in +it) rather than hardcoding prompts/schema inline — so the notebook can't drift +from the source of truth. Evaluators without a notebook (e.g. config-only ones) +are skipped. + +This is a lightweight heuristic: it confirms the notebook's code references +config.json and loads the prompt files (by the `source_path` loader pattern, or +by naming each prompt file). It does not execute the notebook. +""" + +from __future__ import annotations + +import glob +import json +import os + +from .base import Check, Result, Violation, evaluator_configs, load_json + + +class EvalNotebook(Check): + name = "eval-notebook" + description = "Where an evaluator ships a notebook, confirm it loads config + prompts from disk" + + def run(self, fix: bool) -> Result: + result = Result(self.name) + for cfg_path in evaluator_configs(): + self._check_notebook(cfg_path, result) + return result + + def _check_notebook(self, cfg_path: str, result: Result) -> None: + base = os.path.dirname(cfg_path) + notebooks = glob.glob(os.path.join(base, "*.ipynb")) + if not notebooks: + return # config-only evaluator — nothing to check + + config = load_json(cfg_path) + prompt_files = [ + m["source_path"] + for m in config.get("steps", [{}])[0].get("prompt", {}).get("messages", []) + if m.get("source_path") + ] + + for nb_path in notebooks: + src = self._code_source(nb_path) + + if "config.json" not in src: + result.violations.append( + Violation(nb_path, "does not load config.json — should read the config, not hardcode it") + ) + # Either the dynamic loader pattern, or every prompt file named explicitly. + loads_prompts = "source_path" in src or ( + prompt_files and all(pf in src for pf in prompt_files) + ) + if not loads_prompts: + result.violations.append( + Violation(nb_path, "does not load the prompt files from disk (no source_path loader nor prompt filenames)") + ) + + @staticmethod + def _code_source(nb_path: str) -> str: + """Concatenated source of a notebook's code cells.""" + try: + nb = load_json(nb_path) + except (OSError, json.JSONDecodeError): + return "" + return "\n".join( + "".join(c.get("source", [])) + for c in nb.get("cells", []) + if c.get("cell_type") == "code" + ) diff --git a/scripts/checks/eval_schemas.py b/scripts/checks/eval_schemas.py new file mode 100644 index 0000000..ffa0f8b --- /dev/null +++ b/scripts/checks/eval_schemas.py @@ -0,0 +1,51 @@ +"""Meta-validate each evaluator's input_schema.json / output_schema.json. + +These files *are* JSON Schemas (they describe an evaluator's input and output), +so the check is: is each one a well-formed JSON Schema document? We run the +Draft 2020-12 meta-schema against them. Existence of the $ref is already covered +by eval-config; here we only judge validity of the schema documents themselves. +""" + +from __future__ import annotations + +import json +import os + +from jsonschema import Draft202012Validator +from jsonschema.exceptions import SchemaError + +from .base import Check, Result, Violation, evaluator_configs, load_json + + +class EvalSchemas(Check): + name = "eval-schemas" + description = "Meta-validate each evaluator's input_schema.json / output_schema.json" + + def run(self, fix: bool) -> Result: + result = Result(self.name) + for cfg_path in evaluator_configs(): + base = os.path.dirname(cfg_path) + try: + config = load_json(cfg_path) + except (OSError, json.JSONDecodeError): + continue # eval-config reports unreadable configs + for key in ("input_schema", "output_schema"): + self._check_schema_file(config.get(key, {}), base, result) + return result + + def _check_schema_file(self, ref: dict, base: str, result: Result) -> None: + if not (isinstance(ref, dict) and "$ref" in ref): + return + rel = ref["$ref"] + path = os.path.join(base, rel) + if not os.path.exists(path): + return # eval-config already flags missing $ref targets + try: + doc = load_json(path) + except json.JSONDecodeError as e: + result.violations.append(Violation(path, f"invalid JSON: {e}")) + return + try: + Draft202012Validator.check_schema(doc) + except SchemaError as e: + result.violations.append(Violation(path, f"not a valid JSON Schema: {e.message}")) From 5db7191e1236af3099e438f868d6844f23afdba5 Mon Sep 17 00:00:00 2001 From: Adnan Rashid Hussain Date: Mon, 29 Jun 2026 12:34:12 -0700 Subject: [PATCH 2/3] docs: drop model-snapshot from roadmap; link harness from root README --- README.md | 1 + scripts/README.md | 7 ++----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b342fea..fb1f338 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ Evaluators and the supporting datasets are built in collaboration with leading l | :------------------------ | :--------------------------------------------------------------- | | [`evals`](./evals/) | Evaluators code and prompts | | [`datasets`](./datasets/) | Expert annotated datasets used to create and validate evaluators | +| [`scripts`](./scripts/) | Repo check harness — run `python scripts/check.py --fix` before committing (see [scripts/README](./scripts/README.md)) | | [`LICENSE`](./LICENSE.md) | Open source license details | Check out the [Evaluators docs](https://docs.learningcommons.org/evaluators) for complete setup instructions and usage examples. diff --git a/scripts/README.md b/scripts/README.md index 5cde5f0..15a9ee8 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -45,15 +45,12 @@ fails the PR (it never auto-fixes) — so fixing locally saves a round-trip. This is the seed of a broader self-service harness. Planned additions: -- **model-snapshot pinning** — require `model.name` to reference a pinned - snapshot (provider-specific), so evaluators don't float across model updates. - **delegation to the SDK suites** (`sdks/typescript` lint/type/test, `sdks/python` Makefile) so one local command covers everything. CI keeps the per-SDK workflows separate for parallelism and path-filtered signal. -> **TODO** (once there are more checks): surface this from the root `README.md` -> / `CONTRIBUTING.md` so newcomers discover it — e.g. "Before committing, run -> `python scripts/check.py --fix` (see `scripts/README.md`)." +> **TODO**: the root `README.md` links here for discoverability; give this a +> proper home in a `CONTRIBUTING.md` once one exists. ## Adding a check From 86269c6acd1c1697ad3400c22c6270448849c091 Mon Sep 17 00:00:00 2001 From: Adnan Rashid Hussain Date: Mon, 29 Jun 2026 12:56:18 -0700 Subject: [PATCH 3/3] fix: harden eval-fixtures/eval-notebook against unreadable inputs; use tracked_files for notebooks --- scripts/checks/eval_fixtures.py | 28 ++++++++++++++++++++-------- scripts/checks/eval_notebook.py | 20 +++++++++++++------- 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/scripts/checks/eval_fixtures.py b/scripts/checks/eval_fixtures.py index fd9397f..bef79c2 100644 --- a/scripts/checks/eval_fixtures.py +++ b/scripts/checks/eval_fixtures.py @@ -35,7 +35,10 @@ def run(self, fix: bool) -> Result: def _check_fixtures(self, cfg_path: str, shared: Draft202012Validator, result: Result) -> None: base = os.path.dirname(cfg_path) - config = load_json(cfg_path) + try: + config = load_json(cfg_path) + except (OSError, json.JSONDecodeError): + return # eval-config reports unreadable configs def fail(msg: str) -> None: result.violations.append(Violation(cfg_path, msg)) @@ -77,9 +80,8 @@ def _validator_for(self, config: dict, base: str, key: str): if not (isinstance(ref, dict) and "$ref" in ref): return None path = os.path.join(base, ref["$ref"]) - if not os.path.exists(path): - return None - return self._safe_validator(load_json(path)) + doc = self._load_or_none(path) + return self._safe_validator(doc) if doc is not None else None def _expected_validator(self, config: dict, base: str): """A partial validator: each expected field must match its output property. @@ -92,9 +94,9 @@ def _expected_validator(self, config: dict, base: str): if not (isinstance(ref, dict) and "$ref" in ref): return None path = os.path.join(base, ref["$ref"]) - if not os.path.exists(path): + out = self._load_or_none(path) + if out is None: return None - out = load_json(path) schema = { "type": "object", "additionalProperties": False, @@ -105,12 +107,22 @@ def _expected_validator(self, config: dict, base: str): return self._safe_validator(schema) @staticmethod - def _safe_validator(schema: dict): - """Build a validator only if `schema` is itself a valid JSON Schema. + def _load_or_none(path: str): + """Load a JSON file, or None if missing/unreadable/invalid. A malformed input/output schema is reported by eval-schemas; here we just skip binding rather than crash on it. """ + if not os.path.exists(path): + return None + try: + return load_json(path) + except (OSError, json.JSONDecodeError): + return None + + @staticmethod + def _safe_validator(schema: dict): + """Build a validator only if `schema` is itself a valid JSON Schema.""" try: Draft202012Validator.check_schema(schema) except SchemaError: diff --git a/scripts/checks/eval_notebook.py b/scripts/checks/eval_notebook.py index 125aef3..54a1951 100644 --- a/scripts/checks/eval_notebook.py +++ b/scripts/checks/eval_notebook.py @@ -13,11 +13,10 @@ from __future__ import annotations -import glob import json import os -from .base import Check, Result, Violation, evaluator_configs, load_json +from .base import Check, Result, Violation, evaluator_configs, load_json, tracked_files class EvalNotebook(Check): @@ -32,11 +31,15 @@ def run(self, fix: bool) -> Result: def _check_notebook(self, cfg_path: str, result: Result) -> None: base = os.path.dirname(cfg_path) - notebooks = glob.glob(os.path.join(base, "*.ipynb")) + # Git-tracked only, so local/untracked notebooks and checkpoints don't trip the check. + notebooks = tracked_files(os.path.join(base, "*.ipynb")) if not notebooks: return # config-only evaluator — nothing to check - config = load_json(cfg_path) + try: + config = load_json(cfg_path) + except (OSError, json.JSONDecodeError): + return # eval-config reports unreadable configs prompt_files = [ m["source_path"] for m in config.get("steps", [{}])[0].get("prompt", {}).get("messages", []) @@ -45,6 +48,9 @@ def _check_notebook(self, cfg_path: str, result: Result) -> None: for nb_path in notebooks: src = self._code_source(nb_path) + if src is None: + result.violations.append(Violation(nb_path, "invalid notebook JSON")) + continue if "config.json" not in src: result.violations.append( @@ -60,12 +66,12 @@ def _check_notebook(self, cfg_path: str, result: Result) -> None: ) @staticmethod - def _code_source(nb_path: str) -> str: - """Concatenated source of a notebook's code cells.""" + def _code_source(nb_path: str) -> str | None: + """Concatenated source of a notebook's code cells, or None if unreadable.""" try: nb = load_json(nb_path) except (OSError, json.JSONDecodeError): - return "" + return None return "\n".join( "".join(c.get("source", [])) for c in nb.get("cells", [])