diff --git a/README.md b/README.md index b342fea..fb1f338 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ Evaluators and the supporting datasets are built in collaboration with leading l | :------------------------ | :--------------------------------------------------------------- | | [`evals`](./evals/) | Evaluators code and prompts | | [`datasets`](./datasets/) | Expert annotated datasets used to create and validate evaluators | +| [`scripts`](./scripts/) | Repo check harness — run `python scripts/check.py --fix` before committing (see [scripts/README](./scripts/README.md)) | | [`LICENSE`](./LICENSE.md) | Open source license details | Check out the [Evaluators docs](https://docs.learningcommons.org/evaluators) for complete setup instructions and usage examples. diff --git a/scripts/README.md b/scripts/README.md index bc8c17a..15a9ee8 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -37,25 +37,20 @@ fails the PR (it never auto-fixes) — so fixing locally saves a round-trip. |-------|--------------| | `strip-notebooks` | Clears outputs, execution counts, and cell metadata from `.ipynb` files | | `eval-config` | Validates each evaluator `config.json` against the shared schema (`evals/_schemas/config.schema.json`, referenced by the config's own `$schema`), plus cross-file rules: referenced files exist, prompt `sha256` matches, placeholders ⇄ template `{vars}` are in sync, system prompts carry no placeholders, and no obsolete `format_instructions` survive | +| `eval-schemas` | Meta-validates each `input_schema.json` / `output_schema.json` is a well-formed JSON Schema document | +| `eval-fixtures` | Validates `fixtures.json` against the shared `evals/_schemas/fixtures.schema.json`, and binds each case's `input`/`expected` to that evaluator's own input/output schema | +| `eval-notebook` | Where an evaluator ships an example notebook, confirms it loads the config + prompt files from disk rather than hardcoding them | ## Coming next This is the seed of a broader self-service harness. Planned additions: -- **`eval-schemas`** — meta-validate each `input_schema.json` / `output_schema.json` - is a well-formed JSON Schema document. -- **`eval-fixtures`** — validate `fixtures.json` against the shared - `evals/_schemas/fixtures.schema.json`, and bind each case's `input`/`expected` - to that evaluator's own input/output schema. -- **`eval-notebook`** — where an evaluator ships an example notebook, confirm it - loads the config + prompt files from disk rather than hardcoding them. - **delegation to the SDK suites** (`sdks/typescript` lint/type/test, `sdks/python` Makefile) so one local command covers everything. CI keeps the per-SDK workflows separate for parallelism and path-filtered signal. -> **TODO** (once there are more checks): surface this from the root `README.md` -> / `CONTRIBUTING.md` so newcomers discover it — e.g. "Before committing, run -> `python scripts/check.py --fix` (see `scripts/README.md`)." +> **TODO**: the root `README.md` links here for discoverability; give this a +> proper home in a `CONTRIBUTING.md` once one exists. ## Adding a check diff --git a/scripts/checks/__init__.py b/scripts/checks/__init__.py index 521f657..1049a6b 100644 --- a/scripts/checks/__init__.py +++ b/scripts/checks/__init__.py @@ -1,9 +1,15 @@ """Check registry. Add new checks here to wire them into the harness + CI.""" from .eval_config import EvalConfig +from .eval_fixtures import EvalFixtures +from .eval_notebook import EvalNotebook +from .eval_schemas import EvalSchemas from .strip_notebooks import StripNotebooks ALL_CHECKS = [ StripNotebooks(), EvalConfig(), + EvalSchemas(), + EvalFixtures(), + EvalNotebook(), ] diff --git a/scripts/checks/base.py b/scripts/checks/base.py index 22e78a6..7fbf163 100644 --- a/scripts/checks/base.py +++ b/scripts/checks/base.py @@ -60,9 +60,9 @@ def evaluator_configs() -> list[str]: return tracked_files("evals/**/config.json") -def shared_schema_path(spec_version: str, name: str) -> str: - """Path to a version-pinned shared schema, e.g. evals/_schemas/1.0/.""" - return os.path.join(SCHEMA_ROOT, spec_version, name) +def shared_schema_path(name: str) -> str: + """Path to a shared schema, e.g. evals/_schemas/fixtures.schema.json.""" + return os.path.join(SCHEMA_ROOT, name) def load_json(path: str): diff --git a/scripts/checks/eval_fixtures.py b/scripts/checks/eval_fixtures.py new file mode 100644 index 0000000..bef79c2 --- /dev/null +++ b/scripts/checks/eval_fixtures.py @@ -0,0 +1,134 @@ +"""Validate each evaluator's fixtures.json. + +Two layers, mirroring eval-config: + - shared structure: every case is {id, input, expected} per the shared + evals/_schemas/fixtures.schema.json. + - per-evaluator binding: each case's `input` must satisfy that evaluator's own + input_schema.json, and each field in `expected` must satisfy the matching + property in its output_schema.json. (`expected` is a subset of the full + output — just the label(s) — so we validate present fields, not requireds.) +""" + +from __future__ import annotations + +import json +import os + +from jsonschema import Draft202012Validator +from jsonschema.exceptions import SchemaError + +from .base import Check, Result, Violation, evaluator_configs, load_json, shared_schema_path + +_FIXTURES_SCHEMA = "fixtures.schema.json" + + +class EvalFixtures(Check): + name = "eval-fixtures" + description = "Validate fixtures.json structure + bind each case's input/expected to the evaluator's schemas" + + def run(self, fix: bool) -> Result: + result = Result(self.name) + shared = Draft202012Validator(load_json(shared_schema_path(_FIXTURES_SCHEMA))) + for cfg_path in evaluator_configs(): + self._check_fixtures(cfg_path, shared, result) + return result + + def _check_fixtures(self, cfg_path: str, shared: Draft202012Validator, result: Result) -> None: + base = os.path.dirname(cfg_path) + try: + config = load_json(cfg_path) + except (OSError, json.JSONDecodeError): + return # eval-config reports unreadable configs + + def fail(msg: str) -> None: + result.violations.append(Violation(cfg_path, msg)) + + rel = config.get("fixtures", {}).get("path") + if not rel: + return + path = os.path.join(base, rel) + if not os.path.exists(path): + return # eval-config reports the missing file + try: + fixtures = load_json(path) + except json.JSONDecodeError as e: + fail(f"{rel}: invalid JSON: {e}") + return + + # Layer 1: shared structure. + structurally_ok = True + for err in sorted(shared.iter_errors(fixtures), key=lambda e: list(e.path)): + structurally_ok = False + fail(f"{rel}: {self._loc(err)}: {err.message}") + if not structurally_ok: + return # per-case binding below assumes well-formed cases + + # Layer 2: bind input/expected to this evaluator's own schemas. + input_validator = self._validator_for(config, base, "input_schema") + expected_validator = self._expected_validator(config, base) + for case in fixtures: + cid = case.get("id", "?") + if input_validator: + for err in input_validator.iter_errors(case.get("input", {})): + fail(f"{rel}[{cid}].input: {self._loc(err)}: {err.message}") + if expected_validator: + for err in expected_validator.iter_errors(case.get("expected", {})): + fail(f"{rel}[{cid}].expected: {self._loc(err)}: {err.message}") + + def _validator_for(self, config: dict, base: str, key: str): + ref = config.get(key, {}) + if not (isinstance(ref, dict) and "$ref" in ref): + return None + path = os.path.join(base, ref["$ref"]) + doc = self._load_or_none(path) + return self._safe_validator(doc) if doc is not None else None + + def _expected_validator(self, config: dict, base: str): + """A partial validator: each expected field must match its output property. + + Built from output_schema.properties (+ $defs for $ref resolution) with no + `required`, so a fixture carrying only the label validates, and an unknown + expected field is rejected. + """ + ref = config.get("output_schema", {}) + if not (isinstance(ref, dict) and "$ref" in ref): + return None + path = os.path.join(base, ref["$ref"]) + out = self._load_or_none(path) + if out is None: + return None + schema = { + "type": "object", + "additionalProperties": False, + "properties": out.get("properties", {}), + } + if "$defs" in out: + schema["$defs"] = out["$defs"] + return self._safe_validator(schema) + + @staticmethod + def _load_or_none(path: str): + """Load a JSON file, or None if missing/unreadable/invalid. + + A malformed input/output schema is reported by eval-schemas; here we just + skip binding rather than crash on it. + """ + if not os.path.exists(path): + return None + try: + return load_json(path) + except (OSError, json.JSONDecodeError): + return None + + @staticmethod + def _safe_validator(schema: dict): + """Build a validator only if `schema` is itself a valid JSON Schema.""" + try: + Draft202012Validator.check_schema(schema) + except SchemaError: + return None + return Draft202012Validator(schema) + + @staticmethod + def _loc(err) -> str: + return "/".join(str(p) for p in err.path) or "(root)" diff --git a/scripts/checks/eval_notebook.py b/scripts/checks/eval_notebook.py new file mode 100644 index 0000000..54a1951 --- /dev/null +++ b/scripts/checks/eval_notebook.py @@ -0,0 +1,79 @@ +"""Check that an evaluator's example notebook reads from its config, not copies. + +Where an evaluator ships a notebook alongside its config, the notebook should +load the canonical assets from disk (config.json + the prompt files declared in +it) rather than hardcoding prompts/schema inline — so the notebook can't drift +from the source of truth. Evaluators without a notebook (e.g. config-only ones) +are skipped. + +This is a lightweight heuristic: it confirms the notebook's code references +config.json and loads the prompt files (by the `source_path` loader pattern, or +by naming each prompt file). It does not execute the notebook. +""" + +from __future__ import annotations + +import json +import os + +from .base import Check, Result, Violation, evaluator_configs, load_json, tracked_files + + +class EvalNotebook(Check): + name = "eval-notebook" + description = "Where an evaluator ships a notebook, confirm it loads config + prompts from disk" + + def run(self, fix: bool) -> Result: + result = Result(self.name) + for cfg_path in evaluator_configs(): + self._check_notebook(cfg_path, result) + return result + + def _check_notebook(self, cfg_path: str, result: Result) -> None: + base = os.path.dirname(cfg_path) + # Git-tracked only, so local/untracked notebooks and checkpoints don't trip the check. + notebooks = tracked_files(os.path.join(base, "*.ipynb")) + if not notebooks: + return # config-only evaluator — nothing to check + + try: + config = load_json(cfg_path) + except (OSError, json.JSONDecodeError): + return # eval-config reports unreadable configs + prompt_files = [ + m["source_path"] + for m in config.get("steps", [{}])[0].get("prompt", {}).get("messages", []) + if m.get("source_path") + ] + + for nb_path in notebooks: + src = self._code_source(nb_path) + if src is None: + result.violations.append(Violation(nb_path, "invalid notebook JSON")) + continue + + if "config.json" not in src: + result.violations.append( + Violation(nb_path, "does not load config.json — should read the config, not hardcode it") + ) + # Either the dynamic loader pattern, or every prompt file named explicitly. + loads_prompts = "source_path" in src or ( + prompt_files and all(pf in src for pf in prompt_files) + ) + if not loads_prompts: + result.violations.append( + Violation(nb_path, "does not load the prompt files from disk (no source_path loader nor prompt filenames)") + ) + + @staticmethod + def _code_source(nb_path: str) -> str | None: + """Concatenated source of a notebook's code cells, or None if unreadable.""" + try: + nb = load_json(nb_path) + except (OSError, json.JSONDecodeError): + return None + return "\n".join( + "".join(c.get("source", [])) + for c in nb.get("cells", []) + if c.get("cell_type") == "code" + ) diff --git a/scripts/checks/eval_schemas.py b/scripts/checks/eval_schemas.py new file mode 100644 index 0000000..ffa0f8b --- /dev/null +++ b/scripts/checks/eval_schemas.py @@ -0,0 +1,51 @@ +"""Meta-validate each evaluator's input_schema.json / output_schema.json. + +These files *are* JSON Schemas (they describe an evaluator's input and output), +so the check is: is each one a well-formed JSON Schema document? We run the +Draft 2020-12 meta-schema against them. Existence of the $ref is already covered +by eval-config; here we only judge validity of the schema documents themselves. +""" + +from __future__ import annotations + +import json +import os + +from jsonschema import Draft202012Validator +from jsonschema.exceptions import SchemaError + +from .base import Check, Result, Violation, evaluator_configs, load_json + + +class EvalSchemas(Check): + name = "eval-schemas" + description = "Meta-validate each evaluator's input_schema.json / output_schema.json" + + def run(self, fix: bool) -> Result: + result = Result(self.name) + for cfg_path in evaluator_configs(): + base = os.path.dirname(cfg_path) + try: + config = load_json(cfg_path) + except (OSError, json.JSONDecodeError): + continue # eval-config reports unreadable configs + for key in ("input_schema", "output_schema"): + self._check_schema_file(config.get(key, {}), base, result) + return result + + def _check_schema_file(self, ref: dict, base: str, result: Result) -> None: + if not (isinstance(ref, dict) and "$ref" in ref): + return + rel = ref["$ref"] + path = os.path.join(base, rel) + if not os.path.exists(path): + return # eval-config already flags missing $ref targets + try: + doc = load_json(path) + except json.JSONDecodeError as e: + result.violations.append(Violation(path, f"invalid JSON: {e}")) + return + try: + Draft202012Validator.check_schema(doc) + except SchemaError as e: + result.violations.append(Violation(path, f"not a valid JSON Schema: {e.message}"))