diff --git a/evals/_schemas/config.schema.json b/evals/_schemas/config.schema.json new file mode 100644 index 00000000..db1f5649 --- /dev/null +++ b/evals/_schemas/config.schema.json @@ -0,0 +1,170 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "config.schema.json", + "title": "Evaluator config", + "description": "Shared contract for evals/**/config.json. One living schema, referenced by each config via its $schema field. Strict: additionalProperties is false at every level, so any new field must be added here deliberately.", + "type": "object", + "additionalProperties": false, + "required": ["$schema", "evaluator", "input_schema", "steps", "output_schema"], + "properties": { + "$schema": { "type": "string" }, + "evaluator": { + "type": "object", + "additionalProperties": false, + "required": ["id", "name", "description", "supported_grades"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "name": { "type": "string", "minLength": 1 }, + "description": { "type": "string", "minLength": 1 }, + "supported_grades": { + "type": "array", + "minItems": 1, + "items": { + "enum": ["K", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"] + } + } + } + }, + "input_schema": { "type": "object" }, + "output_schema": { "type": "object" }, + "preprocessing": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "type", "kind"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "type": { "enum": ["api", "computation"] }, + "kind": { "type": "string", "minLength": 1 }, + "description": { "type": "string" }, + "input": { "type": "string" }, + "output": { "type": "string" }, + "implementation": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "object", + "additionalProperties": false, + "required": ["library", "function"], + "properties": { + "library": { "type": "string", "minLength": 1 }, + "function": { "type": "string", "minLength": 1 }, + "post_transform": { + "type": "object", + "additionalProperties": false, + "required": ["type"], + "properties": { + "type": { "enum": ["round"] }, + "precision": { "type": "integer" } + } + } + } + } + }, + "endpoint": { "type": "string" }, + "params": { "type": "object" }, + "auth": { "type": "string" }, + "pagination": { "type": "string" } + }, + "allOf": [ + { + "if": { "properties": { "type": { "const": "api" } } }, + "then": { "required": ["endpoint", "params", "auth"] } + }, + { + "if": { "properties": { "type": { "const": "computation" } } }, + "then": { "required": ["implementation", "input", "output"] } + } + ] + } + }, + "steps": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "type"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "type": { "enum": ["llm", "api"] }, + "description": { "type": "string" }, + "prompt": { + "type": "object", + "additionalProperties": false, + "required": ["messages", "placeholders"], + "properties": { + "messages": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["role", "source_path"], + "properties": { + "role": { "enum": ["system", "user", "assistant"] }, + "source_path": { "type": "string", "minLength": 1 }, + "sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" } + } + } + }, + "placeholders": { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": false, + "required": ["required", "source"], + "properties": { + "required": { "type": "boolean" }, + "source": { "type": "string" }, + "note": { "type": "string" } + } + } + } + } + }, + "model": { + "type": "object", + "additionalProperties": false, + "required": ["provider", "name"], + "properties": { + "provider": { "enum": ["google", "openai", "anthropic"] }, + "name": { "type": "string", "minLength": 1 }, + "alias": { "type": "string" } + } + }, + "generation": { + "type": "object", + "additionalProperties": false, + "properties": { + "temperature": { "type": "number", "minimum": 0, "maximum": 1 } + } + }, + "parser": { + "type": "object", + "additionalProperties": false, + "required": ["kind"], + "properties": { + "kind": { "const": "structured_output" } + } + } + }, + "allOf": [ + { + "if": { "properties": { "type": { "const": "llm" } } }, + "then": { "required": ["prompt", "model", "parser"] } + } + ] + } + }, + "fixtures": { + "type": "object", + "additionalProperties": false, + "properties": { + "path": { "type": "string" }, + "tolerance": { "type": "object" } + } + } + } +} diff --git a/evals/_schemas/fixtures.schema.json b/evals/_schemas/fixtures.schema.json new file mode 100644 index 00000000..db582faa --- /dev/null +++ b/evals/_schemas/fixtures.schema.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "fixtures.schema.json", + "title": "Evaluator fixtures", + "description": "Shared contract for evals/**/fixtures.json. Each case carries inputs and the expected label(s); per-evaluator input/expected shapes are validated against the evaluator's own input_schema/output_schema by the eval-fixtures check, not here.", + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "input", "expected"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "description": { "type": "string" }, + "source": { "type": "string" }, + "input": { "type": "object" }, + "expected": { "type": "object" } + } + } +} diff --git a/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/config.json b/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/config.json index e0df2c35..722c9c59 100644 --- a/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/config.json +++ b/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/config.json @@ -1,9 +1,10 @@ { - "spec_version": "1.0", + "$schema": "../../../_schemas/config.schema.json", "evaluator": { "id": "feedback.productive_coaching_writing_feedback.is_acknowledges_strength", "name": "Acknowledges Strength Feedback-Quality Evaluator", - "description": "Does the feedback identify what the student did well?" + "description": "Does the feedback identify what the student did well?", + "supported_grades": ["8", "9"] }, "input_schema": { "$ref": "input_schema.json" @@ -12,9 +13,9 @@ "steps": [ { "id": "evaluate_is_acknowledges_strength", + "type": "llm", "description": "Single-call LLM step that produces the binary EvaluatorOutput JSON.", "prompt": { - "type": "chat", "messages": [ { "role": "system", @@ -46,17 +47,14 @@ "temperature": 1 }, "parser": { - "kind": "structured_output", - "output_schema_ref": "#/output_schema", - "notes": "Uses LangChain JsonOutputParser." - }, - "output_binding": "formatted_output" + "kind": "structured_output" + } } ], "output_schema": { "$ref": "output_schema.json" }, "fixtures": { - "sniff_test_path": "fixtures.json" + "path": "fixtures.json" } } diff --git a/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/example_notebook.ipynb b/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/example_notebook.ipynb index 9b695d32..102c2099 100644 --- a/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/example_notebook.ipynb +++ b/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/example_notebook.ipynb @@ -231,7 +231,7 @@ "metadata": {}, "outputs": [], "source": [ - "fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"sniff_test_path\"]\n", + "fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"path\"]\n", "if not fixtures_path.exists():\n", " print(f\"(no fixtures.json yet at {fixtures_path}; skipping fixture run)\")\n", "else:\n", diff --git a/evals/feedback/productive-coaching-writing-feedback/actionable-revision/config.json b/evals/feedback/productive-coaching-writing-feedback/actionable-revision/config.json index 190bb361..1effa48c 100644 --- a/evals/feedback/productive-coaching-writing-feedback/actionable-revision/config.json +++ b/evals/feedback/productive-coaching-writing-feedback/actionable-revision/config.json @@ -1,9 +1,10 @@ { - "spec_version": "1.0", + "$schema": "../../../_schemas/config.schema.json", "evaluator": { "id": "feedback.productive_coaching_writing_feedback.is_actionable_revision", "name": "Actionable Revision Feedback-Quality Evaluator", - "description": "If revision is needed, does the feedback give a clear next step?" + "description": "If revision is needed, does the feedback give a clear next step?", + "supported_grades": ["8", "9"] }, "input_schema": { "$ref": "input_schema.json" @@ -12,9 +13,9 @@ "steps": [ { "id": "evaluate_is_actionable_revision", + "type": "llm", "description": "Single-call LLM step that produces the binary EvaluatorOutput JSON.", "prompt": { - "type": "chat", "messages": [ { "role": "system", @@ -46,17 +47,14 @@ "temperature": 1 }, "parser": { - "kind": "structured_output", - "output_schema_ref": "#/output_schema", - "notes": "Uses LangChain JsonOutputParser." - }, - "output_binding": "formatted_output" + "kind": "structured_output" + } } ], "output_schema": { "$ref": "output_schema.json" }, "fixtures": { - "sniff_test_path": "fixtures.json" + "path": "fixtures.json" } } diff --git a/evals/feedback/productive-coaching-writing-feedback/actionable-revision/example_notebook.ipynb b/evals/feedback/productive-coaching-writing-feedback/actionable-revision/example_notebook.ipynb index bd6fcfeb..8bba61f0 100644 --- a/evals/feedback/productive-coaching-writing-feedback/actionable-revision/example_notebook.ipynb +++ b/evals/feedback/productive-coaching-writing-feedback/actionable-revision/example_notebook.ipynb @@ -231,7 +231,7 @@ "metadata": {}, "outputs": [], "source": [ - "fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"sniff_test_path\"]\n", + "fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"path\"]\n", "if not fixtures_path.exists():\n", " print(f\"(no fixtures.json yet at {fixtures_path}; skipping fixture run)\")\n", "else:\n", diff --git a/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/config.json b/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/config.json index e787e2ec..4b6b2478 100644 --- a/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/config.json +++ b/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/config.json @@ -1,9 +1,10 @@ { - "spec_version": "1.0", + "$schema": "../../../_schemas/config.schema.json", "evaluator": { "id": "feedback.productive_coaching_writing_feedback.is_anchored_in_student_response", "name": "Anchored In Student Response Feedback-Quality Evaluator", - "description": "Is the feedback clearly based on the student's specific response? (Key features generated from rubric text; concept paper did not list explicit key feature items.)" + "description": "Is the feedback clearly based on the student's specific response? (Key features generated from rubric text; concept paper did not list explicit key feature items.)", + "supported_grades": ["8", "9"] }, "input_schema": { "$ref": "input_schema.json" @@ -12,9 +13,9 @@ "steps": [ { "id": "evaluate_is_anchored_in_student_response", + "type": "llm", "description": "Single-call LLM step that produces the binary EvaluatorOutput JSON.", "prompt": { - "type": "chat", "messages": [ { "role": "system", @@ -46,17 +47,14 @@ "temperature": 1 }, "parser": { - "kind": "structured_output", - "output_schema_ref": "#/output_schema", - "notes": "Uses LangChain JsonOutputParser." - }, - "output_binding": "formatted_output" + "kind": "structured_output" + } } ], "output_schema": { "$ref": "output_schema.json" }, "fixtures": { - "sniff_test_path": "fixtures.json" + "path": "fixtures.json" } -} \ No newline at end of file +} diff --git a/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/example_notebook.ipynb b/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/example_notebook.ipynb index 81fb3a8b..36dda059 100644 --- a/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/example_notebook.ipynb +++ b/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/example_notebook.ipynb @@ -231,7 +231,7 @@ "metadata": {}, "outputs": [], "source": [ - "fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"sniff_test_path\"]\n", + "fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"path\"]\n", "if not fixtures_path.exists():\n", " print(f\"(no fixtures.json yet at {fixtures_path}; skipping fixture run)\")\n", "else:\n", diff --git a/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/config.json b/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/config.json index 443ba599..410a168e 100644 --- a/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/config.json +++ b/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/config.json @@ -1,9 +1,10 @@ { - "spec_version": "1.0", + "$schema": "../../../_schemas/config.schema.json", "evaluator": { "id": "feedback.productive_coaching_writing_feedback.is_appropriate_feedback", "name": "Appropriate Feedback Feedback-Quality Evaluator", - "description": "Did the feedback correctly identify whether the student needed to revise? (Key features generated from rubric text; concept paper did not list explicit key feature items.)" + "description": "Did the feedback correctly identify whether the student needed to revise? (Key features generated from rubric text; concept paper did not list explicit key feature items.)", + "supported_grades": ["8", "9"] }, "input_schema": { "$ref": "input_schema.json" @@ -12,9 +13,9 @@ "steps": [ { "id": "evaluate_is_appropriate_feedback", + "type": "llm", "description": "Single-call LLM step that produces the binary EvaluatorOutput JSON.", "prompt": { - "type": "chat", "messages": [ { "role": "system", @@ -46,17 +47,14 @@ "temperature": 1 }, "parser": { - "kind": "structured_output", - "output_schema_ref": "#/output_schema", - "notes": "Uses LangChain JsonOutputParser." - }, - "output_binding": "formatted_output" + "kind": "structured_output" + } } ], "output_schema": { "$ref": "output_schema.json" }, "fixtures": { - "sniff_test_path": "fixtures.json" + "path": "fixtures.json" } } diff --git a/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/example_notebook.ipynb b/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/example_notebook.ipynb index 92c760f0..92066d85 100644 --- a/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/example_notebook.ipynb +++ b/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/example_notebook.ipynb @@ -231,7 +231,7 @@ "metadata": {}, "outputs": [], "source": [ - "fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"sniff_test_path\"]\n", + "fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"path\"]\n", "if not fixtures_path.exists():\n", " print(f\"(no fixtures.json yet at {fixtures_path}; skipping fixture run)\")\n", "else:\n", diff --git a/evals/feedback/productive-coaching-writing-feedback/manageable/config.json b/evals/feedback/productive-coaching-writing-feedback/manageable/config.json index 18b19d86..3e9c5aac 100644 --- a/evals/feedback/productive-coaching-writing-feedback/manageable/config.json +++ b/evals/feedback/productive-coaching-writing-feedback/manageable/config.json @@ -1,9 +1,10 @@ { - "spec_version": "1.0", + "$schema": "../../../_schemas/config.schema.json", "evaluator": { "id": "feedback.productive_coaching_writing_feedback.is_manageable", "name": "Manageable Feedback-Quality Evaluator", - "description": "Is the amount of feedback manageable for the student?" + "description": "Is the amount of feedback manageable for the student?", + "supported_grades": ["8", "9"] }, "input_schema": { "$ref": "input_schema.json" @@ -12,9 +13,9 @@ "steps": [ { "id": "evaluate_is_manageable", + "type": "llm", "description": "Single-call LLM step that produces the binary EvaluatorOutput JSON.", "prompt": { - "type": "chat", "messages": [ { "role": "system", @@ -46,17 +47,14 @@ "temperature": 1 }, "parser": { - "kind": "structured_output", - "output_schema_ref": "#/output_schema", - "notes": "Uses LangChain JsonOutputParser." - }, - "output_binding": "formatted_output" + "kind": "structured_output" + } } ], "output_schema": { "$ref": "output_schema.json" }, "fixtures": { - "sniff_test_path": "fixtures.json" + "path": "fixtures.json" } -} \ No newline at end of file +} diff --git a/evals/feedback/productive-coaching-writing-feedback/manageable/example_notebook.ipynb b/evals/feedback/productive-coaching-writing-feedback/manageable/example_notebook.ipynb index 8324ab35..b5be0a24 100644 --- a/evals/feedback/productive-coaching-writing-feedback/manageable/example_notebook.ipynb +++ b/evals/feedback/productive-coaching-writing-feedback/manageable/example_notebook.ipynb @@ -231,7 +231,7 @@ "metadata": {}, "outputs": [], "source": [ - "fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"sniff_test_path\"]\n", + "fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"path\"]\n", "if not fixtures_path.exists():\n", " print(f\"(no fixtures.json yet at {fixtures_path}; skipping fixture run)\")\n", "else:\n", diff --git a/evals/literacy/qualitative-text-complexity/intertextuality/config.json b/evals/literacy/qualitative-text-complexity/intertextuality/config.json index a9be79aa..6c759fd0 100644 --- a/evals/literacy/qualitative-text-complexity/intertextuality/config.json +++ b/evals/literacy/qualitative-text-complexity/intertextuality/config.json @@ -1,9 +1,10 @@ { - "spec_version": "1.0", + "$schema": "../../../_schemas/config.schema.json", "evaluator": { "id": "literacy.gla.intertextuality", "name": "Intertextuality Dimension Text Complexity Evaluator", - "description": "Evaluates the Intertextuality dimension of qualitative text complexity for grades 3-12 reading assessment, producing a 4-level rubric rating with structured pedagogical detail." + "description": "Evaluates the Intertextuality dimension of qualitative text complexity for grades 3-12 reading assessment, producing a 4-level rubric rating with structured pedagogical detail.", + "supported_grades": ["3", "4", "5", "6", "7", "8", "9", "10", "11", "12"] }, "input_schema": { "$ref": "input_schema.json" @@ -11,6 +12,7 @@ "preprocessing": [ { "id": "fk_score", + "type": "computation", "kind": "flesch_kincaid_grade", "description": "Compute the Flesch-Kincaid Grade Level for the input text and bind it to {fk_score} in the prompt.", "input": "text", @@ -32,9 +34,9 @@ "steps": [ { "id": "evaluate_intertextuality", + "type": "llm", "description": "Single-call LLM step that produces the EvaluatorOutput JSON.", "prompt": { - "type": "chat", "messages": [ { "role": "system", @@ -59,10 +61,6 @@ "fk_score": { "required": true, "source": "preprocessing.fk_score" - }, - "format_instructions": { - "required": true, - "source": "parser.format_instructions" } } }, @@ -75,32 +73,18 @@ "temperature": 0.0 }, "parser": { - "kind": "structured_output", - "format_instructions_placeholder": "format_instructions", - "output_schema_ref": "#/output_schema", - "notes": "Uses LangChain JsonOutputParser-style format_instructions injected into the system prompt." - }, - "output_binding": "formatted_output" + "kind": "structured_output" + } } ], "output_schema": { "$ref": "output_schema.json" }, "fixtures": { - "sniff_test_path": "fixtures.json", + "path": "fixtures.json", "tolerance": { "allow_adjacent_levels": true, "notes": "If allow_adjacent_levels is true, predictions within one rubric step of the expected label count as a pass." } - }, - "tracing": { - "expose": [ - "rendered_prompt", - "raw_output", - "raw_text", - "formatted_output", - "usage" - ], - "notes": "Runtime engines MUST expose these five trace fields per evaluation." } -} \ No newline at end of file +} diff --git a/evals/literacy/qualitative-text-complexity/intertextuality/example_notebook.ipynb b/evals/literacy/qualitative-text-complexity/intertextuality/example_notebook.ipynb index 50f93a14..246e9dcc 100644 --- a/evals/literacy/qualitative-text-complexity/intertextuality/example_notebook.ipynb +++ b/evals/literacy/qualitative-text-complexity/intertextuality/example_notebook.ipynb @@ -257,16 +257,16 @@ "# Sniff-test runner: load fixtures.json and check predictions against expected\n", "# -------------------------------------------------------------------------\n", "# Fixtures live next to config.json + system.txt + user.txt and follow the\n", - "# schema declared in CONFIG['fixtures']['schema']. Each case has:\n", + "# shared fixtures schema (evals/_schemas/fixtures.schema.json). Each case has:\n", "# - id, description (optional)\n", "# - input: {text, grade_level} -- runtime evaluator inputs\n", - "# - expected: {complexity_level} -- ground-truth label from rubric\n", + "# - expected: {complexity_score} -- ground-truth label from rubric\n", "#\n", - "# Note: the fixture key 'complexity_level' maps to the runtime output's\n", + "# Note: the fixture key 'complexity_score' maps to the runtime output's\n", "# 'complexity_score' field. We test that single value only -- the model's\n", "# free-text 'reasoning' field is non-deterministic across runs.\n", "\n", - "fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"sniff_test_path\"]\n", + "fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"path\"]\n", "with open(fixtures_path) as f:\n", " fixtures = json.load(f)\n", "print(f\"Loaded {len(fixtures)} fixtures from {fixtures_path.name}\\n\")\n", @@ -289,7 +289,7 @@ "# Run each fixture, accumulate results\n", "results = []\n", "for fx in fixtures:\n", - " expected = fx[\"expected\"][\"complexity_level\"]\n", + " expected = fx[\"expected\"][\"complexity_score\"]\n", " out = evaluate_text_complexity(\n", " text=fx[\"input\"][\"text\"],\n", " grade_level=fx[\"input\"][\"grade_level\"],\n", diff --git a/evals/literacy/qualitative-text-complexity/intertextuality/fixtures.json b/evals/literacy/qualitative-text-complexity/intertextuality/fixtures.json index 10ce6868..7702bad9 100644 --- a/evals/literacy/qualitative-text-complexity/intertextuality/fixtures.json +++ b/evals/literacy/qualitative-text-complexity/intertextuality/fixtures.json @@ -8,7 +8,7 @@ "grade_level": 3 }, "expected": { - "complexity_level": "slightly_complex" + "complexity_score": "slightly_complex" } }, { @@ -20,7 +20,7 @@ "grade_level": 4 }, "expected": { - "complexity_level": "moderately_complex" + "complexity_score": "moderately_complex" } }, { @@ -32,7 +32,7 @@ "grade_level": 7 }, "expected": { - "complexity_level": "moderately_complex" + "complexity_score": "moderately_complex" } }, { @@ -44,7 +44,7 @@ "grade_level": 10 }, "expected": { - "complexity_level": "exceedingly_complex" + "complexity_score": "exceedingly_complex" } } ] diff --git a/evals/prompts/purpose/config.json b/evals/prompts/purpose/config.json index fd7342da..9df37758 100644 --- a/evals/prompts/purpose/config.json +++ b/evals/prompts/purpose/config.json @@ -1,9 +1,10 @@ { - "spec_version": "1.0", + "$schema": "../../_schemas/config.schema.json", "evaluator": { "id": "literacy.gla.purpose", "name": "Purpose Dimension Text Complexity Evaluator", - "description": "Evaluates the Purpose dimension of qualitative text complexity for K-12 reading assessment, producing a 5-level rubric rating with structured pedagogical detail." + "description": "Evaluates the Purpose dimension of qualitative text complexity for K-12 reading assessment, producing a 5-level rubric rating with structured pedagogical detail.", + "supported_grades": ["3", "4", "5", "6", "7", "8", "9", "10", "11", "12"] }, "input_schema": { "$ref": "input_schema.json" @@ -11,6 +12,7 @@ "preprocessing": [ { "id": "fk_score", + "type": "computation", "kind": "flesch_kincaid_grade", "description": "Compute the Flesch-Kincaid Grade Level for the input text and bind it to {fk_score} in the prompt.", "input": "text", @@ -38,9 +40,9 @@ "steps": [ { "id": "evaluate_purpose", + "type": "llm", "description": "Single-call LLM step that produces the EvaluatorOutput JSON.", "prompt": { - "type": "chat", "messages": [ { "role": "system", @@ -77,60 +79,17 @@ }, "parser": { "kind": "structured_output" - }, - "output_binding": "formatted_output" + } } ], "output_schema": { "$ref": "output_schema.json" }, "fixtures": { - "sniff_test_path": "fixtures.json", + "path": "fixtures.json", "tolerance": { "allow_adjacent_levels": true, "notes": "If allow_adjacent_levels is true, predictions within one rubric step of the expected label count as a pass. Slightly Complex < Moderately Complex < Very Complex < Exceedingly Complex; more_context_needed has no adjacency." - }, - "schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "fixtures_schema.json", - "type": "array", - "items": { - "type": "object", - "required": [ - "id", - "input", - "expected" - ], - "additionalProperties": false, - "properties": { - "id": { - "type": "string" - }, - "description": { - "type": "string" - }, - "source": { - "type": "string", - "description": "Optional provenance label (e.g., the source CSV or dataset). Not required." - }, - "input": { - "$ref": "input_schema.json" - }, - "expected": { - "type": "object", - "required": [ - "complexity_level" - ], - "additionalProperties": false, - "properties": { - "complexity_level": { - "$ref": "output_schema.json#/properties/complexity_score", - "description": "Expected complexity rating for this fixture. Renamed from the runtime field 'complexity_score' for clarity in the test file -- only the level itself is being tested; reasoning is non-deterministic across LLM runs." - } - } - } - } - } } } } diff --git a/evals/prompts/purpose/fixtures.json b/evals/prompts/purpose/fixtures.json index be6826ff..56784a33 100644 --- a/evals/prompts/purpose/fixtures.json +++ b/evals/prompts/purpose/fixtures.json @@ -8,7 +8,7 @@ "grade_level": 3 }, "expected": { - "complexity_level": "slightly_complex" + "complexity_score": "slightly_complex" } }, { @@ -20,7 +20,7 @@ "grade_level": 9 }, "expected": { - "complexity_level": "slightly_complex" + "complexity_score": "slightly_complex" } }, { @@ -32,7 +32,7 @@ "grade_level": 11 }, "expected": { - "complexity_level": "moderately_complex" + "complexity_score": "moderately_complex" } }, { @@ -44,7 +44,7 @@ "grade_level": 9 }, "expected": { - "complexity_level": "moderately_complex" + "complexity_score": "moderately_complex" } }, { @@ -56,7 +56,7 @@ "grade_level": 6 }, "expected": { - "complexity_level": "moderately_complex" + "complexity_score": "moderately_complex" } } ] diff --git a/evals/purpose_evaluator.ipynb b/evals/purpose_evaluator.ipynb index 1568794e..40a2e0cc 100644 --- a/evals/purpose_evaluator.ipynb +++ b/evals/purpose_evaluator.ipynb @@ -87,7 +87,7 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": "# -------------------------------------------------------------------------\n# Sniff-test runner: load fixtures.json and check predictions against expected\n# -------------------------------------------------------------------------\n# Fixtures live next to config.json + system.txt + user.txt and follow the\n# schema declared in CONFIG['fixtures']['schema']. Each case has:\n# - id, description (optional)\n# - input: {text, grade_level} -- runtime evaluator inputs\n# - expected: {complexity_level} -- ground-truth label from rubric\n#\n# Note: the fixture key 'complexity_level' maps to the runtime output's\n# 'complexity_score' field. We test that single value only -- the model's\n# free-text 'reasoning' field is non-deterministic across runs.\n\nfixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"sniff_test_path\"]\nwith open(fixtures_path) as f:\n fixtures = json.load(f)\nprint(f\"Loaded {len(fixtures)} fixtures from {fixtures_path.name}\\n\")\n\n# Adjacency tolerance per CONFIG['fixtures']['tolerance'].\n# Derive rubric order from OUTPUT_SCHEMA -- 'more_context_needed' excluded as it has no adjacency.\n_RUBRIC_ORDER = [\n level for level in OUTPUT_SCHEMA[\"properties\"][\"complexity_score\"][\"enum\"]\n if level != \"more_context_needed\"\n]\n_ALLOW_ADJ = bool(CONFIG[\"fixtures\"][\"tolerance\"].get(\"allow_adjacent_levels\", False))\n\ndef _score_outcome(predicted: str, expected: str):\n \"\"\"Return ('exact' | 'adjacent' | 'fail', distance_or_None).\"\"\"\n if predicted == expected:\n return \"exact\", 0\n if _ALLOW_ADJ and predicted in _RUBRIC_ORDER and expected in _RUBRIC_ORDER:\n d = abs(_RUBRIC_ORDER.index(predicted) - _RUBRIC_ORDER.index(expected))\n if d == 1:\n return \"adjacent\", d\n return \"fail\", None\n\n# Run each fixture, accumulate results\nresults = []\nfor fx in fixtures:\n expected = fx[\"expected\"][\"complexity_level\"]\n out = evaluate_text_complexity(\n text=fx[\"input\"][\"text\"],\n grade_level=fx[\"input\"][\"grade_level\"],\n )\n if isinstance(out, str): # error path\n results.append({\"id\": fx[\"id\"], \"status\": \"error\", \"predicted\": None, \"expected\": expected, \"error\": out})\n continue\n predicted = out[\"formatted_output\"][\"complexity_score\"]\n status, _ = _score_outcome(predicted, expected)\n results.append({\n \"id\": fx[\"id\"], \"status\": status,\n \"predicted\": predicted, \"expected\": expected,\n \"description\": fx.get(\"description\", \"\"),\n })\n\n# Per-case report\nprint(\"\\n\" + \"=\" * 78)\nprint(f\"{'ID':>5} {'STATUS':<8} {'PREDICTED':<22} {'EXPECTED':<22} DESCRIPTION\")\nprint(\"=\" * 78)\nfor r in results:\n icon = {\"exact\": \"PASS\", \"adjacent\": \"PASS*\", \"fail\": \"FAIL\", \"error\": \"ERR\"}[r[\"status\"]]\n print(f\"{r['id']:>5} {icon:<8} {(r['predicted'] or '-'):<22} {r['expected']:<22} {r.get('description','')[:25]}\")\n\n# Summary\nn_total = len(results)\nn_exact = sum(1 for r in results if r[\"status\"] == \"exact\")\nn_adj = sum(1 for r in results if r[\"status\"] == \"adjacent\")\nn_fail = sum(1 for r in results if r[\"status\"] == \"fail\")\nn_err = sum(1 for r in results if r[\"status\"] == \"error\")\nprint(\"=\" * 78)\nprint(f\"Summary: {n_exact} exact, {n_adj} adjacent (tolerated), {n_fail} fail, {n_err} error -- total {n_total}\")\nif _ALLOW_ADJ:\n print(\"(Adjacency tolerance ON: predictions within +/-1 rubric step of the expected label count as PASS*.)\")" + "source": "# -------------------------------------------------------------------------\n# Sniff-test runner: load fixtures.json and check predictions against expected\n# -------------------------------------------------------------------------\n# Fixtures live next to config.json + system.txt + user.txt and follow the\n# shared fixtures schema (evals/_schemas/fixtures.schema.json). Each case has:\n# - id, description (optional)\n# - input: {text, grade_level} -- runtime evaluator inputs\n# - expected: {complexity_score} -- ground-truth label from rubric\n#\n# Note: the fixture key 'complexity_score' maps to the runtime output's\n# 'complexity_score' field. We test that single value only -- the model's\n# free-text 'reasoning' field is non-deterministic across runs.\n\nfixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"path\"]\nwith open(fixtures_path) as f:\n fixtures = json.load(f)\nprint(f\"Loaded {len(fixtures)} fixtures from {fixtures_path.name}\\n\")\n\n# Adjacency tolerance per CONFIG['fixtures']['tolerance'].\n# Derive rubric order from OUTPUT_SCHEMA -- 'more_context_needed' excluded as it has no adjacency.\n_RUBRIC_ORDER = [\n level for level in OUTPUT_SCHEMA[\"properties\"][\"complexity_score\"][\"enum\"]\n if level != \"more_context_needed\"\n]\n_ALLOW_ADJ = bool(CONFIG[\"fixtures\"][\"tolerance\"].get(\"allow_adjacent_levels\", False))\n\ndef _score_outcome(predicted: str, expected: str):\n \"\"\"Return ('exact' | 'adjacent' | 'fail', distance_or_None).\"\"\"\n if predicted == expected:\n return \"exact\", 0\n if _ALLOW_ADJ and predicted in _RUBRIC_ORDER and expected in _RUBRIC_ORDER:\n d = abs(_RUBRIC_ORDER.index(predicted) - _RUBRIC_ORDER.index(expected))\n if d == 1:\n return \"adjacent\", d\n return \"fail\", None\n\n# Run each fixture, accumulate results\nresults = []\nfor fx in fixtures:\n expected = fx[\"expected\"][\"complexity_score\"]\n out = evaluate_text_complexity(\n text=fx[\"input\"][\"text\"],\n grade_level=fx[\"input\"][\"grade_level\"],\n )\n if isinstance(out, str): # error path\n results.append({\"id\": fx[\"id\"], \"status\": \"error\", \"predicted\": None, \"expected\": expected, \"error\": out})\n continue\n predicted = out[\"formatted_output\"][\"complexity_score\"]\n status, _ = _score_outcome(predicted, expected)\n results.append({\n \"id\": fx[\"id\"], \"status\": status,\n \"predicted\": predicted, \"expected\": expected,\n \"description\": fx.get(\"description\", \"\"),\n })\n\n# Per-case report\nprint(\"\\n\" + \"=\" * 78)\nprint(f\"{'ID':>5} {'STATUS':<8} {'PREDICTED':<22} {'EXPECTED':<22} DESCRIPTION\")\nprint(\"=\" * 78)\nfor r in results:\n icon = {\"exact\": \"PASS\", \"adjacent\": \"PASS*\", \"fail\": \"FAIL\", \"error\": \"ERR\"}[r[\"status\"]]\n print(f\"{r['id']:>5} {icon:<8} {(r['predicted'] or '-'):<22} {r['expected']:<22} {r.get('description','')[:25]}\")\n\n# Summary\nn_total = len(results)\nn_exact = sum(1 for r in results if r[\"status\"] == \"exact\")\nn_adj = sum(1 for r in results if r[\"status\"] == \"adjacent\")\nn_fail = sum(1 for r in results if r[\"status\"] == \"fail\")\nn_err = sum(1 for r in results if r[\"status\"] == \"error\")\nprint(\"=\" * 78)\nprint(f\"Summary: {n_exact} exact, {n_adj} adjacent (tolerated), {n_fail} fail, {n_err} error -- total {n_total}\")\nif _ALLOW_ADJ:\n print(\"(Adjacency tolerance ON: predictions within +/-1 rubric step of the expected label count as PASS*.)\")" }, { "cell_type": "code", diff --git a/evals/standards/math-question-alignment/config.json b/evals/standards/math-question-alignment/config.json index 4e930908..1470d147 100644 --- a/evals/standards/math-question-alignment/config.json +++ b/evals/standards/math-question-alignment/config.json @@ -1,5 +1,5 @@ { - "spec_version": "1.0", + "$schema": "../../_schemas/config.schema.json", "evaluator": { "id": "math.standards-alignment", "name": "Math Standards Alignment", @@ -12,6 +12,7 @@ "preprocessing": [ { "id": "get_standard_uuid", + "type": "api", "kind": "http_get", "description": "Resolve the statementCode to its caseIdentifierUUID via the Knowledge Graph.", "endpoint": "https://api.learningcommons.org/knowledge-graph/v0/academic-standards/search", @@ -24,6 +25,7 @@ }, { "id": "get_learning_components", + "type": "api", "kind": "http_get", "description": "Fetch all learning components for the standard using the caseIdentifierUUID from get_standard_uuid.", "endpoint": "https://api.learningcommons.org/knowledge-graph/v0/academic-standards/{caseIdentifierUUID}/learning-components", @@ -38,9 +40,9 @@ "steps": [ { "id": "evaluate_standards_alignment", + "type": "llm", "description": "Single LLM call that evaluates the question against all learning components for the standard at once, returning one result per LC.", "prompt": { - "type": "chat", "messages": [ { "role": "system", @@ -66,8 +68,7 @@ }, "parser": { "kind": "structured_output" - }, - "output_binding": "formatted_output" + } } ], "output_schema": { diff --git a/scripts/README.md b/scripts/README.md index d3f120ef..bc8c17a9 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -36,14 +36,19 @@ fails the PR (it never auto-fixes) — so fixing locally saves a round-trip. | Check | What it does | |-------|--------------| | `strip-notebooks` | Clears outputs, execution counts, and cell metadata from `.ipynb` files | +| `eval-config` | Validates each evaluator `config.json` against the shared schema (`evals/_schemas/config.schema.json`, referenced by the config's own `$schema`), plus cross-file rules: referenced files exist, prompt `sha256` matches, placeholders ⇄ template `{vars}` are in sync, system prompts carry no placeholders, and no obsolete `format_instructions` survive | ## Coming next This is the seed of a broader self-service harness. Planned additions: -- **eval-config validation** — parser kind, prompt `sha256` drift, no dangling - placeholders, fixture labels within the output schema enum. -- **naming / structure conventions** for `evals/`. +- **`eval-schemas`** — meta-validate each `input_schema.json` / `output_schema.json` + is a well-formed JSON Schema document. +- **`eval-fixtures`** — validate `fixtures.json` against the shared + `evals/_schemas/fixtures.schema.json`, and bind each case's `input`/`expected` + to that evaluator's own input/output schema. +- **`eval-notebook`** — where an evaluator ships an example notebook, confirm it + loads the config + prompt files from disk rather than hardcoding them. - **delegation to the SDK suites** (`sdks/typescript` lint/type/test, `sdks/python` Makefile) so one local command covers everything. CI keeps the per-SDK workflows separate for parallelism and path-filtered signal. diff --git a/scripts/checks/__init__.py b/scripts/checks/__init__.py index 927c2a8c..521f657c 100644 --- a/scripts/checks/__init__.py +++ b/scripts/checks/__init__.py @@ -1,7 +1,9 @@ """Check registry. Add new checks here to wire them into the harness + CI.""" +from .eval_config import EvalConfig from .strip_notebooks import StripNotebooks ALL_CHECKS = [ StripNotebooks(), + EvalConfig(), ] diff --git a/scripts/checks/base.py b/scripts/checks/base.py index dfb82a42..22e78a67 100644 --- a/scripts/checks/base.py +++ b/scripts/checks/base.py @@ -8,9 +8,14 @@ from __future__ import annotations +import json +import os import subprocess from dataclasses import dataclass, field +# Shared, version-pinned schemas live here (not duplicated per evaluator). +SCHEMA_ROOT = "evals/_schemas" + @dataclass class Violation: @@ -48,3 +53,18 @@ def tracked_files(pattern: str) -> list[str]: """Git-tracked paths matching `pattern`, excluding checkpoint dirs.""" out = subprocess.check_output(["git", "ls-files", pattern], text=True) return [p for p in out.splitlines() if ".ipynb_checkpoints" not in p] + + +def evaluator_configs() -> list[str]: + """Every evaluator's config.json (an evaluator = a dir under evals/ with one).""" + return tracked_files("evals/**/config.json") + + +def shared_schema_path(spec_version: str, name: str) -> str: + """Path to a version-pinned shared schema, e.g. evals/_schemas/1.0/.""" + return os.path.join(SCHEMA_ROOT, spec_version, name) + + +def load_json(path: str): + with open(path, encoding="utf-8") as f: + return json.load(f) diff --git a/scripts/checks/eval_config.py b/scripts/checks/eval_config.py new file mode 100644 index 00000000..88b76d81 --- /dev/null +++ b/scripts/checks/eval_config.py @@ -0,0 +1,179 @@ +"""Validate evaluator config.json files. + +Each config is validated in two complementary layers: + + 1. Schema — structural rules a JSON Schema can express (required fields, + enums, types). Every config points at the shared contract via its own + `$schema` field; we load that and report every schema error. + + 2. Cross-file — rules a schema cannot express because they span files: + - referenced files actually exist ($ref schemas, prompt files, fixtures) + - each declared sha256 matches the prompt file on disk (drift tripwire) + - placeholders declared in config line up with the {vars} in the prompts + - system prompts carry no user-input placeholders + - no obsolete format-instruction placeholders survive anywhere + +Every rule reports *all* offenders (never stops at the first) so one run lists +everything to fix. Check-only: config intent isn't safely auto-fixable — notably +sha256, which is a deliberate drift tripwire, not something to silently rewrite. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re + +from jsonschema import Draft202012Validator + +from .base import Check, Result, Violation, evaluator_configs, load_json + +# {placeholder} tokens in a prompt template. +_PLACEHOLDER = re.compile(r"\{([a-zA-Z0-9_]+)\}") + +# Placeholder names that should never appear: structured_output makes +# LangChain-era format-instruction injection obsolete. Substring match so +# variants ("format_instructions", "json_format_instructions", …) are caught. +_OBSOLETE_PLACEHOLDERS = ("format_instructions",) + + +class EvalConfig(Check): + name = "eval-config" + description = "Validate config.json against the shared schema + cross-file refs/sha/placeholders" + + def run(self, fix: bool) -> Result: + result = Result(self.name) + for cfg_path in evaluator_configs(): + self._check_config(cfg_path, result) + return result + + def _check_config(self, cfg_path: str, result: Result) -> None: + """Run every rule against one config, collecting violations.""" + base = os.path.dirname(cfg_path) + + def fail(msg: str) -> None: + result.violations.append(Violation(cfg_path, msg)) + + try: + config = load_json(cfg_path) + except (OSError, json.JSONDecodeError) as e: + fail(f"invalid JSON: {e}") + return + + self._check_schema(config, base, fail) + self._check_referenced_files(config, base, fail) + template_vars = self._check_prompts(config, base, fail) + self._check_placeholders(config, template_vars, fail) + self._check_fixtures_path(config, base, fail) + + # --- Layer 1: schema ----------------------------------------------------- + + def _check_schema(self, config: dict, base: str, fail) -> None: + """Validate the config against the schema named in its own `$schema`.""" + ref = config.get("$schema") + if not ref: + fail("missing $schema reference") + return + schema_path = os.path.normpath(os.path.join(base, ref)) + if not os.path.exists(schema_path): + fail(f"$schema not found: {ref}") + return + try: + schema = load_json(schema_path) + except (OSError, json.JSONDecodeError) as e: + fail(f"$schema unreadable ({ref}): {e}") + return + validator = Draft202012Validator(schema) + for err in sorted(validator.iter_errors(config), key=lambda e: list(e.path)): + loc = "/".join(str(p) for p in err.path) or "(root)" + fail(f"schema: {loc}: {err.message}") + + # --- Layer 2: cross-file ------------------------------------------------- + + def _check_referenced_files(self, config: dict, base: str, fail) -> None: + """The schema $refs for input/output must resolve to real files.""" + for key in ("input_schema", "output_schema"): + ref = config.get(key, {}) + if isinstance(ref, dict) and "$ref" in ref: + if not os.path.exists(os.path.join(base, ref["$ref"])): + fail(f"{key} $ref not found: {ref['$ref']}") + + def _check_prompts(self, config: dict, base: str, fail) -> set[str]: + """Validate each prompt file; return the set of {vars} used across them. + + Per message: the file exists, its sha256 matches, system prompts hold no + placeholders, and no obsolete placeholders appear. + """ + template_vars: set[str] = set() + for msg in self._messages(config): + src = msg.get("source_path") + if not src: + continue + path = os.path.join(base, src) + if not os.path.exists(path): + fail(f"prompt file not found: {src}") + continue + + try: + with open(path, "rb") as f: + raw = f.read() + except OSError as e: + fail(f"prompt file unreadable: {src}: {e}") + continue + + used = set(_PLACEHOLDER.findall(raw.decode("utf-8", "replace"))) + template_vars |= used + + self._check_sha(msg, src, raw, fail) + self._check_role_placeholders(msg, src, used, fail) + return template_vars + + @staticmethod + def _check_sha(msg: dict, src: str, raw: bytes, fail) -> None: + """Declared sha256 must match the file — guards against silent drift. + + Hash the raw bytes (not decoded text) so the result is identical across + platforms — text-mode reads translate newlines and would skew the hash. + """ + declared = msg.get("sha256") + if not declared: + return + actual = hashlib.sha256(raw).hexdigest() + if actual != declared: + fail(f"sha256 drift for {src}: declared {declared[:12]}…, actual {actual[:12]}…") + + @staticmethod + def _check_role_placeholders(msg: dict, src: str, used: set[str], fail) -> None: + """System prompts take no inputs; no prompt may inject format instructions.""" + # All runtime inputs belong in user-role prompts, never the system prompt. + if msg.get("role") == "system" and used: + fail(f"system prompt {src} contains placeholders {sorted(used)}; " + "user inputs belong only in user-role prompts") + for var in sorted(used): + if any(token in var for token in _OBSOLETE_PLACEHOLDERS): + fail(f'{src} references "{{{var}}}"; structured_output makes ' + "format-instruction placeholders obsolete") + + def _check_placeholders(self, config: dict, template_vars: set[str], fail) -> None: + """Declared placeholders and template {vars} must be exactly in sync.""" + declared = set(self._prompt(config).get("placeholders", {}).keys()) + for ph in sorted(declared - template_vars): + fail(f'placeholder "{ph}" declared but not used in any prompt template') + for var in sorted(template_vars - declared): + fail(f'template variable "{{{var}}}" used but not declared in placeholders') + + def _check_fixtures_path(self, config: dict, base: str, fail) -> None: + """Fixtures file must exist (its contents are validated by eval-fixtures).""" + path = config.get("fixtures", {}).get("path") + if path and not os.path.exists(os.path.join(base, path)): + fail(f"fixtures file not found: {path}") + + # --- helpers ------------------------------------------------------------- + + @staticmethod + def _prompt(config: dict) -> dict: + return (config.get("steps") or [{}])[0].get("prompt", {}) + + def _messages(self, config: dict) -> list[dict]: + return self._prompt(config).get("messages", []) diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 725bc3d1..c3ea42c1 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -1,3 +1,4 @@ # Dependencies for the repo check harness (scripts/check.py). -# Pinned so local runs and CI strip notebooks identically. +# Pinned so local runs and CI behave identically. nbstripout==0.9.1 +jsonschema==4.23.0