Skip to content

Commit 1b2378a

Browse files
adnanrhussainadnanrhussain
committed
feat: schema-driven eval-config validation with shared config schema
1 parent 9ce2973 commit 1b2378a

25 files changed

Lines changed: 462 additions & 265 deletions

File tree

‎evals/_schemas/config.schema.json‎

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
{
2+
"$schema": "https://json-schema.org/draft/2020-12/schema",
3+
"$id": "config.schema.json",
4+
"title": "Evaluator config",
5+
"description": "Shared contract for evals/**/config.json. One living schema, referenced by each config via its $schema field. Strict: additionalProperties is false at every level, so any new field must be added here deliberately.",
6+
"type": "object",
7+
"additionalProperties": false,
8+
"required": ["$schema", "evaluator", "input_schema", "steps", "output_schema"],
9+
"properties": {
10+
"$schema": { "type": "string" },
11+
"evaluator": {
12+
"type": "object",
13+
"additionalProperties": false,
14+
"required": ["id", "name", "description", "supported_grades"],
15+
"properties": {
16+
"id": { "type": "string", "minLength": 1 },
17+
"name": { "type": "string", "minLength": 1 },
18+
"description": { "type": "string", "minLength": 1 },
19+
"supported_grades": {
20+
"type": "array",
21+
"minItems": 1,
22+
"items": {
23+
"enum": ["K", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"]
24+
}
25+
}
26+
}
27+
},
28+
"input_schema": { "type": "object" },
29+
"output_schema": { "type": "object" },
30+
"preprocessing": {
31+
"type": "array",
32+
"items": {
33+
"type": "object",
34+
"additionalProperties": false,
35+
"required": ["id", "type", "kind"],
36+
"properties": {
37+
"id": { "type": "string", "minLength": 1 },
38+
"type": { "enum": ["api", "computation"] },
39+
"kind": { "type": "string", "minLength": 1 },
40+
"description": { "type": "string" },
41+
"input": { "type": "string" },
42+
"output": { "type": "string" },
43+
"implementation": {
44+
"type": "object",
45+
"minProperties": 1,
46+
"additionalProperties": {
47+
"type": "object",
48+
"additionalProperties": false,
49+
"required": ["library", "function"],
50+
"properties": {
51+
"library": { "type": "string", "minLength": 1 },
52+
"function": { "type": "string", "minLength": 1 },
53+
"post_transform": {
54+
"type": "object",
55+
"additionalProperties": false,
56+
"required": ["type"],
57+
"properties": {
58+
"type": { "enum": ["round"] },
59+
"precision": { "type": "integer" }
60+
}
61+
}
62+
}
63+
}
64+
},
65+
"endpoint": { "type": "string" },
66+
"params": { "type": "object" },
67+
"auth": { "type": "string" },
68+
"pagination": { "type": "string" }
69+
},
70+
"allOf": [
71+
{
72+
"if": { "properties": { "type": { "const": "api" } } },
73+
"then": { "required": ["endpoint", "params", "auth"] }
74+
},
75+
{
76+
"if": { "properties": { "type": { "const": "computation" } } },
77+
"then": { "required": ["implementation", "input", "output"] }
78+
}
79+
]
80+
}
81+
},
82+
"steps": {
83+
"type": "array",
84+
"minItems": 1,
85+
"items": {
86+
"type": "object",
87+
"additionalProperties": false,
88+
"required": ["id", "type"],
89+
"properties": {
90+
"id": { "type": "string", "minLength": 1 },
91+
"type": { "enum": ["llm", "api"] },
92+
"description": { "type": "string" },
93+
"prompt": {
94+
"type": "object",
95+
"additionalProperties": false,
96+
"required": ["messages", "placeholders"],
97+
"properties": {
98+
"messages": {
99+
"type": "array",
100+
"minItems": 1,
101+
"items": {
102+
"type": "object",
103+
"additionalProperties": false,
104+
"required": ["role", "source_path"],
105+
"properties": {
106+
"role": { "enum": ["system", "user", "assistant"] },
107+
"source_path": { "type": "string", "minLength": 1 },
108+
"sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" }
109+
}
110+
}
111+
},
112+
"placeholders": {
113+
"type": "object",
114+
"additionalProperties": {
115+
"type": "object",
116+
"additionalProperties": false,
117+
"required": ["required", "source"],
118+
"properties": {
119+
"required": { "type": "boolean" },
120+
"source": { "type": "string" },
121+
"note": { "type": "string" }
122+
}
123+
}
124+
}
125+
}
126+
},
127+
"model": {
128+
"type": "object",
129+
"additionalProperties": false,
130+
"required": ["provider", "name"],
131+
"properties": {
132+
"provider": { "enum": ["google", "openai", "anthropic"] },
133+
"name": { "type": "string", "minLength": 1 },
134+
"alias": { "type": "string" }
135+
}
136+
},
137+
"generation": {
138+
"type": "object",
139+
"additionalProperties": false,
140+
"properties": {
141+
"temperature": { "type": "number", "minimum": 0, "maximum": 1 }
142+
}
143+
},
144+
"parser": {
145+
"type": "object",
146+
"additionalProperties": false,
147+
"required": ["kind"],
148+
"properties": {
149+
"kind": { "const": "structured_output" }
150+
}
151+
}
152+
},
153+
"allOf": [
154+
{
155+
"if": { "properties": { "type": { "const": "llm" } } },
156+
"then": { "required": ["prompt", "model", "parser"] }
157+
}
158+
]
159+
}
160+
},
161+
"fixtures": {
162+
"type": "object",
163+
"additionalProperties": false,
164+
"properties": {
165+
"path": { "type": "string" },
166+
"tolerance": { "type": "object" }
167+
}
168+
}
169+
}
170+
}

‎evals/_schemas/fixtures.schema.json‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
{
2+
"$schema": "https://json-schema.org/draft/2020-12/schema",
3+
"$id": "fixtures.schema.json",
4+
"title": "Evaluator fixtures",
5+
"description": "Shared contract for evals/**/fixtures.json. Each case carries inputs and the expected label(s); per-evaluator input/expected shapes are validated against the evaluator's own input_schema/output_schema by the eval-fixtures check, not here.",
6+
"type": "array",
7+
"items": {
8+
"type": "object",
9+
"additionalProperties": false,
10+
"required": ["id", "input", "expected"],
11+
"properties": {
12+
"id": { "type": "string", "minLength": 1 },
13+
"description": { "type": "string" },
14+
"source": { "type": "string" },
15+
"input": { "type": "object" },
16+
"expected": { "type": "object" }
17+
}
18+
}
19+
}

‎evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/config.json‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
{
2-
"spec_version": "1.0",
2+
"$schema": "../../../_schemas/config.schema.json",
33
"evaluator": {
44
"id": "feedback.productive_coaching_writing_feedback.is_acknowledges_strength",
55
"name": "Acknowledges Strength Feedback-Quality Evaluator",
6-
"description": "Does the feedback identify what the student did well?"
6+
"description": "Does the feedback identify what the student did well?",
7+
"supported_grades": ["8", "9"]
78
},
89
"input_schema": {
910
"$ref": "input_schema.json"
@@ -12,9 +13,9 @@
1213
"steps": [
1314
{
1415
"id": "evaluate_is_acknowledges_strength",
16+
"type": "llm",
1517
"description": "Single-call LLM step that produces the binary EvaluatorOutput JSON.",
1618
"prompt": {
17-
"type": "chat",
1819
"messages": [
1920
{
2021
"role": "system",
@@ -46,17 +47,14 @@
4647
"temperature": 1
4748
},
4849
"parser": {
49-
"kind": "structured_output",
50-
"output_schema_ref": "#/output_schema",
51-
"notes": "Uses LangChain JsonOutputParser."
52-
},
53-
"output_binding": "formatted_output"
50+
"kind": "structured_output"
51+
}
5452
}
5553
],
5654
"output_schema": {
5755
"$ref": "output_schema.json"
5856
},
5957
"fixtures": {
60-
"sniff_test_path": "fixtures.json"
58+
"path": "fixtures.json"
6159
}
6260
}

‎evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/example_notebook.ipynb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -231,7 +231,7 @@
231231
"metadata": {},
232232
"outputs": [],
233233
"source": [
234-
"fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"sniff_test_path\"]\n",
234+
"fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"path\"]\n",
235235
"if not fixtures_path.exists():\n",
236236
" print(f\"(no fixtures.json yet at {fixtures_path}; skipping fixture run)\")\n",
237237
"else:\n",

‎evals/feedback/productive-coaching-writing-feedback/actionable-revision/config.json‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
{
2-
"spec_version": "1.0",
2+
"$schema": "../../../_schemas/config.schema.json",
33
"evaluator": {
44
"id": "feedback.productive_coaching_writing_feedback.is_actionable_revision",
55
"name": "Actionable Revision Feedback-Quality Evaluator",
6-
"description": "If revision is needed, does the feedback give a clear next step?"
6+
"description": "If revision is needed, does the feedback give a clear next step?",
7+
"supported_grades": ["8", "9"]
78
},
89
"input_schema": {
910
"$ref": "input_schema.json"
@@ -12,9 +13,9 @@
1213
"steps": [
1314
{
1415
"id": "evaluate_is_actionable_revision",
16+
"type": "llm",
1517
"description": "Single-call LLM step that produces the binary EvaluatorOutput JSON.",
1618
"prompt": {
17-
"type": "chat",
1819
"messages": [
1920
{
2021
"role": "system",
@@ -46,17 +47,14 @@
4647
"temperature": 1
4748
},
4849
"parser": {
49-
"kind": "structured_output",
50-
"output_schema_ref": "#/output_schema",
51-
"notes": "Uses LangChain JsonOutputParser."
52-
},
53-
"output_binding": "formatted_output"
50+
"kind": "structured_output"
51+
}
5452
}
5553
],
5654
"output_schema": {
5755
"$ref": "output_schema.json"
5856
},
5957
"fixtures": {
60-
"sniff_test_path": "fixtures.json"
58+
"path": "fixtures.json"
6159
}
6260
}

‎evals/feedback/productive-coaching-writing-feedback/actionable-revision/example_notebook.ipynb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -231,7 +231,7 @@
231231
"metadata": {},
232232
"outputs": [],
233233
"source": [
234-
"fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"sniff_test_path\"]\n",
234+
"fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"path\"]\n",
235235
"if not fixtures_path.exists():\n",
236236
" print(f\"(no fixtures.json yet at {fixtures_path}; skipping fixture run)\")\n",
237237
"else:\n",

‎evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/config.json‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
{
2-
"spec_version": "1.0",
2+
"$schema": "../../../_schemas/config.schema.json",
33
"evaluator": {
44
"id": "feedback.productive_coaching_writing_feedback.is_anchored_in_student_response",
55
"name": "Anchored In Student Response Feedback-Quality Evaluator",
6-
"description": "Is the feedback clearly based on the student's specific response? (Key features generated from rubric text; concept paper did not list explicit key feature items.)"
6+
"description": "Is the feedback clearly based on the student's specific response? (Key features generated from rubric text; concept paper did not list explicit key feature items.)",
7+
"supported_grades": ["8", "9"]
78
},
89
"input_schema": {
910
"$ref": "input_schema.json"
@@ -12,9 +13,9 @@
1213
"steps": [
1314
{
1415
"id": "evaluate_is_anchored_in_student_response",
16+
"type": "llm",
1517
"description": "Single-call LLM step that produces the binary EvaluatorOutput JSON.",
1618
"prompt": {
17-
"type": "chat",
1819
"messages": [
1920
{
2021
"role": "system",
@@ -46,17 +47,14 @@
4647
"temperature": 1
4748
},
4849
"parser": {
49-
"kind": "structured_output",
50-
"output_schema_ref": "#/output_schema",
51-
"notes": "Uses LangChain JsonOutputParser."
52-
},
53-
"output_binding": "formatted_output"
50+
"kind": "structured_output"
51+
}
5452
}
5553
],
5654
"output_schema": {
5755
"$ref": "output_schema.json"
5856
},
5957
"fixtures": {
60-
"sniff_test_path": "fixtures.json"
58+
"path": "fixtures.json"
6159
}
62-
}
60+
}

‎evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/example_notebook.ipynb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -231,7 +231,7 @@
231231
"metadata": {},
232232
"outputs": [],
233233
"source": [
234-
"fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"sniff_test_path\"]\n",
234+
"fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"path\"]\n",
235235
"if not fixtures_path.exists():\n",
236236
" print(f\"(no fixtures.json yet at {fixtures_path}; skipping fixture run)\")\n",
237237
"else:\n",

‎evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/config.json‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
{
2-
"spec_version": "1.0",
2+
"$schema": "../../../_schemas/config.schema.json",
33
"evaluator": {
44
"id": "feedback.productive_coaching_writing_feedback.is_appropriate_feedback",
55
"name": "Appropriate Feedback Feedback-Quality Evaluator",
6-
"description": "Did the feedback correctly identify whether the student needed to revise? (Key features generated from rubric text; concept paper did not list explicit key feature items.)"
6+
"description": "Did the feedback correctly identify whether the student needed to revise? (Key features generated from rubric text; concept paper did not list explicit key feature items.)",
7+
"supported_grades": ["8", "9"]
78
},
89
"input_schema": {
910
"$ref": "input_schema.json"
@@ -12,9 +13,9 @@
1213
"steps": [
1314
{
1415
"id": "evaluate_is_appropriate_feedback",
16+
"type": "llm",
1517
"description": "Single-call LLM step that produces the binary EvaluatorOutput JSON.",
1618
"prompt": {
17-
"type": "chat",
1819
"messages": [
1920
{
2021
"role": "system",
@@ -46,17 +47,14 @@
4647
"temperature": 1
4748
},
4849
"parser": {
49-
"kind": "structured_output",
50-
"output_schema_ref": "#/output_schema",
51-
"notes": "Uses LangChain JsonOutputParser."
52-
},
53-
"output_binding": "formatted_output"
50+
"kind": "structured_output"
51+
}
5452
}
5553
],
5654
"output_schema": {
5755
"$ref": "output_schema.json"
5856
},
5957
"fixtures": {
60-
"sniff_test_path": "fixtures.json"
58+
"path": "fixtures.json"
6159
}
6260
}

0 commit comments

Comments
 (0)