Folder reorganization, Intertextuality and Quill evals

evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/prompts/config.json

@@ -0,0 +1,248 @@
+{
+  "spec_version": "1.0",
+  "evaluator": {
+    "id": "acknowledges_strength",
+    "version": "0.1",
+    "name": "Acknowledges Strength Feedback-Quality Evaluator",
+    "description": "Does the feedback identify what the student did well?",
+    "dimension": "acknowledges_strength",
+    "output_type": "binary",
+    "labels": [
+      0,
+      1
+    ],
+    "optimization": {
+      "method": "GEPA",
+      "framework": "DSPy",
+      "base_model": "gpt-5.4",
+      "prompt_source": "outputs/gepa_prompt_gpt_5_4.txt",
+      "source_notebook": "build_evaluator.ipynb",
+      "optimized_at": "2026-06-18"
+    },
+    "owners": [
+      "ariena.chi@chanzuckerberg.com"
+    ]
+  },
+  "input_schema": {
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "title": "AcknowledgesStrengthEvaluatorInput",
+    "type": "object",
+    "required": [
+      "student_text",
+      "feedback_text"
+    ],
+    "additionalProperties": false,
+    "properties": {
+      "student_text": {
+        "type": "string",
+        "minLength": 1,
+        "description": "The student's written response."
+      },
+      "feedback_text": {
+        "type": "string",
+        "minLength": 1,
+        "description": "The teacher feedback on that response."
+      }
+    }
+  },
+  "preprocessing": [],
+  "steps": [
+    {
+      "id": "evaluate_is_acknowledges_strength",
+      "description": "Single-call LLM step that produces the binary EvaluatorOutput JSON.",
+      "prompt": {
+        "type": "chat",
+        "messages": [
+          {
+            "role": "system",
+            "source_path": "system.txt",
+            "sha256": "d66c26d4780f0df12b5099b117d10a5b5c518dd4505c41a15043492b319437e9"
+          },
+          {
+            "role": "human",
+            "source_path": "user.txt",
+            "sha256": "eaac7e6eadcc43716e4a98b9e7f34c493c1cbe46adb38367dd8697b904d11bfa"
+          }
+        ],
+        "placeholders": {
+          "student_text": {
+            "required": true,
+            "source": "input"
+          },
+          "feedback_text": {
+            "required": true,
+            "source": "input"
+          },
+          "format_instructions": {
+            "required": true,
+            "source": "parser.format_instructions"
+          }
+        }
+      },
+      "model": {
+        "provider": "openai",
+        "name": "gpt-5.4",
+        "alias": "is_acknowledges_strength-evaluator-default"
+      },
+      "generation": {
+        "temperature": 1
+      },
+      "parser": {
+        "kind": "json_output_parser",
+        "format_instructions_placeholder": "format_instructions",
+        "output_schema_ref": "#/output_schema",
+        "notes": "Uses LangChain JsonOutputParser-style format_instructions injected into the system prompt."
+      },
+      "output_binding": "formatted_output"
+    }
+  ],
+  "output_schema": {
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "title": "EvaluatorOutput",
+    "description": "Final evaluator output for the is_acknowledges_strength dimension.",
+    "type": "object",
+    "additionalProperties": false,
+    "required": [
+      "reasoning",
+      "key_features",
+      "proposed_adjustment",
+      "acknowledges_strength_score"
+    ],
+    "properties": {
+      "reasoning": {
+        "type": "string",
+        "description": "Step-by-step reasoning before the final answer: assess the student response against the task goal, then judge whether the teacher feedback meets the criterion."
+      },
+      "key_features": {
+        "type": "object",
+        "description": "Per-key-feature assessment; each feature judged independently.",
+        "additionalProperties": false,
+        "required": [
+          "presence_of_praise",
+          "specificity",
+          "anchoring_to_evidence",
+          "process_vs_trait_framing"
+        ],
+        "properties": {
+          "presence_of_praise": {
+            "type": "object",
+            "description": "Presence of praise that is specific and authentic",
+            "required": [
+              "met",
+              "justification"
+            ],
+            "additionalProperties": false,
+            "properties": {
+              "met": {
+                "type": "integer",
+                "enum": [
+                  0,
+                  1
+                ],
+                "description": "1 if this key feature is satisfied by the feedback, 0 otherwise."
+              },
+              "justification": {
+                "type": "string",
+                "description": "One or two sentences grounding the met/not-met decision in the specific student response and teacher feedback."
+              }
+            }
+          },
+          "specificity": {
+            "type": "object",
+            "description": "Specificity of the acknowledgment rather than generic praise",
+            "required": [
+              "met",
+              "justification"
+            ],
+            "additionalProperties": false,
+            "properties": {
+              "met": {
+                "type": "integer",
+                "enum": [
+                  0,
+                  1
+                ],
+                "description": "1 if this key feature is satisfied by the feedback, 0 otherwise."
+              },
+              "justification": {
+                "type": "string",
+                "description": "One or two sentences grounding the met/not-met decision in the specific student response and teacher feedback."
+              }
+            }
+          },
+          "anchoring_to_evidence": {
+            "type": "object",
+            "description": "Anchoring to evidence in the student response",
+            "required": [
+              "met",
+              "justification"
+            ],
+            "additionalProperties": false,
+            "properties": {
+              "met": {
+                "type": "integer",
+                "enum": [
+                  0,
+                  1
+                ],
+                "description": "1 if this key feature is satisfied by the feedback, 0 otherwise."
+              },
+              "justification": {
+                "type": "string",
+                "description": "One or two sentences grounding the met/not-met decision in the specific student response and teacher feedback."
+              }
+            }
+          },
+          "process_vs_trait_framing": {
+            "type": "object",
+            "description": "Process-vs-trait framing",
+            "required": [
+              "met",
+              "justification"
+            ],
+            "additionalProperties": false,
+            "properties": {
+              "met": {
+                "type": "integer",
+                "enum": [
+                  0,
+                  1
+                ],
+                "description": "1 if this key feature is satisfied by the feedback, 0 otherwise."
+              },
+              "justification": {
+                "type": "string",
+                "description": "One or two sentences grounding the met/not-met decision in the specific student response and teacher feedback."
+              }
+            }
+          }
+        }
+      },
+      "proposed_adjustment": {
+        "type": "string",
+        "description": "How the teacher feedback could be modified to meet the criterion. If it already meets the criterion, say so briefly."
+      },
+      "acknowledges_strength_score": {
+        "type": "integer",
+        "enum": [
+          0,
+          1
+        ],
+        "description": "Overall: 1 if the feedback meets the criterion, 0 otherwise."
+      }
+    }
+  },
+  "fixtures": {
+    "sniff_test_path": "fixtures.json"  
+  },
+  "tracing": {
+    "expose": [
+      "rendered_prompt",
+      "raw_output",
+      "raw_text",
+      "formatted_output",
+      "usage"
+    ],
+    "notes": "Runtime engines MUST expose these five trace fields per evaluation."
+  }
+}

evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/prompts/fixtures.json

@@ -0,0 +1,26 @@
+[
+  {
+    "id": "0",
+    "description": "Example 1: Student writes about how AI powered pets help around the house. Feedback asks for additional details to strengthen the claim.",
+    "source": "Quill data",
+    "input": {
+      "student_text": "Some people think AI-powered pets are a good alternative to real pets because they could help around the house etc.",
+      "feedback_text": "You're right, the AI pets could help around the house. Can you find some other details from the article that you could add to make your claim stronger?"
+    },
+    "expected": {
+      "acknowledges_strength_score": 1
+    }
+  },
+  {
+    "id": "1",
+    "description": "Example 2: Student writes about how AI powered pets can help people with health issues. Feedback asks for an example and corrects spelling.",
+    "source": "Quill data",
+    "input": {
+      "student_text": "Some people think AI-powered pets are a good alternative to real pets because AI powered pets can be helpfull for people with health issues.",
+      "feedback_text": "How so? Can you give an example? Check your spelling of helpful."
+    },
+    "expected": {
+      "acknowledges_strength_score": 0
+    }
+  }
+]

evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/prompts/input_schema.json

@@ -0,0 +1,21 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "input_schema.json",
+  "title": "EvaluatorInput",
+  "type": "object",
+  "required": ["student_text", "feedback_text"],
+  "additionalProperties": false,
+  "properties": {
+    "student_text": {
+      "type": "string",
+      "minLength": 1,
+      "description": "The student's written response."
+    },
+    "feedback_text": {
+      "type": "string",
+      "minLength": 1,
+      "description": "The teacher's feedback to the student."
+    }
+
+  }
+}

evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/prompts/output_schema.json

@@ -0,0 +1,89 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "output_schema.json",
+  "title": "EvaluatorOutput",
+  "description": "Final evaluator output for the is_acknowledges_strength dimension.",
+  "type": "object",
+  "required": [
+    "reasoning",
+    "key_features",
+    "proposed_adjustment",
+    "acknowledges_strength_score"
+  ],
+  "additionalProperties": false,
+  "properties": {
+    "reasoning": {
+      "type": "string",
+      "description": "Step-by-step reasoning before the final answer: assess the student response against the task goal, then judge whether the teacher feedback meets the criterion."
+    },
+    "key_features": {
+      "$ref": "#/$defs/KeyFeatures"
+    },
+    "proposed_adjustment": {
+      "type": "string",
+      "description": "How the teacher feedback could be modified to meet the criterion. If it already meets the criterion, say so briefly."
+    },
+    "acknowledges_strength_score": {
+      "type": "integer",
+      "enum": [
+        0,
+        1
+      ],
+      "description": "Overall: 1 if the feedback meets the criterion, 0 otherwise."
+    }
+  },
+  "$defs": {
+    "KeyFeatureAssessment": {
+      "type": "object",
+      "description": "Independent assessment of a single key feature.",
+      "required": [
+        "met",
+        "justification"
+      ],
+      "additionalProperties": false,
+      "properties": {
+        "met": {
+          "type": "integer",
+          "enum": [
+            0,
+            1
+          ],
+          "description": "1 if this key feature is satisfied by the feedback, 0 otherwise."
+        },
+        "justification": {
+          "type": "string",
+          "description": "One or two sentences grounding the met/not-met decision in the specific student response and teacher feedback."
+        }
+      }
+    },
+    "KeyFeatures": {
+      "type": "object",
+      "description": "Per-key-feature assessment; each feature judged independently.",
+      "required": [
+        "presence_of_praise",
+        "specificity",
+        "anchoring_to_evidence",
+        "process_vs_trait_framing"
+      ],
+      "additionalProperties": false,
+      "properties": {
+        "presence_of_praise": {
+          "$ref": "#/$defs/KeyFeatureAssessment",
+          "description": "Presence of praise that is specific and authentic"
+        },
+        "specificity": {
+          "$ref": "#/$defs/KeyFeatureAssessment",
+          "description": "Specificity of the acknowledgment rather than generic praise"
+        },
+        "anchoring_to_evidence": {
+          "$ref": "#/$defs/KeyFeatureAssessment",
+          "description": "Anchoring to evidence in the student response"
+        },
+        "process_vs_trait_framing": {
+          "$ref": "#/$defs/KeyFeatureAssessment",
+          "description": "Process-vs-trait framing"
+        }
+      }
+    }
+  }
+}