From c7a4eb39c2b960e6647c1ac0f82bc64e46e61ded Mon Sep 17 00:00:00 2001 From: Adnan Rashid Hussain Date: Thu, 25 Jun 2026 16:38:26 -0700 Subject: [PATCH 1/2] chore: add repo check harness with notebook stripping --- .github/workflows/checks.yml | 33 ++ .../example_notebook.ipynb | 66 +-- .../example_notebook.ipynb | 415 +------------ .../example_notebook.ipynb | 66 +-- .../example_notebook.ipynb | 550 +----------------- .../manageable/example_notebook.ipynb | 66 +-- evals/grade_level_evaluator.ipynb | 159 +---- evals/sentence_structure_evaluator.ipynb | 145 +---- evals/text_complexity_combo.ipynb | 212 +++---- evals/vocabulary_evaluator.ipynb | 131 +---- scripts/README.md | 59 ++ scripts/check.py | 68 +++ scripts/checks/__init__.py | 7 + scripts/checks/base.py | 50 ++ scripts/checks/strip_notebooks.py | 58 ++ scripts/requirements.txt | 3 + 16 files changed, 454 insertions(+), 1634 deletions(-) create mode 100644 .github/workflows/checks.yml create mode 100644 scripts/README.md create mode 100755 scripts/check.py create mode 100644 scripts/checks/__init__.py create mode 100644 scripts/checks/base.py create mode 100644 scripts/checks/strip_notebooks.py create mode 100644 scripts/requirements.txt diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 00000000..1324e7e4 --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,33 @@ +name: ✅ Repo Checks + +on: + push: + branches: + - main + pull_request: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + checks: + name: Run repo checks + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - name: Install check dependencies + run: pip install -r scripts/requirements.txt + + - name: Run checks + run: python scripts/check.py diff --git a/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/example_notebook.ipynb b/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/example_notebook.ipynb index 1f1503c6..9b695d32 100644 --- a/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/example_notebook.ipynb +++ b/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/example_notebook.ipynb @@ -2,16 +2,7 @@ "cells": [ { "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "019a2e29-26c8-4cdd-8d6b-88de27b95384", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "source": [ "# Acknowledges Strength Evaluator\n", "\n", @@ -35,16 +26,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "1958719e-0bd2-43fa-a6a0-ca1d090da428", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "outputs": [], "source": [ "import getpass\n", @@ -80,16 +62,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "c2ba0d01-9d12-40f7-9057-22df59eaeb05", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Load config + verify prompt hashes" - } - }, + "metadata": {}, "outputs": [], "source": [ "# -------------------------------------------------------------------------\n", @@ -144,16 +117,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "4e8def3b-028c-4b71-acee-13fb1553b2ef", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Evaluator function (config-driven)" - } - }, + "metadata": {}, "outputs": [], "source": [ "_STEP = CONFIG[\"steps\"][0]\n", @@ -220,16 +184,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "45aff123-c2d0-4041-9737-f06ebbdceee4", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Smoke test on a sample" - } - }, + "metadata": {}, "outputs": [], "source": [ "sample_student_text = \"\"\"Some people think AI-powered pets are a good alternative to real pets because they could help around the house etc.\"\"\"\n", @@ -273,16 +228,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "421b04bd-a7bb-41c7-ba8b-0716ae03ff82", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Sniff-test fixture runner" - } - }, + "metadata": {}, "outputs": [], "source": [ "fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"sniff_test_path\"]\n", diff --git a/evals/feedback/productive-coaching-writing-feedback/actionable-revision/example_notebook.ipynb b/evals/feedback/productive-coaching-writing-feedback/actionable-revision/example_notebook.ipynb index 15d84bef..bd6fcfeb 100644 --- a/evals/feedback/productive-coaching-writing-feedback/actionable-revision/example_notebook.ipynb +++ b/evals/feedback/productive-coaching-writing-feedback/actionable-revision/example_notebook.ipynb @@ -2,16 +2,7 @@ "cells": [ { "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "019a2e29-26c8-4cdd-8d6b-88de27b95384", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "source": [ "# Actionable Revision Evaluator\n", "\n", @@ -34,17 +25,8 @@ }, { "cell_type": "code", - "execution_count": 11, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "1958719e-0bd2-43fa-a6a0-ca1d090da428", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "import getpass\n", @@ -66,7 +48,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -79,31 +61,9 @@ }, { "cell_type": "code", - "execution_count": 13, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "c2ba0d01-9d12-40f7-9057-22df59eaeb05", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Load config + verify prompt hashes" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loaded feedback.productive_coaching_writing_feedback.is_actionable_revision from /Users/achi/Documents/evaluators/evals/feedback/productive-coaching-writing-feedback/actionable-revision\n", - " model: gpt-5.4\n", - " temperature: 1\n", - " prompts:\n", - " system system.txt ( 5731 chars, sha d4504cc60437)\n", - " user user.txt ( 69 chars, sha eaac7e6eadcc)\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# -------------------------------------------------------------------------\n", "# Load source-of-truth assets: config.json + every prompt file declared\n", @@ -156,17 +116,8 @@ }, { "cell_type": "code", - "execution_count": 14, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "4e8def3b-028c-4b71-acee-13fb1553b2ef", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Evaluator function (config-driven)" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "_STEP = CONFIG[\"steps\"][0]\n", @@ -232,58 +183,9 @@ }, { "cell_type": "code", - "execution_count": 15, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "45aff123-c2d0-4041-9737-f06ebbdceee4", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Smoke test on a sample" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'rendered_prompt': [{'content': 'You are an expert evaluator of written teacher feedback to students. Your task is to judge how well the teacher feedback supports productive coaching by giving students an actionable path for revision.\\n\\nProductive coaching goal:\\nProductive coaching helps students enter productive struggle: cognitively challenging but emotionally manageable work that helps learners improve by grappling with problems just beyond their current ability.\\n\\nCriterion to evaluate: Is Actionable Revision\\n\\nDefinition:\\nIf the student response needs revision, the teacher feedback should give a clear, usable next step. The feedback may use a directive statement or a focused question, but the student should be able to tell what to revise and what to do next without needing additional clarification.\\n\\nThe feedback MEETS the criterion when:\\n- It gives a clear next step such as add, explain, clarify, revise, support, define, include, connect, or reconsider a specific part of the response.\\n- Or it asks a focused question that clearly implies a revision move and is tied to the student’s actual writing.\\n- The student could reasonably act on the feedback independently.\\n- The feedback does not need to identify the exact sentence to write or the exact evidence to use, as long as the next move is clear enough.\\n\\nThe feedback does NOT meet the criterion when:\\n- It is purely evaluative or praise-only.\\n- It states a broad expectation without a concrete revision action.\\n- It asks vague, tangential, or curiosity-driven questions that do not clearly guide revision.\\n- It identifies a possible issue but does not make clear what the student should change, add, explain, or clarify.\\n- The student would likely need to ask, “What exactly should I do?”\\n\\nImportant calibration examples:\\n- Student: “Some people think AI-powered pets are a good alternative to real pets because they wont die or need mediceine”\\n Feedback: “This is a good answer, thank you for your response. However, lets stick to providing facts from the reading to support your argument.”\\n Correct judgment: 0. The feedback points generally toward using facts from the reading, but “stick to providing facts” is too broad and does not give a concrete next step such as adding a detail from the article to the claim.\\n\\n- Student: “Some people think AI-powered pets are a good alternative to real pets because they wont die or need mediceine”\\n Feedback: “Yes, AI powered pets could last a long time. Can you think of some other details the article included that you might want to add to your claim?”\\n Correct judgment: 1. The question is actionable because it directs the student to add other details from the article to the claim. It does not name the exact details, but the student can reasonably reread the article and add supporting details.\\n\\n- Student: “Some people think AI-powered pets are a good alternative to real pets because if someone cannot have a dog due to allergies, they can now have one risk free.”\\n Feedback: “What about other pets, such as cats? Would that work too? What do you mean by risk-free?”\\n Correct judgment: 0. Although the feedback asks questions, they do not provide clear, targeted, actionable guidance for improving the student’s statement. The questions about cats and “risk-free” do not clearly tell the student what to revise, add, support, or clarify in the response. Do not over-credit feedback just because it contains questions.\\n\\nInput format:\\nYou will receive:\\n- student_text: the student’s written response\\n- feedback_text: the teacher feedback being evaluated\\n\\nEvaluate only the written student response and written teacher feedback. Do not infer unstated teacher intent. Do not assume access to the original article, assignment, or rubric unless explicitly provided. Score only what is written.\\n\\nFor each case, assess these key features independently:\\n\\n1. directive_verb_or_focused_question\\n - Mark 1 if the feedback contains a clear directive verb or a genuinely focused question aimed at revision.\\n - Examples of directive verbs: add, explain, clarify, revise, include, support, connect, define, give, describe.\\n - A question counts only if it clearly guides revision. Broad, tangential, or curiosity-style questions should not automatically receive credit.\\n\\n2. clarity_of_target\\n - Mark 1 if the feedback clearly identifies what part or aspect of the student response should be revised.\\n - Examples: the claim, evidence from the article, an unclear phrase, a missing explanation, a specific example.\\n - Mark 0 if the target is vague or the student would not know what part of the response to work on.\\n\\n3. specificity_of_next_move\\n - Mark 1 if the feedback gives a specific enough action for revision.\\n - It does not need to provide exact wording or exact evidence, but it should give a usable action such as “add another detail from the article to your claim” or “explain what you mean by risk-free.”\\n - Mark 0 if the feedback only gives a general standard such as “use facts,” “be clearer,” or “think more,” without a concrete next step.\\n\\n4. reasonable_student_action\\n - Mark 1 if the student could reasonably revise based on the feedback without needing additional clarification.\\n - Mark 0 if the student would likely be unsure what to change, add, explain, or clarify.\\n\\nOverall answer:\\n- Return answer = 1 only if the feedback meets the actionable revision criterion.\\n- In most cases, this requires that the feedback has a clear revision target, a specific next move, and a reasonable action the student can take.\\n- If the student response clearly does not need revision, the feedback does not need to include a next step.\\n- If revision is needed and the feedback lacks a clear usable next step, return answer = 0.',\n", - " 'additional_kwargs': {},\n", - " 'response_metadata': {},\n", - " 'type': 'system',\n", - " 'name': None,\n", - " 'id': None},\n", - " {'content': \"Analyze:\\nStudent text: Some people think AI-powered pets are a good alternative to real pets because they could help around the house etc.\\nFeedback text: You're right, the AI pets could help around the house. Can you find some other details from the article that you could add to make your claim stronger?\\n\",\n", - " 'additional_kwargs': {},\n", - " 'response_metadata': {},\n", - " 'type': 'human',\n", - " 'name': None,\n", - " 'id': None}],\n", - " 'raw_output': AIMessage(content='{\"reasoning\":\"The student response gives one reason (“help around the house”) but is underdeveloped, especially with the vague “etc.,” so revision is needed. The teacher feedback does more than praise: it asks a focused revision question tied to the student’s claim and directs the student to add other details from the article. The target is clear—strengthening the claim by adding supporting details—and the next move is usable even though it does not name the exact details. A student could reasonably reread the article and incorporate additional relevant evidence.\",\"key_features\":{\"directive_verb_or_focused_question\":{\"met\":1,\"justification\":\"The feedback includes a focused question: “Can you find some other details from the article that you could add...?” This clearly prompts a revision action.\"},\"clarity_of_target\":{\"met\":1,\"justification\":\"The feedback identifies the target as the student’s claim and indicates that it needs stronger support through additional details from the article.\"},\"specificity_of_next_move\":{\"met\":1,\"justification\":\"The next move is specific enough: add other details from the article to strengthen the claim. The student does not need exact wording to act on this.\"},\"reasonable_student_action\":{\"met\":1,\"justification\":\"A student could independently revise by rereading the article, selecting more details, and adding them to the response.\"}},\"proposed_adjustment\":\"It already meets the criterion. A slightly stronger version could name the current idea and prompt evidence, such as: “Add one or two more details from the article besides helping around the house to strengthen your claim.”\",\"actionable_revision_score\":1}', additional_kwargs={'parsed': None, 'refusal': None}, response_metadata={'token_usage': {'completion_tokens': 337, 'prompt_tokens': 1618, 'total_tokens': 1955, 'completion_tokens_details': {'accepted_prediction_tokens': 0, 'audio_tokens': 0, 'reasoning_tokens': 0, 'rejected_prediction_tokens': 0}, 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 0}}, 'model_provider': 'openai', 'model_name': 'gpt-5.4-2026-03-05', 'system_fingerprint': None, 'id': 'chatcmpl-Du4LxzskcxYQJMJteDOsAKzivoH89', 'service_tier': 'default', 'finish_reason': 'stop', 'logprobs': None}, id='lc_run--019ef6b4-4f74-7a31-82fc-71c8510684ec-0', tool_calls=[], invalid_tool_calls=[], usage_metadata={'input_tokens': 1618, 'output_tokens': 337, 'total_tokens': 1955, 'input_token_details': {'audio': 0, 'cache_read': 0}, 'output_token_details': {'audio': 0, 'reasoning': 0}}),\n", - " 'raw_text': '{\"reasoning\":\"The student response gives one reason (“help around the house”) but is underdeveloped, especially with the vague “etc.,” so revision is needed. The teacher feedback does more than praise: it asks a focused revision question tied to the student’s claim and directs the student to add other details from the article. The target is clear—strengthening the claim by adding supporting details—and the next move is usable even though it does not name the exact details. A student could reasonably reread the article and incorporate additional relevant evidence.\",\"key_features\":{\"directive_verb_or_focused_question\":{\"met\":1,\"justification\":\"The feedback includes a focused question: “Can you find some other details from the article that you could add...?” This clearly prompts a revision action.\"},\"clarity_of_target\":{\"met\":1,\"justification\":\"The feedback identifies the target as the student’s claim and indicates that it needs stronger support through additional details from the article.\"},\"specificity_of_next_move\":{\"met\":1,\"justification\":\"The next move is specific enough: add other details from the article to strengthen the claim. The student does not need exact wording to act on this.\"},\"reasonable_student_action\":{\"met\":1,\"justification\":\"A student could independently revise by rereading the article, selecting more details, and adding them to the response.\"}},\"proposed_adjustment\":\"It already meets the criterion. A slightly stronger version could name the current idea and prompt evidence, such as: “Add one or two more details from the article besides helping around the house to strengthen your claim.”\",\"actionable_revision_score\":1}',\n", - " 'formatted_output': {'reasoning': 'The student response gives one reason (“help around the house”) but is underdeveloped, especially with the vague “etc.,” so revision is needed. The teacher feedback does more than praise: it asks a focused revision question tied to the student’s claim and directs the student to add other details from the article. The target is clear—strengthening the claim by adding supporting details—and the next move is usable even though it does not name the exact details. A student could reasonably reread the article and incorporate additional relevant evidence.',\n", - " 'key_features': {'directive_verb_or_focused_question': {'met': 1,\n", - " 'justification': 'The feedback includes a focused question: “Can you find some other details from the article that you could add...?” This clearly prompts a revision action.'},\n", - " 'clarity_of_target': {'met': 1,\n", - " 'justification': 'The feedback identifies the target as the student’s claim and indicates that it needs stronger support through additional details from the article.'},\n", - " 'specificity_of_next_move': {'met': 1,\n", - " 'justification': 'The next move is specific enough: add other details from the article to strengthen the claim. The student does not need exact wording to act on this.'},\n", - " 'reasonable_student_action': {'met': 1,\n", - " 'justification': 'A student could independently revise by rereading the article, selecting more details, and adding them to the response.'}},\n", - " 'proposed_adjustment': 'It already meets the criterion. A slightly stronger version could name the current idea and prompt evidence, such as: “Add one or two more details from the article besides helping around the house to strengthen your claim.”',\n", - " 'actionable_revision_score': 1},\n", - " 'usage': {'input_tokens': 1618,\n", - " 'output_tokens': 337,\n", - " 'total_tokens': 1955,\n", - " 'input_token_details': {'audio': 0, 'cache_read': 0},\n", - " 'output_token_details': {'audio': 0, 'reasoning': 0}}}" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "sample_student_text = \"\"\"Some people think AI-powered pets are a good alternative to real pets because they could help around the house etc.\"\"\"\n", "sample_feedback_text = \"\"\"You're right, the AI pets could help around the house. Can you find some other details from the article that you could add to make your claim stronger?\"\"\"\n", @@ -296,267 +198,9 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "============================================================\n", - "RENDERED PROMPT (input sent to the LLM)\n", - "============================================================\n", - "[{'additional_kwargs': {},\n", - " 'content': 'You are an expert evaluator of written teacher feedback to '\n", - " 'students. Your task is to judge how well the teacher feedback '\n", - " 'supports productive coaching by giving students an actionable '\n", - " 'path for revision.\\n'\n", - " '\\n'\n", - " 'Productive coaching goal:\\n'\n", - " 'Productive coaching helps students enter productive struggle: '\n", - " 'cognitively challenging but emotionally manageable work that '\n", - " 'helps learners improve by grappling with problems just beyond '\n", - " 'their current ability.\\n'\n", - " '\\n'\n", - " 'Criterion to evaluate: Is Actionable Revision\\n'\n", - " '\\n'\n", - " 'Definition:\\n'\n", - " 'If the student response needs revision, the teacher feedback '\n", - " 'should give a clear, usable next step. The feedback may use a '\n", - " 'directive statement or a focused question, but the student '\n", - " 'should be able to tell what to revise and what to do next '\n", - " 'without needing additional clarification.\\n'\n", - " '\\n'\n", - " 'The feedback MEETS the criterion when:\\n'\n", - " '- It gives a clear next step such as add, explain, clarify, '\n", - " 'revise, support, define, include, connect, or reconsider a '\n", - " 'specific part of the response.\\n'\n", - " '- Or it asks a focused question that clearly implies a revision '\n", - " 'move and is tied to the student’s actual writing.\\n'\n", - " '- The student could reasonably act on the feedback '\n", - " 'independently.\\n'\n", - " '- The feedback does not need to identify the exact sentence to '\n", - " 'write or the exact evidence to use, as long as the next move is '\n", - " 'clear enough.\\n'\n", - " '\\n'\n", - " 'The feedback does NOT meet the criterion when:\\n'\n", - " '- It is purely evaluative or praise-only.\\n'\n", - " '- It states a broad expectation without a concrete revision '\n", - " 'action.\\n'\n", - " '- It asks vague, tangential, or curiosity-driven questions that '\n", - " 'do not clearly guide revision.\\n'\n", - " '- It identifies a possible issue but does not make clear what '\n", - " 'the student should change, add, explain, or clarify.\\n'\n", - " '- The student would likely need to ask, “What exactly should I '\n", - " 'do?”\\n'\n", - " '\\n'\n", - " 'Important calibration examples:\\n'\n", - " '- Student: “Some people think AI-powered pets are a good '\n", - " 'alternative to real pets because they wont die or need '\n", - " 'mediceine”\\n'\n", - " ' Feedback: “This is a good answer, thank you for your response. '\n", - " 'However, lets stick to providing facts from the reading to '\n", - " 'support your argument.”\\n'\n", - " ' Correct judgment: 0. The feedback points generally toward '\n", - " 'using facts from the reading, but “stick to providing facts” is '\n", - " 'too broad and does not give a concrete next step such as adding '\n", - " 'a detail from the article to the claim.\\n'\n", - " '\\n'\n", - " '- Student: “Some people think AI-powered pets are a good '\n", - " 'alternative to real pets because they wont die or need '\n", - " 'mediceine”\\n'\n", - " ' Feedback: “Yes, AI powered pets could last a long time. Can '\n", - " 'you think of some other details the article included that you '\n", - " 'might want to add to your claim?”\\n'\n", - " ' Correct judgment: 1. The question is actionable because it '\n", - " 'directs the student to add other details from the article to the '\n", - " 'claim. It does not name the exact details, but the student can '\n", - " 'reasonably reread the article and add supporting details.\\n'\n", - " '\\n'\n", - " '- Student: “Some people think AI-powered pets are a good '\n", - " 'alternative to real pets because if someone cannot have a dog '\n", - " 'due to allergies, they can now have one risk free.”\\n'\n", - " ' Feedback: “What about other pets, such as cats? Would that '\n", - " 'work too? What do you mean by risk-free?”\\n'\n", - " ' Correct judgment: 0. Although the feedback asks questions, '\n", - " 'they do not provide clear, targeted, actionable guidance for '\n", - " 'improving the student’s statement. The questions about cats and '\n", - " '“risk-free” do not clearly tell the student what to revise, add, '\n", - " 'support, or clarify in the response. Do not over-credit feedback '\n", - " 'just because it contains questions.\\n'\n", - " '\\n'\n", - " 'Input format:\\n'\n", - " 'You will receive:\\n'\n", - " '- student_text: the student’s written response\\n'\n", - " '- feedback_text: the teacher feedback being evaluated\\n'\n", - " '\\n'\n", - " 'Evaluate only the written student response and written teacher '\n", - " 'feedback. Do not infer unstated teacher intent. Do not assume '\n", - " 'access to the original article, assignment, or rubric unless '\n", - " 'explicitly provided. Score only what is written.\\n'\n", - " '\\n'\n", - " 'For each case, assess these key features independently:\\n'\n", - " '\\n'\n", - " '1. directive_verb_or_focused_question\\n'\n", - " ' - Mark 1 if the feedback contains a clear directive verb or a '\n", - " 'genuinely focused question aimed at revision.\\n'\n", - " ' - Examples of directive verbs: add, explain, clarify, revise, '\n", - " 'include, support, connect, define, give, describe.\\n'\n", - " ' - A question counts only if it clearly guides revision. '\n", - " 'Broad, tangential, or curiosity-style questions should not '\n", - " 'automatically receive credit.\\n'\n", - " '\\n'\n", - " '2. clarity_of_target\\n'\n", - " ' - Mark 1 if the feedback clearly identifies what part or '\n", - " 'aspect of the student response should be revised.\\n'\n", - " ' - Examples: the claim, evidence from the article, an unclear '\n", - " 'phrase, a missing explanation, a specific example.\\n'\n", - " ' - Mark 0 if the target is vague or the student would not know '\n", - " 'what part of the response to work on.\\n'\n", - " '\\n'\n", - " '3. specificity_of_next_move\\n'\n", - " ' - Mark 1 if the feedback gives a specific enough action for '\n", - " 'revision.\\n'\n", - " ' - It does not need to provide exact wording or exact '\n", - " 'evidence, but it should give a usable action such as “add '\n", - " 'another detail from the article to your claim” or “explain what '\n", - " 'you mean by risk-free.”\\n'\n", - " ' - Mark 0 if the feedback only gives a general standard such '\n", - " 'as “use facts,” “be clearer,” or “think more,” without a '\n", - " 'concrete next step.\\n'\n", - " '\\n'\n", - " '4. reasonable_student_action\\n'\n", - " ' - Mark 1 if the student could reasonably revise based on the '\n", - " 'feedback without needing additional clarification.\\n'\n", - " ' - Mark 0 if the student would likely be unsure what to '\n", - " 'change, add, explain, or clarify.\\n'\n", - " '\\n'\n", - " 'Overall answer:\\n'\n", - " '- Return answer = 1 only if the feedback meets the actionable '\n", - " 'revision criterion.\\n'\n", - " '- In most cases, this requires that the feedback has a clear '\n", - " 'revision target, a specific next move, and a reasonable action '\n", - " 'the student can take.\\n'\n", - " '- If the student response clearly does not need revision, the '\n", - " 'feedback does not need to include a next step.\\n'\n", - " '- If revision is needed and the feedback lacks a clear usable '\n", - " 'next step, return answer = 0.',\n", - " 'id': None,\n", - " 'name': None,\n", - " 'response_metadata': {},\n", - " 'type': 'system'},\n", - " {'additional_kwargs': {},\n", - " 'content': 'Analyze:\\n'\n", - " 'Student text: Some people think AI-powered pets are a good '\n", - " 'alternative to real pets because they could help around the '\n", - " 'house etc.\\n'\n", - " \"Feedback text: You're right, the AI pets could help around the \"\n", - " 'house. Can you find some other details from the article that you '\n", - " 'could add to make your claim stronger?\\n',\n", - " 'id': None,\n", - " 'name': None,\n", - " 'response_metadata': {},\n", - " 'type': 'human'}]\n", - "\n", - "============================================================\n", - "RAW LLM TEXT (model's verbatim output)\n", - "============================================================\n", - "{\"reasoning\":\"The student response gives one reason (“help around the house”) but is underdeveloped, especially with the vague “etc.,” so revision is needed. The teacher feedback does more than praise: it asks a focused revision question tied to the student’s claim and directs the student to add other details from the article. The target is clear—strengthening the claim by adding supporting details—and the next move is usable even though it does not name the exact details. A student could reasonably reread the article and incorporate additional relevant evidence.\",\"key_features\":{\"directive_verb_or_focused_question\":{\"met\":1,\"justification\":\"The feedback includes a focused question: “Can you find some other details from the article that you could add...?” This clearly prompts a revision action.\"},\"clarity_of_target\":{\"met\":1,\"justification\":\"The feedback identifies the target as the student’s claim and indicates that it needs stronger support through additional details from the article.\"},\"specificity_of_next_move\":{\"met\":1,\"justification\":\"The next move is specific enough: add other details from the article to strengthen the claim. The student does not need exact wording to act on this.\"},\"reasonable_student_action\":{\"met\":1,\"justification\":\"A student could independently revise by rereading the article, selecting more details, and adding them to the response.\"}},\"proposed_adjustment\":\"It already meets the criterion. A slightly stronger version could name the current idea and prompt evidence, such as: “Add one or two more details from the article besides helping around the house to strengthen your claim.”\",\"actionable_revision_score\":1}\n", - "\n", - "============================================================\n", - "PARSED OUTPUT (output_schema)\n", - "============================================================\n", - "{'actionable_revision_score': 1,\n", - " 'key_features': {'clarity_of_target': {'justification': 'The feedback '\n", - " 'identifies the '\n", - " 'target as the '\n", - " 'student’s claim and '\n", - " 'indicates that it '\n", - " 'needs stronger '\n", - " 'support through '\n", - " 'additional details '\n", - " 'from the article.',\n", - " 'met': 1},\n", - " 'directive_verb_or_focused_question': {'justification': 'The '\n", - " 'feedback '\n", - " 'includes '\n", - " 'a '\n", - " 'focused '\n", - " 'question: '\n", - " '“Can '\n", - " 'you '\n", - " 'find '\n", - " 'some '\n", - " 'other '\n", - " 'details '\n", - " 'from '\n", - " 'the '\n", - " 'article '\n", - " 'that '\n", - " 'you '\n", - " 'could '\n", - " 'add...?” '\n", - " 'This '\n", - " 'clearly '\n", - " 'prompts '\n", - " 'a '\n", - " 'revision '\n", - " 'action.',\n", - " 'met': 1},\n", - " 'reasonable_student_action': {'justification': 'A student '\n", - " 'could '\n", - " 'independently '\n", - " 'revise by '\n", - " 'rereading '\n", - " 'the article, '\n", - " 'selecting '\n", - " 'more '\n", - " 'details, and '\n", - " 'adding them '\n", - " 'to the '\n", - " 'response.',\n", - " 'met': 1},\n", - " 'specificity_of_next_move': {'justification': 'The next move '\n", - " 'is specific '\n", - " 'enough: add '\n", - " 'other details '\n", - " 'from the '\n", - " 'article to '\n", - " 'strengthen '\n", - " 'the claim. '\n", - " 'The student '\n", - " 'does not need '\n", - " 'exact wording '\n", - " 'to act on '\n", - " 'this.',\n", - " 'met': 1}},\n", - " 'proposed_adjustment': 'It already meets the criterion. A slightly stronger '\n", - " 'version could name the current idea and prompt '\n", - " 'evidence, such as: “Add one or two more details from '\n", - " 'the article besides helping around the house to '\n", - " 'strengthen your claim.”',\n", - " 'reasoning': 'The student response gives one reason (“help around the house”) '\n", - " 'but is underdeveloped, especially with the vague “etc.,” so '\n", - " 'revision is needed. The teacher feedback does more than praise: '\n", - " 'it asks a focused revision question tied to the student’s claim '\n", - " 'and directs the student to add other details from the article. '\n", - " 'The target is clear—strengthening the claim by adding '\n", - " 'supporting details—and the next move is usable even though it '\n", - " 'does not name the exact details. A student could reasonably '\n", - " 'reread the article and incorporate additional relevant '\n", - " 'evidence.'}\n", - "\n", - "============================================================\n", - "USAGE METADATA\n", - "============================================================\n", - "{'input_token_details': {'audio': 0, 'cache_read': 0},\n", - " 'input_tokens': 1618,\n", - " 'output_token_details': {'audio': 0, 'reasoning': 0},\n", - " 'output_tokens': 337,\n", - " 'total_tokens': 1955}\n" - ] - } - ], + "outputs": [], "source": [ "import pprint as pp\n", "# I/O trace breakdown -- this is what the SDK engineer will replicate in TS.\n", @@ -583,34 +227,9 @@ }, { "cell_type": "code", - "execution_count": 17, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "421b04bd-a7bb-41c7-ba8b-0716ae03ff82", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Sniff-test fixture runner" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loaded 2 fixtures from fixtures.json\n", - "\n", - "==============================================================================\n", - " ID STATUS PREDICTED EXPECTED DESCRIPTION\n", - "==============================================================================\n", - " 0 PASS 1 1 Example 1: Student writes\n", - " 1 PASS 1 1 Example 2: Student writes\n", - "==============================================================================\n", - "Summary: 2 exact, 0 adjacent, 0 fail, 0 error -- total 2\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"sniff_test_path\"]\n", "if not fixtures_path.exists():\n", diff --git a/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/example_notebook.ipynb b/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/example_notebook.ipynb index a8792f90..81fb3a8b 100644 --- a/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/example_notebook.ipynb +++ b/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/example_notebook.ipynb @@ -2,16 +2,7 @@ "cells": [ { "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "019a2e29-26c8-4cdd-8d6b-88de27b95384", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "source": [ "# Anchored in Student Response Evaluator\n", "\n", @@ -35,16 +26,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "1958719e-0bd2-43fa-a6a0-ca1d090da428", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "outputs": [], "source": [ "import getpass\n", @@ -80,16 +62,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "c2ba0d01-9d12-40f7-9057-22df59eaeb05", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Load config + verify prompt hashes" - } - }, + "metadata": {}, "outputs": [], "source": [ "# -------------------------------------------------------------------------\n", @@ -144,16 +117,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "4e8def3b-028c-4b71-acee-13fb1553b2ef", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Evaluator function (config-driven)" - } - }, + "metadata": {}, "outputs": [], "source": [ "_STEP = CONFIG[\"steps\"][0]\n", @@ -220,16 +184,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "45aff123-c2d0-4041-9737-f06ebbdceee4", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Smoke test on a sample" - } - }, + "metadata": {}, "outputs": [], "source": [ "sample_student_text = \"\"\"Some people think AI-powered pets are a good alternative to real pets because they could help around the house etc.\"\"\"\n", @@ -273,16 +228,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "421b04bd-a7bb-41c7-ba8b-0716ae03ff82", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Sniff-test fixture runner" - } - }, + "metadata": {}, "outputs": [], "source": [ "fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"sniff_test_path\"]\n", diff --git a/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/example_notebook.ipynb b/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/example_notebook.ipynb index 72b911d7..92c760f0 100644 --- a/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/example_notebook.ipynb +++ b/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/example_notebook.ipynb @@ -2,16 +2,7 @@ "cells": [ { "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "019a2e29-26c8-4cdd-8d6b-88de27b95384", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "source": [ "# Appropriate Feedback Evaluator\n", "\n", @@ -34,17 +25,8 @@ }, { "cell_type": "code", - "execution_count": 11, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "1958719e-0bd2-43fa-a6a0-ca1d090da428", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "import getpass\n", @@ -66,7 +48,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -79,31 +61,9 @@ }, { "cell_type": "code", - "execution_count": 13, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "c2ba0d01-9d12-40f7-9057-22df59eaeb05", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Load config + verify prompt hashes" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loaded feedback.productive_coaching_writing_feedback.is_appropriate_feedback from /Users/achi/Documents/evaluators/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback\n", - " model: gpt-5.4\n", - " temperature: 1\n", - " prompts:\n", - " system system.txt (10166 chars, sha 4c9b1cc69aa0)\n", - " user user.txt ( 69 chars, sha eaac7e6eadcc)\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# -------------------------------------------------------------------------\n", "# Load source-of-truth assets: config.json + every prompt file declared\n", @@ -156,17 +116,8 @@ }, { "cell_type": "code", - "execution_count": 14, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "4e8def3b-028c-4b71-acee-13fb1553b2ef", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Evaluator function (config-driven)" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "_STEP = CONFIG[\"steps\"][0]\n", @@ -232,56 +183,9 @@ }, { "cell_type": "code", - "execution_count": 15, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "45aff123-c2d0-4041-9737-f06ebbdceee4", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Smoke test on a sample" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'rendered_prompt': [{'content': \"You are an expert evaluator of written teacher feedback to students. Your job is to judge whether the teacher feedback meets the productive-coaching criterion: Appropriate Feedback Decision.\\n\\nCriterion: Appropriate Feedback Decision\\n\\nCore question:\\nDid the teacher feedback correctly identify whether the student needed to revise, or whether the student had met the relevant task requirements and could move on?\\n\\nYou will receive:\\n- student_text: the student’s response\\n- feedback_text: the teacher’s feedback\\n\\nYour job:\\n1. Determine what task requirements are relevant.\\n2. Decide whether the student response actually meets those requirements.\\n3. Decide whether the teacher feedback’s revise/move-on decision matches that reality.\\n4. Score the listed key features independently.\\n5. Return a JSON object only.\\n\\nImportant: Infer the relevant task requirements carefully.\\n- The basic task goal is that the student response should successfully complete a claim with relevant evidence/reasons.\\n- However, if the teacher feedback explicitly states or clearly reveals an original task requirement, treat that requirement as part of the task goal too.\\n - Example: If the feedback says the task was to “write one sentence with the word because in it,” then the student must satisfy that format requirement as well.\\n - If the student gives a claim and evidence but writes multiple sentences when the revealed task required one sentence with “because,” the response still needs revision.\\n - Feedback asking the student to combine sentences to meet that stated task requirement is an appropriate revision decision.\\n- Do not ignore concrete task requirements just because the student has a clear claim and evidence.\\n- Do not invent unstated requirements. Only use format/structure requirements that are explicit or clearly inferable from the feedback or input.\\n\\nBasic claim-and-support calibration:\\nA student response MEETS the claim/evidence part of the goal when it includes:\\n- a clear claim, and\\n- at least one relevant, reasonably accurate supporting reason or piece of evidence.\\n\\nA student response does NOT meet the claim/evidence part of the goal when:\\n- it gives only a claim with no real support,\\n- the evidence/reason is missing,\\n- the evidence/reason is irrelevant,\\n- the evidence/reason is too vague to function as support,\\n- the evidence/reason is factually questionable,\\n- the support is stated in an overly absolute or inaccurate way that should be revised.\\n\\nDomain-specific calibration for AI-powered pet examples:\\n- “Some people think AI-powered pets are a good alternative to real pets because some people have health problems” does NOT meet the basic claim/evidence goal. It gives a claim/topic but does not explain why AI pets are better for people with health problems.\\n- “AI pets are good because they help older people, people with memory loss, and can provide emotional support” DOES meet the basic claim/evidence goal. These are relevant reasons, even if they could be explained more fully.\\n- Cost-based reasons can be relevant support. For example, saying AI-powered pets may be more cost efficient because real pets require food, vet bills, and litter can count as relevant evidence.\\n- But “AI pets don’t require cost after purchasing” or “AI pets require no additional cost” is too absolute/questionable. AI pets may have lower ongoing costs, but saying they require no cost after purchase may be inaccurate and should be revised.\\n- Feedback that simply accepts an overly absolute cost claim as complete or valid without calling for revision should be treated as an inappropriate decision if revision is needed.\\n- However, if feedback begins with praise or agreement but then clearly asks the student to revise, explain, or be more specific, it can still make the correct revise/move-on decision.\\n\\nHow to interpret teacher feedback:\\n- Revision-oriented feedback includes language such as:\\n - “revise”\\n - “add”\\n - “explain”\\n - “be more specific”\\n - “you need to”\\n - “remember the task was...”\\n - “can you take these ideas and combine them...”\\n - questions that clearly ask the student to improve the response.\\n- A teacher does not need to use the word “revise” for the feedback to be revision-oriented.\\n- Feedback like “can you combine your second and third sentences with your first sentence” is revision-oriented, especially if the feedback explains that the original task required one sentence.\\n- Move-on feedback must clearly indicate that the response has met the task goal or that the student can move on.\\n - Acceptable signals include “you’ve met the goal,” “this is complete,” “you can move on,” or an unmistakable equivalent.\\n- Mere praise such as “Good job,” “Well stated,” “Great start,” “Excellent start,” or “valid evidence” is NOT enough by itself to count as a clear move-on signal.\\n- “Excellent start” or “Great start” alone is also NOT enough to count as a clear revision request. It is vague praise unless paired with a clear direction to revise.\\n- If the student has met all relevant task requirements, the teacher may still give optional enrichment suggestions, but the feedback must clearly signal that revision is optional, not required.\\n- If the student has met all relevant task requirements and the teacher says the student “needs to” revise, add more, or reorganize in order to satisfy the task, the feedback does NOT meet this criterion.\\n- If the student has not met the relevant task requirements and the teacher says or implies the work is complete, valid, or ready to move on, the feedback does NOT meet this criterion.\\n- If the student has not met the relevant task requirements and the teacher clearly asks for a revision that would help the student meet those requirements, the feedback DOES meet this criterion.\\n- Score only what is written in the feedback, not what the teacher may have intended.\\n\\nKey decision rules:\\n- If the student response does NOT meet the relevant task requirements, answer = 1 only if the feedback clearly signals that revision is needed.\\n- If the student response DOES meet the relevant task requirements, answer = 1 only if the feedback clearly signals that the student has met the goal or can move on, and does not frame unnecessary revisions as required.\\n- Otherwise, answer = 0.\\n\\nExamples of correct reasoning:\\n1. Student: “Some people think AI-powered pets are a good alternative to real pets because AI-powered pets don't come with the additional costs like vet bills or food.”\\n Feedback: “That's true—AI pets don't require expenses like food, vet bills, or toys! Now be more specific. Why does this make them a good alternative to real pets?”\\n Correct evaluation:\\n - Student has a claim and a relevant cost reason, but the support is somewhat absolute/underexplained and needs clarification.\\n - Feedback clearly asks the student to be more specific and explain.\\n - The revise decision is appropriate.\\n - answer = 1.\\n\\n2. Student: “Some people think AI-powered pets are a good alternative to real pets because it's more cost efficient. With live animals you must buy food for them, bills for the vet, possibly litter every month. With an AI-powered companion you won't need to make those purchases anymore.”\\n Feedback: “Great start remember, though that your task was to write one sentence with the word because in it. So can you take the ideas from your second and third sentence and combine them with your first sentence.”\\n Correct evaluation:\\n - Student has a clear claim and relevant support.\\n - But the feedback reveals that the original task required one sentence with the word “because.”\\n - The student wrote multiple sentences, so revision is needed to meet the original task requirements.\\n - The teacher appropriately asks the student to combine the ideas into one sentence.\\n - answer = 1.\\n\\n3. Student: “Some people think AI-powered pets are a good alternative to real pets because they don't require additional cost after they're purchased.”\\n Feedback: “This is an excellent start to your response.”\\n Correct evaluation:\\n - Student has a clear claim, but the reason is overly absolute/questionable because AI pets may still have some costs.\\n - Revision is needed.\\n - The feedback gives only vague praise and does not clearly ask for revision.\\n - answer = 0.\\n\\nKey features to assess:\\n\\n1. accurate_task_assessment\\nJudge whether the teacher feedback accurately assesses the student response’s status relative to the relevant task requirements.\\n- met = 1 if the feedback’s assessment matches the actual status of the student response.\\n - Example: The student violates a revealed one-sentence requirement, and the feedback points out that requirement and asks the student to revise.\\n- met = 0 if the feedback praises/accepts a flawed or incomplete response as complete, or treats a complete response as insufficient.\\n\\n2. revision_need_identification\\nJudge whether the feedback correctly distinguishes between “revision needed” and “ready to move on.”\\n- met = 1 if:\\n - the student response needs revision and the feedback clearly asks for revision, OR\\n - the student response is complete and the feedback clearly treats it as complete/ready to move on.\\n- met = 0 if:\\n - the feedback asks for required revision when the response already meets all relevant task requirements,\\n - the feedback fails to ask for revision when revision is needed,\\n - the feedback is only vague praise/agreement and does not clearly identify the correct next step.\\n\\n3. appropriate_signal_when_task_complete\\nJudge whether, when the student response meets all relevant task requirements, the feedback clearly signals completion/move-on.\\n- If the student response meets all relevant task requirements:\\n - met = 1 only if the feedback clearly says or unmistakably implies that the student has met the goal or can move on.\\n - met = 0 if the feedback merely praises the student, agrees with the content, or gives required revision directions without a move-on signal.\\n- If the student response does not meet the relevant task requirements:\\n - met = 0, with a justification explaining that the feature is not applicable because the task is not complete.\\n\",\n", - " 'additional_kwargs': {},\n", - " 'response_metadata': {},\n", - " 'type': 'system',\n", - " 'name': None,\n", - " 'id': None},\n", - " {'content': \"Analyze:\\nStudent text: Some people think AI-powered pets are a good alternative to real pets because they could help around the house etc.\\nFeedback text: You're right, the AI pets could help around the house. Can you find some other details from the article that you could add to make your claim stronger?\\n\",\n", - " 'additional_kwargs': {},\n", - " 'response_metadata': {},\n", - " 'type': 'human',\n", - " 'name': None,\n", - " 'id': None}],\n", - " 'raw_output': AIMessage(content='{\"reasoning\":\"The relevant task goal is to produce a claim with at least one relevant supporting reason or piece of evidence. No additional format requirement is revealed in the feedback. The student response includes a clear claim: AI-powered pets are a good alternative to real pets. It also includes a relevant supporting reason: they could help around the house. Although the wording ends with \\'etc.,\\' the response still gives at least one concrete, relevant reason, so it meets the basic claim-and-support requirement. The teacher feedback, however, asks the student to add other details from the article \\'to make your claim stronger,\\' which frames revision as needed rather than optional. Because the student already meets the relevant task requirements and the feedback does not clearly signal that the student is ready to move on, the feedback makes an inappropriate revise/move-on decision.\",\"key_features\":{\"accurate_task_assessment\":{\"met\":0,\"justification\":\"The student already provides a claim plus a relevant supporting reason, so the response meets the basic task requirement. The feedback treats the response as needing more added detail to satisfy the task, which overstates the need for revision.\"},\"revision_need_identification\":{\"met\":0,\"justification\":\"Since the student response is complete enough to meet the task, the feedback should clearly indicate completion or make enrichment optional. Instead, it asks the student to add more details, signaling required revision when revision is not necessary.\"},\"appropriate_signal_when_task_complete\":{\"met\":0,\"justification\":\"This feature applies because the student response meets the task goal. The feedback offers agreement and then asks for additional details, but it never clearly signals that the student has met the goal or can move on.\"}},\"proposed_adjustment\":\"Revise the feedback to clearly acknowledge that the student has met the goal and make any extension optional, for example: \\'You have a clear claim and a relevant reason, so you’ve met the task goal. If you want to make your argument even stronger, you could add another detail from the article.\\'\",\"appropriate_feedback_score\":0}', additional_kwargs={'parsed': None, 'refusal': None}, response_metadata={'token_usage': {'completion_tokens': 420, 'prompt_tokens': 2463, 'total_tokens': 2883, 'completion_tokens_details': {'accepted_prediction_tokens': 0, 'audio_tokens': 0, 'reasoning_tokens': 0, 'rejected_prediction_tokens': 0}, 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 0}}, 'model_provider': 'openai', 'model_name': 'gpt-5.4-2026-03-05', 'system_fingerprint': None, 'id': 'chatcmpl-Du4OLqp27pqJEfYgsGeyUIeweDKqC', 'service_tier': 'default', 'finish_reason': 'stop', 'logprobs': None}, id='lc_run--019ef6b6-90bc-7322-937b-57458714743f-0', tool_calls=[], invalid_tool_calls=[], usage_metadata={'input_tokens': 2463, 'output_tokens': 420, 'total_tokens': 2883, 'input_token_details': {'audio': 0, 'cache_read': 0}, 'output_token_details': {'audio': 0, 'reasoning': 0}}),\n", - " 'raw_text': '{\"reasoning\":\"The relevant task goal is to produce a claim with at least one relevant supporting reason or piece of evidence. No additional format requirement is revealed in the feedback. The student response includes a clear claim: AI-powered pets are a good alternative to real pets. It also includes a relevant supporting reason: they could help around the house. Although the wording ends with \\'etc.,\\' the response still gives at least one concrete, relevant reason, so it meets the basic claim-and-support requirement. The teacher feedback, however, asks the student to add other details from the article \\'to make your claim stronger,\\' which frames revision as needed rather than optional. Because the student already meets the relevant task requirements and the feedback does not clearly signal that the student is ready to move on, the feedback makes an inappropriate revise/move-on decision.\",\"key_features\":{\"accurate_task_assessment\":{\"met\":0,\"justification\":\"The student already provides a claim plus a relevant supporting reason, so the response meets the basic task requirement. The feedback treats the response as needing more added detail to satisfy the task, which overstates the need for revision.\"},\"revision_need_identification\":{\"met\":0,\"justification\":\"Since the student response is complete enough to meet the task, the feedback should clearly indicate completion or make enrichment optional. Instead, it asks the student to add more details, signaling required revision when revision is not necessary.\"},\"appropriate_signal_when_task_complete\":{\"met\":0,\"justification\":\"This feature applies because the student response meets the task goal. The feedback offers agreement and then asks for additional details, but it never clearly signals that the student has met the goal or can move on.\"}},\"proposed_adjustment\":\"Revise the feedback to clearly acknowledge that the student has met the goal and make any extension optional, for example: \\'You have a clear claim and a relevant reason, so you’ve met the task goal. If you want to make your argument even stronger, you could add another detail from the article.\\'\",\"appropriate_feedback_score\":0}',\n", - " 'formatted_output': {'reasoning': \"The relevant task goal is to produce a claim with at least one relevant supporting reason or piece of evidence. No additional format requirement is revealed in the feedback. The student response includes a clear claim: AI-powered pets are a good alternative to real pets. It also includes a relevant supporting reason: they could help around the house. Although the wording ends with 'etc.,' the response still gives at least one concrete, relevant reason, so it meets the basic claim-and-support requirement. The teacher feedback, however, asks the student to add other details from the article 'to make your claim stronger,' which frames revision as needed rather than optional. Because the student already meets the relevant task requirements and the feedback does not clearly signal that the student is ready to move on, the feedback makes an inappropriate revise/move-on decision.\",\n", - " 'key_features': {'accurate_task_assessment': {'met': 0,\n", - " 'justification': 'The student already provides a claim plus a relevant supporting reason, so the response meets the basic task requirement. The feedback treats the response as needing more added detail to satisfy the task, which overstates the need for revision.'},\n", - " 'revision_need_identification': {'met': 0,\n", - " 'justification': 'Since the student response is complete enough to meet the task, the feedback should clearly indicate completion or make enrichment optional. Instead, it asks the student to add more details, signaling required revision when revision is not necessary.'},\n", - " 'appropriate_signal_when_task_complete': {'met': 0,\n", - " 'justification': 'This feature applies because the student response meets the task goal. The feedback offers agreement and then asks for additional details, but it never clearly signals that the student has met the goal or can move on.'}},\n", - " 'proposed_adjustment': \"Revise the feedback to clearly acknowledge that the student has met the goal and make any extension optional, for example: 'You have a clear claim and a relevant reason, so you’ve met the task goal. If you want to make your argument even stronger, you could add another detail from the article.'\",\n", - " 'appropriate_feedback_score': 0},\n", - " 'usage': {'input_tokens': 2463,\n", - " 'output_tokens': 420,\n", - " 'total_tokens': 2883,\n", - " 'input_token_details': {'audio': 0, 'cache_read': 0},\n", - " 'output_token_details': {'audio': 0, 'reasoning': 0}}}" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "sample_student_text = \"\"\"Some people think AI-powered pets are a good alternative to real pets because they could help around the house etc.\"\"\"\n", "sample_feedback_text = \"\"\"You're right, the AI pets could help around the house. Can you find some other details from the article that you could add to make your claim stronger?\"\"\"\n", @@ -294,404 +198,9 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "============================================================\n", - "RENDERED PROMPT (input sent to the LLM)\n", - "============================================================\n", - "[{'additional_kwargs': {},\n", - " 'content': 'You are an expert evaluator of written teacher feedback to '\n", - " 'students. Your job is to judge whether the teacher feedback '\n", - " 'meets the productive-coaching criterion: Appropriate Feedback '\n", - " 'Decision.\\n'\n", - " '\\n'\n", - " 'Criterion: Appropriate Feedback Decision\\n'\n", - " '\\n'\n", - " 'Core question:\\n'\n", - " 'Did the teacher feedback correctly identify whether the student '\n", - " 'needed to revise, or whether the student had met the relevant '\n", - " 'task requirements and could move on?\\n'\n", - " '\\n'\n", - " 'You will receive:\\n'\n", - " '- student_text: the student’s response\\n'\n", - " '- feedback_text: the teacher’s feedback\\n'\n", - " '\\n'\n", - " 'Your job:\\n'\n", - " '1. Determine what task requirements are relevant.\\n'\n", - " '2. Decide whether the student response actually meets those '\n", - " 'requirements.\\n'\n", - " '3. Decide whether the teacher feedback’s revise/move-on decision '\n", - " 'matches that reality.\\n'\n", - " '4. Score the listed key features independently.\\n'\n", - " '5. Return a JSON object only.\\n'\n", - " '\\n'\n", - " 'Important: Infer the relevant task requirements carefully.\\n'\n", - " '- The basic task goal is that the student response should '\n", - " 'successfully complete a claim with relevant evidence/reasons.\\n'\n", - " '- However, if the teacher feedback explicitly states or clearly '\n", - " 'reveals an original task requirement, treat that requirement as '\n", - " 'part of the task goal too.\\n'\n", - " ' - Example: If the feedback says the task was to “write one '\n", - " 'sentence with the word because in it,” then the student must '\n", - " 'satisfy that format requirement as well.\\n'\n", - " ' - If the student gives a claim and evidence but writes '\n", - " 'multiple sentences when the revealed task required one sentence '\n", - " 'with “because,” the response still needs revision.\\n'\n", - " ' - Feedback asking the student to combine sentences to meet '\n", - " 'that stated task requirement is an appropriate revision '\n", - " 'decision.\\n'\n", - " '- Do not ignore concrete task requirements just because the '\n", - " 'student has a clear claim and evidence.\\n'\n", - " '- Do not invent unstated requirements. Only use format/structure '\n", - " 'requirements that are explicit or clearly inferable from the '\n", - " 'feedback or input.\\n'\n", - " '\\n'\n", - " 'Basic claim-and-support calibration:\\n'\n", - " 'A student response MEETS the claim/evidence part of the goal '\n", - " 'when it includes:\\n'\n", - " '- a clear claim, and\\n'\n", - " '- at least one relevant, reasonably accurate supporting reason '\n", - " 'or piece of evidence.\\n'\n", - " '\\n'\n", - " 'A student response does NOT meet the claim/evidence part of the '\n", - " 'goal when:\\n'\n", - " '- it gives only a claim with no real support,\\n'\n", - " '- the evidence/reason is missing,\\n'\n", - " '- the evidence/reason is irrelevant,\\n'\n", - " '- the evidence/reason is too vague to function as support,\\n'\n", - " '- the evidence/reason is factually questionable,\\n'\n", - " '- the support is stated in an overly absolute or inaccurate way '\n", - " 'that should be revised.\\n'\n", - " '\\n'\n", - " 'Domain-specific calibration for AI-powered pet examples:\\n'\n", - " '- “Some people think AI-powered pets are a good alternative to '\n", - " 'real pets because some people have health problems” does NOT '\n", - " 'meet the basic claim/evidence goal. It gives a claim/topic but '\n", - " 'does not explain why AI pets are better for people with health '\n", - " 'problems.\\n'\n", - " '- “AI pets are good because they help older people, people with '\n", - " 'memory loss, and can provide emotional support” DOES meet the '\n", - " 'basic claim/evidence goal. These are relevant reasons, even if '\n", - " 'they could be explained more fully.\\n'\n", - " '- Cost-based reasons can be relevant support. For example, '\n", - " 'saying AI-powered pets may be more cost efficient because real '\n", - " 'pets require food, vet bills, and litter can count as relevant '\n", - " 'evidence.\\n'\n", - " '- But “AI pets don’t require cost after purchasing” or “AI pets '\n", - " 'require no additional cost” is too absolute/questionable. AI '\n", - " 'pets may have lower ongoing costs, but saying they require no '\n", - " 'cost after purchase may be inaccurate and should be revised.\\n'\n", - " '- Feedback that simply accepts an overly absolute cost claim as '\n", - " 'complete or valid without calling for revision should be treated '\n", - " 'as an inappropriate decision if revision is needed.\\n'\n", - " '- However, if feedback begins with praise or agreement but then '\n", - " 'clearly asks the student to revise, explain, or be more '\n", - " 'specific, it can still make the correct revise/move-on '\n", - " 'decision.\\n'\n", - " '\\n'\n", - " 'How to interpret teacher feedback:\\n'\n", - " '- Revision-oriented feedback includes language such as:\\n'\n", - " ' - “revise”\\n'\n", - " ' - “add”\\n'\n", - " ' - “explain”\\n'\n", - " ' - “be more specific”\\n'\n", - " ' - “you need to”\\n'\n", - " ' - “remember the task was...”\\n'\n", - " ' - “can you take these ideas and combine them...”\\n'\n", - " ' - questions that clearly ask the student to improve the '\n", - " 'response.\\n'\n", - " '- A teacher does not need to use the word “revise” for the '\n", - " 'feedback to be revision-oriented.\\n'\n", - " '- Feedback like “can you combine your second and third sentences '\n", - " 'with your first sentence” is revision-oriented, especially if '\n", - " 'the feedback explains that the original task required one '\n", - " 'sentence.\\n'\n", - " '- Move-on feedback must clearly indicate that the response has '\n", - " 'met the task goal or that the student can move on.\\n'\n", - " ' - Acceptable signals include “you’ve met the goal,” “this is '\n", - " 'complete,” “you can move on,” or an unmistakable equivalent.\\n'\n", - " '- Mere praise such as “Good job,” “Well stated,” “Great start,” '\n", - " '“Excellent start,” or “valid evidence” is NOT enough by itself '\n", - " 'to count as a clear move-on signal.\\n'\n", - " '- “Excellent start” or “Great start” alone is also NOT enough to '\n", - " 'count as a clear revision request. It is vague praise unless '\n", - " 'paired with a clear direction to revise.\\n'\n", - " '- If the student has met all relevant task requirements, the '\n", - " 'teacher may still give optional enrichment suggestions, but the '\n", - " 'feedback must clearly signal that revision is optional, not '\n", - " 'required.\\n'\n", - " '- If the student has met all relevant task requirements and the '\n", - " 'teacher says the student “needs to” revise, add more, or '\n", - " 'reorganize in order to satisfy the task, the feedback does NOT '\n", - " 'meet this criterion.\\n'\n", - " '- If the student has not met the relevant task requirements and '\n", - " 'the teacher says or implies the work is complete, valid, or '\n", - " 'ready to move on, the feedback does NOT meet this criterion.\\n'\n", - " '- If the student has not met the relevant task requirements and '\n", - " 'the teacher clearly asks for a revision that would help the '\n", - " 'student meet those requirements, the feedback DOES meet this '\n", - " 'criterion.\\n'\n", - " '- Score only what is written in the feedback, not what the '\n", - " 'teacher may have intended.\\n'\n", - " '\\n'\n", - " 'Key decision rules:\\n'\n", - " '- If the student response does NOT meet the relevant task '\n", - " 'requirements, answer = 1 only if the feedback clearly signals '\n", - " 'that revision is needed.\\n'\n", - " '- If the student response DOES meet the relevant task '\n", - " 'requirements, answer = 1 only if the feedback clearly signals '\n", - " 'that the student has met the goal or can move on, and does not '\n", - " 'frame unnecessary revisions as required.\\n'\n", - " '- Otherwise, answer = 0.\\n'\n", - " '\\n'\n", - " 'Examples of correct reasoning:\\n'\n", - " '1. Student: “Some people think AI-powered pets are a good '\n", - " \"alternative to real pets because AI-powered pets don't come with \"\n", - " 'the additional costs like vet bills or food.”\\n'\n", - " \" Feedback: “That's true—AI pets don't require expenses like \"\n", - " 'food, vet bills, or toys! Now be more specific. Why does this '\n", - " 'make them a good alternative to real pets?”\\n'\n", - " ' Correct evaluation:\\n'\n", - " ' - Student has a claim and a relevant cost reason, but the '\n", - " 'support is somewhat absolute/underexplained and needs '\n", - " 'clarification.\\n'\n", - " ' - Feedback clearly asks the student to be more specific and '\n", - " 'explain.\\n'\n", - " ' - The revise decision is appropriate.\\n'\n", - " ' - answer = 1.\\n'\n", - " '\\n'\n", - " '2. Student: “Some people think AI-powered pets are a good '\n", - " \"alternative to real pets because it's more cost efficient. With \"\n", - " 'live animals you must buy food for them, bills for the vet, '\n", - " 'possibly litter every month. With an AI-powered companion you '\n", - " \"won't need to make those purchases anymore.”\\n\"\n", - " ' Feedback: “Great start remember, though that your task was to '\n", - " 'write one sentence with the word because in it. So can you take '\n", - " 'the ideas from your second and third sentence and combine them '\n", - " 'with your first sentence.”\\n'\n", - " ' Correct evaluation:\\n'\n", - " ' - Student has a clear claim and relevant support.\\n'\n", - " ' - But the feedback reveals that the original task required '\n", - " 'one sentence with the word “because.”\\n'\n", - " ' - The student wrote multiple sentences, so revision is needed '\n", - " 'to meet the original task requirements.\\n'\n", - " ' - The teacher appropriately asks the student to combine the '\n", - " 'ideas into one sentence.\\n'\n", - " ' - answer = 1.\\n'\n", - " '\\n'\n", - " '3. Student: “Some people think AI-powered pets are a good '\n", - " \"alternative to real pets because they don't require additional \"\n", - " \"cost after they're purchased.”\\n\"\n", - " ' Feedback: “This is an excellent start to your response.”\\n'\n", - " ' Correct evaluation:\\n'\n", - " ' - Student has a clear claim, but the reason is overly '\n", - " 'absolute/questionable because AI pets may still have some '\n", - " 'costs.\\n'\n", - " ' - Revision is needed.\\n'\n", - " ' - The feedback gives only vague praise and does not clearly '\n", - " 'ask for revision.\\n'\n", - " ' - answer = 0.\\n'\n", - " '\\n'\n", - " 'Key features to assess:\\n'\n", - " '\\n'\n", - " '1. accurate_task_assessment\\n'\n", - " 'Judge whether the teacher feedback accurately assesses the '\n", - " 'student response’s status relative to the relevant task '\n", - " 'requirements.\\n'\n", - " '- met = 1 if the feedback’s assessment matches the actual status '\n", - " 'of the student response.\\n'\n", - " ' - Example: The student violates a revealed one-sentence '\n", - " 'requirement, and the feedback points out that requirement and '\n", - " 'asks the student to revise.\\n'\n", - " '- met = 0 if the feedback praises/accepts a flawed or incomplete '\n", - " 'response as complete, or treats a complete response as '\n", - " 'insufficient.\\n'\n", - " '\\n'\n", - " '2. revision_need_identification\\n'\n", - " 'Judge whether the feedback correctly distinguishes between '\n", - " '“revision needed” and “ready to move on.”\\n'\n", - " '- met = 1 if:\\n'\n", - " ' - the student response needs revision and the feedback clearly '\n", - " 'asks for revision, OR\\n'\n", - " ' - the student response is complete and the feedback clearly '\n", - " 'treats it as complete/ready to move on.\\n'\n", - " '- met = 0 if:\\n'\n", - " ' - the feedback asks for required revision when the response '\n", - " 'already meets all relevant task requirements,\\n'\n", - " ' - the feedback fails to ask for revision when revision is '\n", - " 'needed,\\n'\n", - " ' - the feedback is only vague praise/agreement and does not '\n", - " 'clearly identify the correct next step.\\n'\n", - " '\\n'\n", - " '3. appropriate_signal_when_task_complete\\n'\n", - " 'Judge whether, when the student response meets all relevant task '\n", - " 'requirements, the feedback clearly signals completion/move-on.\\n'\n", - " '- If the student response meets all relevant task requirements:\\n'\n", - " ' - met = 1 only if the feedback clearly says or unmistakably '\n", - " 'implies that the student has met the goal or can move on.\\n'\n", - " ' - met = 0 if the feedback merely praises the student, agrees '\n", - " 'with the content, or gives required revision directions without '\n", - " 'a move-on signal.\\n'\n", - " '- If the student response does not meet the relevant task '\n", - " 'requirements:\\n'\n", - " ' - met = 0, with a justification explaining that the feature is '\n", - " 'not applicable because the task is not complete.\\n',\n", - " 'id': None,\n", - " 'name': None,\n", - " 'response_metadata': {},\n", - " 'type': 'system'},\n", - " {'additional_kwargs': {},\n", - " 'content': 'Analyze:\\n'\n", - " 'Student text: Some people think AI-powered pets are a good '\n", - " 'alternative to real pets because they could help around the '\n", - " 'house etc.\\n'\n", - " \"Feedback text: You're right, the AI pets could help around the \"\n", - " 'house. Can you find some other details from the article that you '\n", - " 'could add to make your claim stronger?\\n',\n", - " 'id': None,\n", - " 'name': None,\n", - " 'response_metadata': {},\n", - " 'type': 'human'}]\n", - "\n", - "============================================================\n", - "RAW LLM TEXT (model's verbatim output)\n", - "============================================================\n", - "{\"reasoning\":\"The relevant task goal is to produce a claim with at least one relevant supporting reason or piece of evidence. No additional format requirement is revealed in the feedback. The student response includes a clear claim: AI-powered pets are a good alternative to real pets. It also includes a relevant supporting reason: they could help around the house. Although the wording ends with 'etc.,' the response still gives at least one concrete, relevant reason, so it meets the basic claim-and-support requirement. The teacher feedback, however, asks the student to add other details from the article 'to make your claim stronger,' which frames revision as needed rather than optional. Because the student already meets the relevant task requirements and the feedback does not clearly signal that the student is ready to move on, the feedback makes an inappropriate revise/move-on decision.\",\"key_features\":{\"accurate_task_assessment\":{\"met\":0,\"justification\":\"The student already provides a claim plus a relevant supporting reason, so the response meets the basic task requirement. The feedback treats the response as needing more added detail to satisfy the task, which overstates the need for revision.\"},\"revision_need_identification\":{\"met\":0,\"justification\":\"Since the student response is complete enough to meet the task, the feedback should clearly indicate completion or make enrichment optional. Instead, it asks the student to add more details, signaling required revision when revision is not necessary.\"},\"appropriate_signal_when_task_complete\":{\"met\":0,\"justification\":\"This feature applies because the student response meets the task goal. The feedback offers agreement and then asks for additional details, but it never clearly signals that the student has met the goal or can move on.\"}},\"proposed_adjustment\":\"Revise the feedback to clearly acknowledge that the student has met the goal and make any extension optional, for example: 'You have a clear claim and a relevant reason, so you’ve met the task goal. If you want to make your argument even stronger, you could add another detail from the article.'\",\"appropriate_feedback_score\":0}\n", - "\n", - "============================================================\n", - "PARSED OUTPUT (output_schema)\n", - "============================================================\n", - "{'appropriate_feedback_score': 0,\n", - " 'key_features': {'accurate_task_assessment': {'justification': 'The student '\n", - " 'already '\n", - " 'provides a '\n", - " 'claim plus a '\n", - " 'relevant '\n", - " 'supporting '\n", - " 'reason, so '\n", - " 'the response '\n", - " 'meets the '\n", - " 'basic task '\n", - " 'requirement. '\n", - " 'The feedback '\n", - " 'treats the '\n", - " 'response as '\n", - " 'needing more '\n", - " 'added detail '\n", - " 'to satisfy '\n", - " 'the task, '\n", - " 'which '\n", - " 'overstates '\n", - " 'the need for '\n", - " 'revision.',\n", - " 'met': 0},\n", - " 'appropriate_signal_when_task_complete': {'justification': 'This '\n", - " 'feature '\n", - " 'applies '\n", - " 'because '\n", - " 'the '\n", - " 'student '\n", - " 'response '\n", - " 'meets '\n", - " 'the '\n", - " 'task '\n", - " 'goal. '\n", - " 'The '\n", - " 'feedback '\n", - " 'offers '\n", - " 'agreement '\n", - " 'and '\n", - " 'then '\n", - " 'asks '\n", - " 'for '\n", - " 'additional '\n", - " 'details, '\n", - " 'but '\n", - " 'it '\n", - " 'never '\n", - " 'clearly '\n", - " 'signals '\n", - " 'that '\n", - " 'the '\n", - " 'student '\n", - " 'has '\n", - " 'met '\n", - " 'the '\n", - " 'goal '\n", - " 'or '\n", - " 'can '\n", - " 'move '\n", - " 'on.',\n", - " 'met': 0},\n", - " 'revision_need_identification': {'justification': 'Since the '\n", - " 'student '\n", - " 'response '\n", - " 'is '\n", - " 'complete '\n", - " 'enough to '\n", - " 'meet the '\n", - " 'task, the '\n", - " 'feedback '\n", - " 'should '\n", - " 'clearly '\n", - " 'indicate '\n", - " 'completion '\n", - " 'or make '\n", - " 'enrichment '\n", - " 'optional. '\n", - " 'Instead, '\n", - " 'it asks '\n", - " 'the '\n", - " 'student '\n", - " 'to add '\n", - " 'more '\n", - " 'details, '\n", - " 'signaling '\n", - " 'required '\n", - " 'revision '\n", - " 'when '\n", - " 'revision '\n", - " 'is not '\n", - " 'necessary.',\n", - " 'met': 0}},\n", - " 'proposed_adjustment': 'Revise the feedback to clearly acknowledge that the '\n", - " 'student has met the goal and make any extension '\n", - " \"optional, for example: 'You have a clear claim and a \"\n", - " 'relevant reason, so you’ve met the task goal. If you '\n", - " 'want to make your argument even stronger, you could '\n", - " \"add another detail from the article.'\",\n", - " 'reasoning': 'The relevant task goal is to produce a claim with at least one '\n", - " 'relevant supporting reason or piece of evidence. No additional '\n", - " 'format requirement is revealed in the feedback. The student '\n", - " 'response includes a clear claim: AI-powered pets are a good '\n", - " 'alternative to real pets. It also includes a relevant '\n", - " 'supporting reason: they could help around the house. Although '\n", - " \"the wording ends with 'etc.,' the response still gives at least \"\n", - " 'one concrete, relevant reason, so it meets the basic '\n", - " 'claim-and-support requirement. The teacher feedback, however, '\n", - " \"asks the student to add other details from the article 'to make \"\n", - " \"your claim stronger,' which frames revision as needed rather \"\n", - " 'than optional. Because the student already meets the relevant '\n", - " 'task requirements and the feedback does not clearly signal that '\n", - " 'the student is ready to move on, the feedback makes an '\n", - " 'inappropriate revise/move-on decision.'}\n", - "\n", - "============================================================\n", - "USAGE METADATA\n", - "============================================================\n", - "{'input_token_details': {'audio': 0, 'cache_read': 0},\n", - " 'input_tokens': 2463,\n", - " 'output_token_details': {'audio': 0, 'reasoning': 0},\n", - " 'output_tokens': 420,\n", - " 'total_tokens': 2883}\n" - ] - } - ], + "outputs": [], "source": [ "import pprint as pp\n", "# I/O trace breakdown -- this is what the SDK engineer will replicate in TS.\n", @@ -718,34 +227,9 @@ }, { "cell_type": "code", - "execution_count": 17, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "421b04bd-a7bb-41c7-ba8b-0716ae03ff82", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Sniff-test fixture runner" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loaded 2 fixtures from fixtures.json\n", - "\n", - "==============================================================================\n", - " ID STATUS PREDICTED EXPECTED DESCRIPTION\n", - "==============================================================================\n", - " 0 PASS 1 1 Example 1: Student writes\n", - " 1 PASS 1 1 Example 2: Student writes\n", - "==============================================================================\n", - "Summary: 2 exact, 0 adjacent, 0 fail, 0 error -- total 2\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"sniff_test_path\"]\n", "if not fixtures_path.exists():\n", diff --git a/evals/feedback/productive-coaching-writing-feedback/manageable/example_notebook.ipynb b/evals/feedback/productive-coaching-writing-feedback/manageable/example_notebook.ipynb index db0ff308..8324ab35 100644 --- a/evals/feedback/productive-coaching-writing-feedback/manageable/example_notebook.ipynb +++ b/evals/feedback/productive-coaching-writing-feedback/manageable/example_notebook.ipynb @@ -2,16 +2,7 @@ "cells": [ { "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "019a2e29-26c8-4cdd-8d6b-88de27b95384", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "source": [ "# Manageable Feedback Evaluator\n", "\n", @@ -35,16 +26,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "1958719e-0bd2-43fa-a6a0-ca1d090da428", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "outputs": [], "source": [ "import getpass\n", @@ -80,16 +62,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "c2ba0d01-9d12-40f7-9057-22df59eaeb05", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Load config + verify prompt hashes" - } - }, + "metadata": {}, "outputs": [], "source": [ "# -------------------------------------------------------------------------\n", @@ -144,16 +117,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "4e8def3b-028c-4b71-acee-13fb1553b2ef", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Evaluator function (config-driven)" - } - }, + "metadata": {}, "outputs": [], "source": [ "_STEP = CONFIG[\"steps\"][0]\n", @@ -220,16 +184,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "45aff123-c2d0-4041-9737-f06ebbdceee4", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Smoke test on a sample" - } - }, + "metadata": {}, "outputs": [], "source": [ "sample_student_text = \"\"\"Some people think AI-powered pets are a good alternative to real pets because they could help around the house etc.\"\"\"\n", @@ -273,16 +228,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "421b04bd-a7bb-41c7-ba8b-0716ae03ff82", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Sniff-test fixture runner" - } - }, + "metadata": {}, "outputs": [], "source": [ "fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"sniff_test_path\"]\n", diff --git a/evals/grade_level_evaluator.ipynb b/evals/grade_level_evaluator.ipynb index f4faafd6..111be3dd 100644 --- a/evals/grade_level_evaluator.ipynb +++ b/evals/grade_level_evaluator.ipynb @@ -17,45 +17,16 @@ }, { "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "7f05c49f-1811-4e4c-9654-45bd1319bae6", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "source": [ "### Install required python packages" ] }, { "cell_type": "code", - "execution_count": 5, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "9fcf1049-2775-4255-a184-6e9452f507b2", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Note: you may need to restart the kernel to use updated packages.\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "#Install relevant packages\n", "%pip install -qU pydantic dotenv langchain langchain_google_genai" @@ -63,20 +34,8 @@ }, { "cell_type": "code", - "execution_count": 6, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "03133834-8d82-4d62-a1f8-0ae8eae5ad18", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "#Load necessary packages\n", @@ -93,36 +52,15 @@ }, { "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "02de88b4-acd9-4f33-991c-43d38ac227ee", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "source": [ "### Preparing prompt " ] }, { "cell_type": "code", - "execution_count": 7, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "cc1d5155-7bfc-4306-b3cd-6cb44388ab54", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "create prompts" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "#This is the system prompt, user prompt and model output setting. Do not change this\n", @@ -145,16 +83,7 @@ }, { "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "c5735352-65ab-40bf-a891-1fb2997456c2", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "source": [ "### Set up the Vocabulary Evaluator function" ] @@ -162,19 +91,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "b5707aa1-b8b9-4b3b-815d-b84efaf75b5d", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "outputs": [], "source": [ "# Check for the API key\n", @@ -225,16 +142,7 @@ }, { "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "1f1a8875-d082-4768-a689-e565cebcc475", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "source": [ "# Try evaluating text to determine appropriate grade level\n", "\n", @@ -245,46 +153,9 @@ }, { "cell_type": "code", - "execution_count": 11, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "f870829e-7aab-46c4-9914-58427e820d91", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - " Target grade band is: 9-10 \n", - "\n", - " Alternative grade band is: 6-8 \n", - "\n", - " Recommended scaffolding for alternative grade band is: For a 6-8 grade audience, this text would require significant scaffolding. This includes: pre-teaching key vocabulary (gravitational, amplitude, semi-diurnal, diurnal), providing definitions or omitting the most technical terms ('amphidromic systems', 'bathymetry'), using diagrams to illustrate the Earth-Moon-Sun system and different tide types, and chunking the text to break down complex sentences. \n", - "\n", - " Reasoning: \n", - "1. **Quantitative Analysis**: The text has a word count of 155, which is too low to be a reliable indicator using the provided word count bands. The Flesch-Kincaid Grade Level calculation results in approximately 11.9. This high score is driven by long sentences (average of 17.2 words per sentence, with one sentence being 43 words long) and a high number of polysyllabic, technical words (average of 1.76 syllables per word). This quantitative measure strongly suggests a high school grade level.\n", - "2. **Qualitative Analysis**: \n", - " * **Text Structure**: Moderately Complex. The text follows a logical, informational structure (definition, influencing factors, types, measurement), but the second sentence lists several unexplained, complex concepts, making the connections between ideas implicitly difficult for a novice.\n", - " * **Language Features**: Very Complex. The text uses dense, academic language. The vocabulary is highly subject-specific and challenging (e.g., 'gravitational', 'amplitude', 'amphidromic systems', 'bathymetry', 'semi-diurnal', 'diurnal'). Sentence structure is also complex, particularly in the first two sentences.\n", - " * **Purpose**: Slightly Complex. The purpose is clearly and explicitly informational—to define and explain tides.\n", - " * **Knowledge Demands**: Very Complex. Full comprehension requires significant discipline-specific knowledge in Earth Science or oceanography, including concepts of gravity and the specific terminology used.\n", - "3. **Background Knowledge**: Students are typically introduced to the basic concept of tides (caused by the Moon's gravity) in middle school (grades 6-8). However, the level of detail, the inclusion of the Sun's role, and the highly technical vocabulary in this text align more with a high school Earth Science curriculum (grades 9-12).\n", - "4. **Synthesis**: The quantitative Flesch-Kincaid score points towards 11th-12th grade. The qualitative analysis confirms this is a complex text, primarily due to its very complex language and knowledge demands. While a middle school student might grasp the first sentence, the rest of the text, especially the technical terms, would be inaccessible without significant support. For independent reading, the text is most appropriate for a high school student who has the foundational science knowledge to decode the vocabulary and concepts. Therefore, the 9-10 grade band is a suitable placement, serving as a challenging text in a science context.\n", - "\n", - " \n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "#Swap out text here for your own text to evaluate\n", "text = \"\"\"\n", diff --git a/evals/sentence_structure_evaluator.ipynb b/evals/sentence_structure_evaluator.ipynb index 0ecbb2b1..84385bfd 100644 --- a/evals/sentence_structure_evaluator.ipynb +++ b/evals/sentence_structure_evaluator.ipynb @@ -2,19 +2,7 @@ "cells": [ { "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "db053f1c-c362-4e97-811c-fe5d9f44ec12", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "source": [ "## Sentence Structure Evaluator (Early Release)\n", "\n" @@ -29,23 +17,8 @@ }, { "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "d56d1000-1d88-48a2-83ce-5d84d9619c53", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - }, - "jupyter": { - "outputs_hidden": true - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "%pip install langchain_openai openai langchain textstat" @@ -54,19 +27,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "321d26ed-6745-447c-b7b1-21ea8d4e4679", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "outputs": [], "source": [ "# Load packages\n", @@ -100,19 +61,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "50031385-d966-41cf-9189-91517f8b0846", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "outputs": [], "source": [ "from prompts import sent_str_prompts as prompts\n", @@ -139,19 +88,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "3346f5f2-40c7-4771-8bab-1411c6f0af31", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "outputs": [], "source": [ "class SentenceAnalysesEvaluatorOutput(BaseModel):\n", @@ -298,19 +235,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "b93024b0-e428-45a9-9a50-d1811422833d", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "outputs": [], "source": [ "async def execute_sentence_analysis(text: str) -> dict:\n", @@ -417,19 +342,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "73dbdc7a-f190-4854-b3ac-4d9b95cf36df", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "outputs": [], "source": [ "def safe_literal_eval(s):\n", @@ -723,19 +636,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "42bf25ef-3ddf-4cd2-813a-4e793f91321b", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "outputs": [], "source": [ "async def classify_complexity_with_grade_level(sentence_features, grade, excerpt):\n", @@ -775,19 +676,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "a489acba-781d-4422-8d7b-0a7df15935fa", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "outputs": [], "source": [ "async def evaluate_df(df: pd.DataFrame, concurrency: int = 5):\n", @@ -860,19 +749,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "5cd2caf5-73c6-40e2-8a2e-d2b892b943d4", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "outputs": [], "source": [ "async def predict_text_complexity_level(text: str, grade: int):\n", diff --git a/evals/text_complexity_combo.ipynb b/evals/text_complexity_combo.ipynb index 7bf28ddd..aeac5c50 100644 --- a/evals/text_complexity_combo.ipynb +++ b/evals/text_complexity_combo.ipynb @@ -19,16 +19,7 @@ }, { "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "5d3ac2f5-40b2-4b2c-afdb-c7400e603af3", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "source": [ "### Install & Load necessary packages" ] @@ -36,19 +27,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "f24faefe-74b5-4c6c-9052-b41d1e9b73cf", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "outputs": [], "source": [ "%pip install -qU pydantic textstat langchain langchain_openai langchain-google-genai" @@ -57,19 +36,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "b243b748-0ed3-4d19-ab1b-d43638d2113e", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "outputs": [], "source": [ "# Load packages\n", @@ -95,16 +62,7 @@ }, { "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "12709664-6132-48a8-b621-73658b4fea90", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "source": [ "### Set up the evaluator's model and prompts" ] @@ -112,19 +70,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "eeffec80-067e-47de-a234-7dcd5078ea53", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "outputs": [], "source": [ "from prompts import vocab_prompts as v_prompts, sent_str_prompts as s_prompts, smk_prompts as smk_prompts, conventionality_prompts as conv_prompts\n", @@ -173,19 +119,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "3fc5ec95-f619-439e-9286-663f3d843502", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "outputs": [], "source": [ "async def get_background_knowledge_assumption(text, grade):\n", @@ -205,22 +139,85 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "04d83e90-b786-4f2d-83d6-8c69c2cf176b", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "outputs": [], "source": [ - "class VocabOutput(BaseModel):\n tier_2_words: str = Field(description=\"List of Tier 2 words\")\n tier_3_words: str = Field(description=\"List of Tier 3 words\")\n archaic_words: str = Field(description=\"List of Archaic words\")\n other_complex_words: str = Field(description=\"List of Other Complex words\")\n complexity_score: str = Field(\n description=\"the complexity of the text, one of: slightly complex, moderately complex, very complex, or exceedingly complex\"\n )\n reasoning: str = Field(description=\"your reasoning for your answer\")\n\n\nprompt_vars = {\n \"inputVars\": [\n \"text\",\n \"student_grade_level\",\n \"student_background_knowledge\",\n \"fk_level\",\n ],\n \"outputParser\": JsonOutputParser(pydantic_object=VocabOutput),\n}\n\n\nclass SmkOutput(BaseModel):\n identified_topics: List[str] = Field(\n description=\"List of major subjects/concepts found in the text.\"\n )\n curriculum_check: str = Field(\n description=\"Analysis of whether these topics are new or review based on the Grade Level and Reference list.\"\n )\n assumptions_and_scaffolding: str = Field(\n description=\"Analysis of what the author assumes vs. what is explained (and if definitions are provided).\"\n )\n friction_analysis: str = Field(\n description=\"Explicit statement distinguishing if difficulty comes from Vocabulary/Structure or actual Knowledge.\"\n )\n complexity_score: Literal[\n \"slightly_complex\",\n \"moderately_complex\",\n \"very_complex\",\n \"exceedingly_complex\",\n ] = Field(description=\"The subject matter knowledge complexity level of the text\")\n reasoning: str = Field(\n description=\"A brief synthesis of why the text fits the chosen complexity level.\"\n )\n\n\nsmk_prompt_vars = {\n \"inputVars\": [\"text\", \"grade\", \"fk_score\"],\n \"outputParser\": JsonOutputParser(pydantic_object=SmkOutput),\n}\n\nclass ConventionalityOutput(BaseModel):\n conventionality_features: List[str] = Field(\n description=\"List of the specific language features driving the complexity (e.g., idioms, metaphors, implied meaning) with direct quotes from the text.\"\n )\n grade_context: str = Field(\n description=\"How the conventionality demands compare to general expectations for the provided target grade.\"\n )\n instructional_insights: str = Field(\n description=\"Actionable pedagogical suggestions for scaffolding the unconventional language features in the classroom.\"\n )\n complexity_score: Literal[\n \"slightly_complex\",\n \"moderately_complex\",\n \"very_complex\",\n \"exceedingly_complex\",\n ] = Field(description=\"The conventionality complexity level of the text\")\n reasoning: str = Field(\n description=\"A synthesis of why the text fits the chosen rubric level.\"\n )\n\n\nconv_prompt_vars = {\n \"inputVars\": [\"text\", \"grade\", \"fk_score\"],\n \"outputParser\": JsonOutputParser(pydantic_object=ConventionalityOutput),\n}" + "class VocabOutput(BaseModel):\n", + " tier_2_words: str = Field(description=\"List of Tier 2 words\")\n", + " tier_3_words: str = Field(description=\"List of Tier 3 words\")\n", + " archaic_words: str = Field(description=\"List of Archaic words\")\n", + " other_complex_words: str = Field(description=\"List of Other Complex words\")\n", + " complexity_score: str = Field(\n", + " description=\"the complexity of the text, one of: slightly complex, moderately complex, very complex, or exceedingly complex\"\n", + " )\n", + " reasoning: str = Field(description=\"your reasoning for your answer\")\n", + "\n", + "\n", + "prompt_vars = {\n", + " \"inputVars\": [\n", + " \"text\",\n", + " \"student_grade_level\",\n", + " \"student_background_knowledge\",\n", + " \"fk_level\",\n", + " ],\n", + " \"outputParser\": JsonOutputParser(pydantic_object=VocabOutput),\n", + "}\n", + "\n", + "\n", + "class SmkOutput(BaseModel):\n", + " identified_topics: List[str] = Field(\n", + " description=\"List of major subjects/concepts found in the text.\"\n", + " )\n", + " curriculum_check: str = Field(\n", + " description=\"Analysis of whether these topics are new or review based on the Grade Level and Reference list.\"\n", + " )\n", + " assumptions_and_scaffolding: str = Field(\n", + " description=\"Analysis of what the author assumes vs. what is explained (and if definitions are provided).\"\n", + " )\n", + " friction_analysis: str = Field(\n", + " description=\"Explicit statement distinguishing if difficulty comes from Vocabulary/Structure or actual Knowledge.\"\n", + " )\n", + " complexity_score: Literal[\n", + " \"slightly_complex\",\n", + " \"moderately_complex\",\n", + " \"very_complex\",\n", + " \"exceedingly_complex\",\n", + " ] = Field(description=\"The subject matter knowledge complexity level of the text\")\n", + " reasoning: str = Field(\n", + " description=\"A brief synthesis of why the text fits the chosen complexity level.\"\n", + " )\n", + "\n", + "\n", + "smk_prompt_vars = {\n", + " \"inputVars\": [\"text\", \"grade\", \"fk_score\"],\n", + " \"outputParser\": JsonOutputParser(pydantic_object=SmkOutput),\n", + "}\n", + "\n", + "class ConventionalityOutput(BaseModel):\n", + " conventionality_features: List[str] = Field(\n", + " description=\"List of the specific language features driving the complexity (e.g., idioms, metaphors, implied meaning) with direct quotes from the text.\"\n", + " )\n", + " grade_context: str = Field(\n", + " description=\"How the conventionality demands compare to general expectations for the provided target grade.\"\n", + " )\n", + " instructional_insights: str = Field(\n", + " description=\"Actionable pedagogical suggestions for scaffolding the unconventional language features in the classroom.\"\n", + " )\n", + " complexity_score: Literal[\n", + " \"slightly_complex\",\n", + " \"moderately_complex\",\n", + " \"very_complex\",\n", + " \"exceedingly_complex\",\n", + " ] = Field(description=\"The conventionality complexity level of the text\")\n", + " reasoning: str = Field(\n", + " description=\"A synthesis of why the text fits the chosen rubric level.\"\n", + " )\n", + "\n", + "\n", + "conv_prompt_vars = {\n", + " \"inputVars\": [\"text\", \"grade\", \"fk_score\"],\n", + " \"outputParser\": JsonOutputParser(pydantic_object=ConventionalityOutput),\n", + "}" ] }, { @@ -351,19 +348,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "23047ec3-79b2-4e45-b452-873eae41b030", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "outputs": [], "source": [ "def calculate_fk_score(text) -> float:\n", @@ -946,7 +931,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1072,16 +1057,7 @@ }, { "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "4437f840-a00b-47cf-a7fa-86d882f0f6af", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "source": [ "# Test out examples" ] @@ -1089,19 +1065,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "767fe555-ef65-46fb-8da3-54f18565cb07", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "outputs": [], "source": [ "# Add your text & the grade level you want to evaluate for vocabulary complexity\n", @@ -1185,4 +1149,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} \ No newline at end of file +} diff --git a/evals/vocabulary_evaluator.ipynb b/evals/vocabulary_evaluator.ipynb index 892ecdf4..4ef1030e 100644 --- a/evals/vocabulary_evaluator.ipynb +++ b/evals/vocabulary_evaluator.ipynb @@ -18,16 +18,7 @@ }, { "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "5d3ac2f5-40b2-4b2c-afdb-c7400e603af3", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "source": [ "### Install & Load necessary packages" ] @@ -35,19 +26,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "f24faefe-74b5-4c6c-9052-b41d1e9b73cf", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "outputs": [], "source": [ "%pip install -qU pydantic textstat langchain langchain_openai langchain-google-genai" @@ -56,19 +35,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "b243b748-0ed3-4d19-ab1b-d43638d2113e", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "outputs": [], "source": [ "# Load packages\n", @@ -88,16 +55,7 @@ }, { "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "12709664-6132-48a8-b621-73658b4fea90", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "source": [ "### Set up the evaluator's model and prompts" ] @@ -105,19 +63,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "eeffec80-067e-47de-a234-7dcd5078ea53", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "outputs": [], "source": [ "from prompts import vocab_prompts as prompts\n", @@ -165,19 +111,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "3fc5ec95-f619-439e-9286-663f3d843502", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "outputs": [], "source": [ "def get_background_knowledge_assumption(text, grade):\n", @@ -197,19 +131,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "04d83e90-b786-4f2d-83d6-8c69c2cf176b", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "outputs": [], "source": [ "class Output(BaseModel):\n", @@ -244,19 +166,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "23047ec3-79b2-4e45-b452-873eae41b030", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "outputs": [], "source": [ "import textwrap\n", @@ -432,16 +342,7 @@ }, { "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "4437f840-a00b-47cf-a7fa-86d882f0f6af", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "source": [ "# Test out examples" ] @@ -449,19 +350,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "767fe555-ef65-46fb-8da3-54f18565cb07", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "outputs": [], "source": [ "# Add your text & the grade level you want to evaluate for vocabulary complexity\n", diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 00000000..aa5c6900 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,59 @@ +# scripts/ — repo check harness + +Repo-wide checks that keep contributions consistent. The **same checks run +locally and in CI**, so what you run here is exactly what gates a PR. + +## Setup + +```bash +pip install -r scripts/requirements.txt # one-time; pinned tools the checks use +``` + +## Usage + +```bash +python scripts/check.py # verify everything (what CI runs) +python scripts/check.py --fix # auto-fix what's safe, report the rest +python scripts/check.py --list # list available checks +python scripts/check.py strip-notebooks # run a single check +``` + +**Run `python scripts/check.py --fix` before you commit.** It catches issues +early with clear messages and fixes what it can. CI runs the same harness and +fails the PR (it never auto-fixes) — so fixing locally saves a round-trip. + +## How it works + +- One check per concern, each with two modes: **`--check`** (verify, non-zero + exit on problems) and **`--fix`** (repair in place where safe). +- The orchestrator runs **every** check and reports **all** failures at once — + no stop-at-first, no whack-a-mole. +- Tools the checks rely on are pinned in `requirements.txt` so local and CI + behave identically. + +## Checks + +| Check | What it does | +|-------|--------------| +| `strip-notebooks` | Clears outputs, execution counts, and cell metadata from `.ipynb` files | + +## Coming next + +This is the seed of a broader self-service harness. Planned additions: + +- **eval-config validation** — parser kind, prompt `sha256` drift, no dangling + placeholders, fixture labels within the output schema enum. +- **naming / structure conventions** for `evals/`. +- **delegation to the SDK suites** (`sdks/typescript` lint/type/test, + `sdks/python` Makefile) so one local command covers everything. CI keeps the + per-SDK workflows separate for parallelism and path-filtered signal. + +> **TODO** (once there are more checks): surface this from the root `README.md` +> / `CONTRIBUTING.md` so newcomers discover it — e.g. "Before committing, run +> `python scripts/check.py --fix` (see `scripts/README.md`)." + +## Adding a check + +Add a `Check` subclass in `scripts/checks/` (see `strip_notebooks.py`) and +register it in `scripts/checks/__init__.py`. It's automatically picked up by +the orchestrator and CI. diff --git a/scripts/check.py b/scripts/check.py new file mode 100755 index 00000000..8e5b320f --- /dev/null +++ b/scripts/check.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Repo check harness — the single entrypoint run locally and in CI. + + python scripts/check.py # verify everything (CI mode) + python scripts/check.py --fix # auto-fix what's safe, report the rest + python scripts/check.py --list # list available checks + python scripts/check.py NAME ... # run only the named check(s) + +Runs every selected check, never stopping at the first failure, and prints one +consolidated report so you see all issues at once. Exit code is non-zero if any +violation remains. See scripts/README.md. +""" + +from __future__ import annotations + +import argparse +import sys + +from checks import ALL_CHECKS + + +def main() -> int: + by_name = {c.name: c for c in ALL_CHECKS} + parser = argparse.ArgumentParser(description="Repo check harness") + parser.add_argument("checks", nargs="*", help="checks to run (default: all)") + parser.add_argument("--fix", action="store_true", help="auto-fix where possible") + parser.add_argument("--list", action="store_true", help="list checks and exit") + args = parser.parse_args() + + if args.list: + for c in ALL_CHECKS: + print(f" {c.name:20} {c.description}") + return 0 + + unknown = [n for n in args.checks if n not in by_name] + if unknown: + print(f"unknown check(s): {', '.join(unknown)}", file=sys.stderr) + print(f"available: {', '.join(by_name)}", file=sys.stderr) + return 2 + + selected = [by_name[n] for n in args.checks] if args.checks else ALL_CHECKS + + results = [c.run(fix=args.fix) for c in selected] + + failures = 0 + for r in results: + for path in r.fixed: + print(f"[fixed] {r.check}: {path}") + if r.ok: + print(f"[pass] {r.check}") + else: + failures += 1 + print(f"[fail] {r.check} ({len(r.violations)})") + for v in r.violations: + print(f" {v.render()}") + + print() + if failures: + print(f"{failures} of {len(results)} checks failed.") + if not args.fix: + print("Run `python scripts/check.py --fix` to auto-fix what's possible.") + return 1 + print(f"All {len(results)} checks passed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/checks/__init__.py b/scripts/checks/__init__.py new file mode 100644 index 00000000..927c2a8c --- /dev/null +++ b/scripts/checks/__init__.py @@ -0,0 +1,7 @@ +"""Check registry. Add new checks here to wire them into the harness + CI.""" + +from .strip_notebooks import StripNotebooks + +ALL_CHECKS = [ + StripNotebooks(), +] diff --git a/scripts/checks/base.py b/scripts/checks/base.py new file mode 100644 index 00000000..dfb82a42 --- /dev/null +++ b/scripts/checks/base.py @@ -0,0 +1,50 @@ +"""Shared contract for repo checks. + +Every check is a `Check` subclass that implements `run(fix)` and returns a +`Result`. A check must, in one pass: + - report *every* violation it finds (no stop-at-first), and + - in fix mode, repair what it safely can and report what's left. +""" + +from __future__ import annotations + +import subprocess +from dataclasses import dataclass, field + + +@dataclass +class Violation: + path: str + message: str + line: int | None = None + + def render(self) -> str: + loc = self.path if self.line is None else f"{self.path}:{self.line}" + return f"{loc} — {self.message}" + + +@dataclass +class Result: + check: str + violations: list[Violation] = field(default_factory=list) + fixed: list[str] = field(default_factory=list) + + @property + def ok(self) -> bool: + return not self.violations + + +class Check: + """Base class. `name` is the CLI selector; `description` shows in --list.""" + + name: str = "" + description: str = "" + + def run(self, fix: bool) -> Result: # pragma: no cover - interface + raise NotImplementedError + + +def tracked_files(pattern: str) -> list[str]: + """Git-tracked paths matching `pattern`, excluding checkpoint dirs.""" + out = subprocess.check_output(["git", "ls-files", pattern], text=True) + return [p for p in out.splitlines() if ".ipynb_checkpoints" not in p] diff --git a/scripts/checks/strip_notebooks.py b/scripts/checks/strip_notebooks.py new file mode 100644 index 00000000..2bdba539 --- /dev/null +++ b/scripts/checks/strip_notebooks.py @@ -0,0 +1,58 @@ +"""Strip transient state from Jupyter notebooks, via nbstripout. + +Drives nbstripout (pinned in scripts/requirements.txt) so we reuse its +battle-tested handling instead of reimplementing it. Flags chosen to match the +repo's intent: + --keep-id don't regenerate cell ids (avoids churn on otherwise-clean + notebooks) + --extra-keys ... also strip the Databricks and Jupyter (outputs_hidden) cell + metadata that nbstripout keeps by default +nbstripout already clears outputs and execution counts. Notebook-level metadata +(kernelspec, language_info) is left intact. +""" + +from __future__ import annotations + +import shutil +import subprocess + +from .base import Check, Result, Violation, tracked_files + +# Cell metadata nbstripout does not strip by default but we want gone. +_EXTRA_KEYS = "cell.metadata.application/vnd.databricks.v1+cell cell.metadata.jupyter" +_FLAGS = ["--keep-id", "--extra-keys", _EXTRA_KEYS] + + +class StripNotebooks(Check): + name = "strip-notebooks" + description = "Clear outputs, execution counts, and cell metadata from notebooks (nbstripout)" + + def run(self, fix: bool) -> Result: + result = Result(self.name) + + if shutil.which("nbstripout") is None: + result.violations.append( + Violation( + "scripts/requirements.txt", + "nbstripout not found; run `pip install -r scripts/requirements.txt`", + ) + ) + return result + + for path in tracked_files("*.ipynb"): + # --verify is a dry run: non-zero exit means the file would change. + clean = subprocess.run( + ["nbstripout", "--verify", *_FLAGS, path], + capture_output=True, + ).returncode == 0 + if clean: + continue + + if fix: + subprocess.run(["nbstripout", *_FLAGS, path], capture_output=True, check=True) + result.fixed.append(path) + else: + result.violations.append( + Violation(path, "has outputs/execution counts/cell metadata; run with --fix") + ) + return result diff --git a/scripts/requirements.txt b/scripts/requirements.txt new file mode 100644 index 00000000..725bc3d1 --- /dev/null +++ b/scripts/requirements.txt @@ -0,0 +1,3 @@ +# Dependencies for the repo check harness (scripts/check.py). +# Pinned so local runs and CI strip notebooks identically. +nbstripout==0.9.1 From 342b46cffb39c71b20307ce7723d4597987cc65e Mon Sep 17 00:00:00 2001 From: Adnan Rashid Hussain Date: Thu, 25 Jun 2026 16:54:06 -0700 Subject: [PATCH 2/2] fix: harden check harness against per-check and subprocess failures --- scripts/README.md | 6 +++--- scripts/check.py | 12 ++++++++++-- scripts/checks/strip_notebooks.py | 9 +++++++-- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/scripts/README.md b/scripts/README.md index aa5c6900..d3f120ef 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -12,7 +12,7 @@ pip install -r scripts/requirements.txt # one-time; pinned tools the checks us ## Usage ```bash -python scripts/check.py # verify everything (what CI runs) +python scripts/check.py # check everything (what CI runs) python scripts/check.py --fix # auto-fix what's safe, report the rest python scripts/check.py --list # list available checks python scripts/check.py strip-notebooks # run a single check @@ -24,8 +24,8 @@ fails the PR (it never auto-fixes) — so fixing locally saves a round-trip. ## How it works -- One check per concern, each with two modes: **`--check`** (verify, non-zero - exit on problems) and **`--fix`** (repair in place where safe). +- One check per concern, two modes: **check** (the default — non-zero exit on + problems) and **`--fix`** (repair in place where safe). - The orchestrator runs **every** check and reports **all** failures at once — no stop-at-first, no whack-a-mole. - Tools the checks rely on are pinned in `requirements.txt` so local and CI diff --git a/scripts/check.py b/scripts/check.py index 8e5b320f..2c4cb537 100755 --- a/scripts/check.py +++ b/scripts/check.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Repo check harness — the single entrypoint run locally and in CI. - python scripts/check.py # verify everything (CI mode) + python scripts/check.py # check everything (CI mode) python scripts/check.py --fix # auto-fix what's safe, report the rest python scripts/check.py --list # list available checks python scripts/check.py NAME ... # run only the named check(s) @@ -17,6 +17,7 @@ import sys from checks import ALL_CHECKS +from checks.base import Result, Violation def main() -> int: @@ -40,7 +41,14 @@ def main() -> int: selected = [by_name[n] for n in args.checks] if args.checks else ALL_CHECKS - results = [c.run(fix=args.fix) for c in selected] + results = [] + for c in selected: + try: + results.append(c.run(fix=args.fix)) + except Exception as e: # one check crashing must not abort the rest + crashed = Result(c.name) + crashed.violations.append(Violation(c.name, f"check crashed: {e}")) + results.append(crashed) failures = 0 for r in results: diff --git a/scripts/checks/strip_notebooks.py b/scripts/checks/strip_notebooks.py index 2bdba539..7aabbaaa 100644 --- a/scripts/checks/strip_notebooks.py +++ b/scripts/checks/strip_notebooks.py @@ -49,8 +49,13 @@ def run(self, fix: bool) -> Result: continue if fix: - subprocess.run(["nbstripout", *_FLAGS, path], capture_output=True, check=True) - result.fixed.append(path) + proc = subprocess.run(["nbstripout", *_FLAGS, path], capture_output=True, text=True) + if proc.returncode == 0: + result.fixed.append(path) + else: # report and keep going, don't abort the whole pass + result.violations.append( + Violation(path, f"nbstripout failed: {proc.stderr.strip() or 'non-zero exit'}") + ) else: result.violations.append( Violation(path, "has outputs/execution counts/cell metadata; run with --fix")