Skip to content

Commit 66a49f4

Browse files
abossardCopilot
andcommitted
Fix token_f1 examples to show precision vs recall tradeoff
Old example had precision=recall=0.5 which doesn't teach anything. New examples show: cautious (high P, low R), overeager (low P, high R), perfect, and completely wrong. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 9c9beba commit 66a49f4

1 file changed

Lines changed: 28 additions & 13 deletions

File tree

notebooks/01_evaluation.ipynb

Lines changed: 28 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -76,28 +76,43 @@
7676
"metadata": {},
7777
"outputs": [],
7878
"source": [
79-
"# Metriken werden immer raffinierter\n",
8079
"from dspy_tasks.calculations import token_f1\n",
8180
"\n",
82-
"f1_val = token_f1([\"apple\", \"banana\"], [\"apple\", \"cherry\"])\n",
83-
"\n",
8481
"print(\"\" * 50)\n",
8582
"print(\"📏 Level 1: Exact Match\")\n",
83+
"print(\" Der einfachste Check: stimmt die Antwort exakt?\")\n",
8684
"print(f\" 'positive' == 'positive' → 1.0 ✅\")\n",
87-
"print(f\" 'positive' == 'POSITIVE' → 1.0 ✅ (normalized)\")\n",
8885
"print(f\" 'positive' == 'negative' → 0.0 ❌\")\n",
8986
"print()\n",
87+
"print(\"\" * 50)\n",
9088
"print(\"📏 Level 2: Token F1 (Partial Credit)\")\n",
91-
"print(f\" gold=['apple','banana'], pred=['apple','cherry'] → {f1_val:.3f}\")\n",
89+
"print(\" Was wenn die Antwort TEILWEISE stimmt?\")\n",
90+
"print(\" F1 balanciert Precision (wie viele Treffer waren korrekt?)\")\n",
91+
"print(\" und Recall (wie viele wurden gefunden?)\")\n",
9292
"print()\n",
93-
"print(\"📏 Level 3: Composite (Ticket Routing)\")\n",
94-
"print(\" Priority korrekt (40%) + Kategorie korrekt (35%) + Team korrekt (25%)\")\n",
95-
"print(\" = Gewichtete Spezifikation von 'was am wichtigsten ist'\")\n",
9693
"\n",
97-
"display_insight(\"Der Kern-Insight\",\n",
98-
" \"Deine Metrik-Funktion IST deine Produkt-Spezifikation. \"\n",
99-
" \"Wenn du die Gewichte änderst, änderst du, wofür das System optimiert. \"\n",
100-
" \"Deshalb ist Evaluation die Software der Zukunft.\")"
94+
"cases = [\n",
95+
" ([\"apple\", \"banana\", \"cherry\"], [\"apple\"],\n",
96+
" \"Vorsichtig: nur 1 von 3 gefunden, aber der war richtig\"),\n",
97+
" ([\"apple\", \"banana\", \"cherry\"], [\"apple\", \"banana\", \"cherry\", \"grape\", \"melon\"],\n",
98+
" \"Übereifrig: alles gefunden, aber 2 Falsche dabei\"),\n",
99+
" ([\"apple\", \"banana\", \"cherry\"], [\"apple\", \"banana\", \"cherry\"],\n",
100+
" \"Perfekt: genau richtig\"),\n",
101+
" ([\"apple\", \"banana\"], [\"cherry\", \"grape\"],\n",
102+
" \"Komplett daneben: nichts stimmt\"),\n",
103+
"]\n",
104+
"for gold, pred, desc in cases:\n",
105+
" f1 = token_f1(gold, pred)\n",
106+
" icon = \"\" if f1 == 1.0 else \"🟡\" if f1 > 0 else \"\"\n",
107+
" print(f\" {icon} {desc}\")\n",
108+
" print(f\" Gold: {gold} | Pred: {pred} → F1 = {f1:.2f}\")\n",
109+
"\n",
110+
"print()\n",
111+
"print(\"\" * 50)\n",
112+
"print(\"📏 Level 3: Composite (Ticket Routing)\")\n",
113+
"print(\" Mehrere Kriterien mit Gewichtung:\")\n",
114+
"print(\" Priorität korrekt (40%) + Kategorie (35%) + Team (25%)\")\n",
115+
"print(\" = Gewichtete Spezifikation von 'was am wichtigsten ist'\")\n"
101116
]
102117
},
103118
{
@@ -305,4 +320,4 @@
305320
},
306321
"nbformat": 4,
307322
"nbformat_minor": 4
308-
}
323+
}

0 commit comments

Comments
 (0)