Refactor optimization notebook and enhance actions module

- Updated the optimization notebook to improve prompt instructions and evaluation process.
- Changed the manual prompt variable name for clarity and adjusted the evaluation metrics.
- Enhanced the `run_optimization` function to accept custom instructions for better prompt optimization.
- Added a new function to format optimized prompts for improved readability.
- Introduced a new CSV analysis script to summarize categories, priorities, and assigned groups from the dataset.

Author: abossard

notebooks/02_optimization.ipynb

@@ -115,15 +115,15 @@
     "from dspy_tasks.actions import run_with_prompt\n",
     "from dspy_tasks.visualize import display_score, display_results_table\n",
     "\n",
-    "# Dein manueller Versuch — ändere den Prompt und führe die Zelle erneut aus!\n",
-    "MEIN_PROMPT = \"\"\"Classify this IT support ticket.\n",
-    "Category MUST be one of: Event NO Customer Impact, Failure, Service Request\n",
-    "Priority MUST be one of: High, Medium, Standard\n",
-    "Assigned group MUST be one of: SDE - Service Desk, OFC - Office & Collaboration, PRM - Premium Support, CDC - Client Design & Standard SW Integration, OUM - Incident, Helpdesk, Service-Center IKT, Service-Center IKT Bestellungen, Smartcard Office, CBCD - Container Basierte Cloud Dienste, OPC - Applikationen, DevOps - Rein, BVX - ePortal Service Line, IOM - Input / Output Mgmt., OPM - DLC Dispatching, O-SDK, ESTV-RSS-Stammdaten, FIB - BIT Store Bollwerk, Immobilien BAZG, Bedarfsmanagement, Güter und Ausrüstung\"\"\"\n",
-    "\n",
-    "result = run_with_prompt(\"ticket_routing\", MEIN_PROMPT, max_eval=3)\n",
-    "display_score(\"Dein manueller Prompt\", result.score)\n",
-    "display_results_table(result.individual_scores)"
+    "# Dein manueller Prompt — ändere ihn und führe die Zelle erneut aus!\n",
+    "MEIN_PROMPT = \"\"\"Klassifiziere dieses IT-Support-Ticket.\n",
+    "Kategorie MUSS eine von diesen sein: Event NO Customer Impact, Failure, Service Request\n",
+    "Priorität MUSS eine von diesen sein: High, Medium, Standard\n",
+    "Zuständige Gruppe MUSS eine von diesen sein: SDE - Service Desk, OFC - Office & Collaboration, PRM - Premium Support, CDC - Client Design & Standard SW Integration, OUM - Incident, Helpdesk, Service-Center IKT, Service-Center IKT Bestellungen, Smartcard Office, CBCD - Container Basierte Cloud Dienste, OPC - Applikationen, DevOps - Rein, BVX - ePortal Service Line, IOM - Input / Output Mgmt., OPM - DLC Dispatching, O-SDK, ESTV-RSS-Stammdaten, FIB - BIT Store Bollwerk, Immobilien BAZG, Bedarfsmanagement, Güter und Ausrüstung\"\"\"\n",
+    "\n",
+    "manual_result = run_with_prompt(\"ticket_routing\", MEIN_PROMPT, max_eval=8)\n",
+    "display_score(\"Dein manueller Prompt\", manual_result.score)\n",
+    "display_results_table(manual_result.individual_scores)"
    ]
   },
   {
@@ -169,39 +169,30 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from dspy_tasks.actions import run_optimization, run_with_prompt\n",
+    "from dspy_tasks.actions import run_optimization\n",
     "from dspy_tasks.tasks import get_task\n",
     "from dspy_tasks.visualize import display_improvement, display_insight, display_prompt_diff, display_score, display_results_table\n",
     "\n",
     "task = get_task(\"ticket_routing\")\n",
     "print(f\"⏳ Optimiere {task.name} mit BootstrapFewShot...\")\n",
+    "print(f\"   Basis: dein manueller Prompt\")\n",
     "print(f\"   Das kann 10-60 Sekunden dauern...\\n\")\n",
     "\n",
-    "result = run_optimization(\"ticket_routing\", \"BootstrapFewShot\", max_eval=8)\n",
+    "bs_result = run_optimization(\"ticket_routing\", \"BootstrapFewShot\", max_eval=8, instructions=MEIN_PROMPT)\n",
     "\n",
-    "# Zeige Baseline (zero-shot, OHNE Prompt) vs. Optimiert\n",
     "print(\"━\" * 60)\n",
-    "print(\"📊 VORHER: Zero-Shot Baseline (kein Prompt, kein Beispiel)\")\n",
-    "display_score(\"Zero-Shot Baseline\", result.baseline_score)\n",
-    "if result.baseline_individual_scores:\n",
-    "    display_results_table(result.baseline_individual_scores)\n",
+    "print(\"📊 NACHHER: Optimierter Prompt (dein Prompt + Few-Shot Beispiele)\")\n",
+    "display_score(\"Nach BootstrapFewShot\", bs_result.optimized_score)\n",
+    "if bs_result.optimized_individual_scores:\n",
+    "    display_results_table(bs_result.optimized_individual_scores)\n",
     "\n",
-    "print(\"\\n\" + \"━\" * 60)\n",
-    "print(\"📊 NACHHER: Optimierter Prompt (mit Few-Shot Beispielen)\")\n",
-    "display_score(\"Nach BootstrapFewShot\", result.optimized_score)\n",
-    "if result.optimized_individual_scores:\n",
-    "    display_results_table(result.optimized_individual_scores)\n",
+    "display_improvement(manual_result.score, bs_result.optimized_score)\n",
     "\n",
-    "display_improvement(result.baseline_score, result.optimized_score)\n",
-    "print(f\"⏱️  Optimierung dauerte {result.elapsed_seconds}s | {result.llm_calls} LLM-Aufrufe\")\n",
-    "\n",
-    "display_prompt_diff(result.prompt_before, result.prompt_after)\n",
+    "display_prompt_diff(MEIN_PROMPT, bs_result.prompt_after)\n",
     "\n",
     "display_insight(\"Was gerade passiert ist\",\n",
-    "    f\"Zero-Shot (ohne Prompt): {result.baseline_score:.0%}. \"\n",
-    "    f\"Nach Optimierung: {result.optimized_score:.0%}. \"\n",
-    "    \"Der Optimizer hat aus den Trainingsdaten die besten Beispiel-Tickets ausgewählt und in den Prompt eingefügt. \"\n",
-    "    \"Vergleich das mit deinem manuellen Prompt oben — ist der Optimizer besser?\")"
+    "    f\"Dein Prompt: {manual_result.score:.0%} → BootstrapFewShot: {bs_result.optimized_score:.0%}. \"\n",
+    "    \"Der Optimizer hat deinen Prompt als Basis genommen und die besten Beispiel-Tickets hinzugefügt.\")"
    ]
   },
   {
@@ -241,15 +232,12 @@
     "\n",
     "task = get_task(\"ticket_routing\")\n",
     "\n",
-    "# --- BootstrapFewShot ---\n",
+    "# --- BootstrapFewShot (basierend auf manuellem Prompt) ---\n",
     "print(f\"⏳ BootstrapFewShot auf {task.name}...\\n\")\n",
-    "r_bs = run_optimization(\"ticket_routing\", \"BootstrapFewShot\", max_eval=8)\n",
-    "print(f\"\\n✅ BootstrapFewShot: {r_bs.baseline_score:.0%} → {r_bs.optimized_score:.0%}\")\n",
-    "display_improvement(r_bs.baseline_score, r_bs.optimized_score)\n",
+    "r_bs = run_optimization(\"ticket_routing\", \"BootstrapFewShot\", max_eval=8, instructions=MEIN_PROMPT)\n",
+    "print(f\"✅ BootstrapFewShot: {r_bs.optimized_score:.0%}\")\n",
     "\n",
-    "print(\"\\n📝 BootstrapFewShot — Prompt VORHER:\")\n",
-    "print(r_bs.prompt_before)\n",
-    "print(\"\\n📝 BootstrapFewShot — Prompt NACHHER:\")\n",
+    "print(\"\\n📝 BootstrapFewShot — Prompt:\")\n",
     "print(r_bs.prompt_after[:2000])\n",
     "if len(r_bs.prompt_after) > 2000:\n",
     "    print(f\"... ({len(r_bs.prompt_after)} Zeichen)\")\n",
@@ -258,16 +246,18 @@
     "if r_bs.optimized_individual_scores:\n",
     "    display_results_table(r_bs.optimized_individual_scores)\n",
     "\n",
-    "# --- MIPROv2 ---\n",
+    "# --- MIPROv2 (basierend auf dem BESSEREN der beiden bisherigen Ergebnisse) ---\n",
     "print(f\"\\n{'━'*60}\")\n",
-    "print(f\"⏳ MIPROv2 auf {task.name}...\\n\")\n",
-    "r_mipro = run_optimization(\"ticket_routing\", \"MIPROv2\", max_eval=8)\n",
-    "print(f\"\\n✅ MIPROv2: {r_mipro.baseline_score:.0%} → {r_mipro.optimized_score:.0%}\")\n",
-    "display_improvement(r_mipro.baseline_score, r_mipro.optimized_score)\n",
-    "\n",
-    "print(\"\\n📝 MIPROv2 — Prompt VORHER:\")\n",
-    "print(r_mipro.prompt_before)\n",
-    "print(\"\\n📝 MIPROv2 — Prompt NACHHER:\")\n",
+    "best_score = max(manual_result.score, r_bs.optimized_score)\n",
+    "best_prompt = MEIN_PROMPT if manual_result.score >= r_bs.optimized_score else r_bs.prompt_after\n",
+    "best_label = \"Manuell\" if manual_result.score >= r_bs.optimized_score else \"BootstrapFewShot\"\n",
+    "print(f\"⏳ MIPROv2 auf {task.name}...\")\n",
+    "print(f\"   Basis: {best_label} ({best_score:.0%}) — das bisherige beste Ergebnis\\n\")\n",
+    "\n",
+    "r_mipro = run_optimization(\"ticket_routing\", \"MIPROv2\", max_eval=8, instructions=best_prompt)\n",
+    "print(f\"✅ MIPROv2: {r_mipro.optimized_score:.0%}\")\n",
+    "\n",
+    "print(\"\\n📝 MIPROv2 — Prompt:\")\n",
     "print(r_mipro.prompt_after[:2000])\n",
     "if len(r_mipro.prompt_after) > 2000:\n",
     "    print(f\"... ({len(r_mipro.prompt_after)} Zeichen)\")\n",
@@ -276,29 +266,22 @@
     "if r_mipro.optimized_individual_scores:\n",
     "    display_results_table(r_mipro.optimized_individual_scores)\n",
     "\n",
-    "# --- Vergleich ---\n",
+    "# --- Vergleich aller 3 ---\n",
     "print(f\"\\n{'━'*60}\")\n",
-    "print(\"📊 Zusammenfassung:\")\n",
-    "print(f\"   Dein manueller Prompt: scroll hoch und vergleich!\")\n",
-    "print(f\"   Zero-Shot Baseline:     {r_bs.baseline_score:.0%}\")\n",
-    "print(f\"   BootstrapFewShot:       {r_bs.optimized_score:.0%}\")\n",
-    "print(f\"   MIPROv2:                {r_mipro.optimized_score:.0%}\")\n",
+    "print(\"📊 Alle 3 Prompts im Vergleich:\")\n",
+    "print(f\"   1. Manueller Prompt:   {manual_result.score:.0%}  (deine Anweisung)\")\n",
+    "print(f\"   2. BootstrapFewShot:   {r_bs.optimized_score:.0%}  (dein Prompt + Beispiele)\")\n",
+    "print(f\"   3. MIPROv2:            {r_mipro.optimized_score:.0%}  (bester Prompt + Beispiele + umgeschrieben)\")\n",
     "\n",
     "scores = {\n",
-    "    \"BootstrapFewShot\": {\"baseline\": r_bs.baseline_score, \"optimized\": r_bs.optimized_score},\n",
-    "    \"MIPROv2\": {\"baseline\": r_mipro.baseline_score, \"optimized\": r_mipro.optimized_score},\n",
+    "    \"Manuell\": {\"baseline\": manual_result.score, \"optimized\": manual_result.score},\n",
+    "    \"BootstrapFewShot\": {\"baseline\": manual_result.score, \"optimized\": r_bs.optimized_score},\n",
+    "    \"MIPROv2\": {\"baseline\": manual_result.score, \"optimized\": r_mipro.optimized_score},\n",
     "}\n",
-    "fig = bar_comparison(\"Ticket Routing: Optimizer-Vergleich\", scores)\n",
+    "fig = bar_comparison(\"Ticket Routing: Manuell vs. Optimizer\", scores)\n",
     "fig.show()"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -470,32 +453,67 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Verfügbare Aufgaben zum Optimieren:\n",
-    "available = [\"multihop_qa\", \"report_generation\"]\n",
-    "print(\"Verfügbare Aufgaben:\")\n",
-    "for tid in available:\n",
-    "    t = get_task(tid)\n",
-    "    examples = t.load_examples()[:2]\n",
-    "    print(f\"\\n  📋 {tid}: {t.name}\")\n",
-    "    print(f\"     {t.description}\")\n",
-    "    if examples:\n",
-    "        sample = examples[0]\n",
-    "        input_preview = \" | \".join(f\"{k}={str(v)[:60]}\" for k, v in sample.inputs().items())\n",
-    "        print(f\"     Beispiel-Input: {input_preview[:120]}...\")\n",
-    "\n",
-    "# ═══ Wähle eine Aufgabe (ändere den String): ═══\n",
-    "TASK = \"multihop_qa\"\n",
+    "# ━━━ Verfügbare Aufgaben ━━━\n",
+    "from dspy_tasks.tasks import list_tasks\n",
+    "from dspy_tasks.actions import run_with_prompt\n",
+    "\n",
+    "print(\"Alle verfügbaren Aufgaben:\\n\")\n",
+    "for tier in [1, 2, 3, 4]:\n",
+    "    tasks_in_tier = [t for t in list_tasks() if t.tier == tier]\n",
+    "    tier_labels = {1: \"Basics\", 2: \"Reasoning\", 3: \"Composition\", 4: \"Agentic\"}\n",
+    "    print(f\"  Tier {tier} — {tier_labels[tier]}:\")\n",
+    "    for t in tasks_in_tier:\n",
+    "        examples = t.load_examples()[:1]\n",
+    "        sample_input = \"\"\n",
+    "        if examples:\n",
+    "            sample_input = \" | \".join(f\"{k}={str(v)[:40]}\" for k, v in examples[0].inputs().items())\n",
+    "        print(f\"    • {t.id:25s} {t.name}\")\n",
+    "        print(f\"      {t.description[:80]}\")\n",
+    "        if sample_input:\n",
+    "            print(f\"      Beispiel: {sample_input[:100]}\")\n",
+    "    print()\n",
+    "\n",
+    "# ━━━ Aufgabe 1: Sentiment (BootstrapFewShot) ━━━\n",
+    "SENT_PROMPT = \"Bestimme ob diese Bewertung positiv, negativ oder neutral ist.\"\n",
+    "print(\"━\" * 60)\n",
+    "print(\"📋 Sentiment Classification — BootstrapFewShot\\n\")\n",
+    "\n",
+    "manual_sent = run_with_prompt(\"sentiment\", SENT_PROMPT, max_eval=8)\n",
+    "display_score(\"Sentiment: Manueller Prompt\", manual_sent.score)\n",
+    "display_results_table(manual_sent.individual_scores)\n",
+    "\n",
+    "print(\"\\n⏳ Optimiere mit BootstrapFewShot (basierend auf manuellem Prompt)...\")\n",
+    "r_sent = run_optimization(\"sentiment\", \"BootstrapFewShot\", max_eval=8, instructions=SENT_PROMPT)\n",
+    "display_score(\"Sentiment: Nach BootstrapFewShot\", r_sent.optimized_score)\n",
+    "if r_sent.optimized_individual_scores:\n",
+    "    display_results_table(r_sent.optimized_individual_scores)\n",
+    "display_improvement(manual_sent.score, r_sent.optimized_score)\n",
     "\n",
+    "# ━━━ Aufgabe 2: Multi-Hop QA (BootstrapFewShot → MIPROv2) ━━━\n",
+    "MHQA_PROMPT = \"Beantworte die Frage basierend auf dem Kontext. Denke Schritt für Schritt.\"\n",
     "print(f\"\\n{'━'*60}\")\n",
-    "print(f\"⏳ Optimiere {TASK}...\")\n",
-    "result = run_optimization(TASK, \"BootstrapFewShot\", max_eval=8)\n",
-    "display_improvement(result.baseline_score, result.optimized_score)\n",
-    "display_prompt_diff(result.prompt_before, result.prompt_after)\n",
-    "\n",
-    "# Zeige Ergebnisse pro Beispiel\n",
-    "print(\"\\n📊 Ergebnisse nach Optimierung:\")\n",
-    "if result.optimized_individual_scores:\n",
-    "    display_results_table(result.optimized_individual_scores)"
+    "print(\"📋 Multi-Hop QA — BootstrapFewShot → MIPROv2\\n\")\n",
+    "\n",
+    "manual_mhqa = run_with_prompt(\"multihop_qa\", MHQA_PROMPT, max_eval=8)\n",
+    "display_score(\"Multi-Hop QA: Manueller Prompt\", manual_mhqa.score)\n",
+    "display_results_table(manual_mhqa.individual_scores)\n",
+    "\n",
+    "# Erst BootstrapFewShot\n",
+    "print(\"\\n⏳ Schritt 1: BootstrapFewShot...\")\n",
+    "r_mhqa_bs = run_optimization(\"multihop_qa\", \"BootstrapFewShot\", max_eval=8, instructions=MHQA_PROMPT)\n",
+    "display_score(\"Multi-Hop QA: Nach BootstrapFewShot\", r_mhqa_bs.optimized_score)\n",
+    "\n",
+    "# Dann MIPROv2 auf dem besseren Ergebnis\n",
+    "best_mhqa_score = max(manual_mhqa.score, r_mhqa_bs.optimized_score)\n",
+    "best_mhqa_prompt = MHQA_PROMPT if manual_mhqa.score >= r_mhqa_bs.optimized_score else r_mhqa_bs.prompt_after\n",
+    "best_mhqa_label = \"Manuell\" if manual_mhqa.score >= r_mhqa_bs.optimized_score else \"BootstrapFewShot\"\n",
+    "print(f\"\\n⏳ Schritt 2: MIPROv2 (basierend auf {best_mhqa_label}: {best_mhqa_score:.0%})...\")\n",
+    "r_mhqa = run_optimization(\"multihop_qa\", \"MIPROv2\", max_eval=8, instructions=best_mhqa_prompt)\n",
+    "display_score(\"Multi-Hop QA: Nach MIPROv2\", r_mhqa.optimized_score)\n",
+    "if r_mhqa.optimized_individual_scores:\n",
+    "    display_results_table(r_mhqa.optimized_individual_scores)\n",
+    "\n",
+    "print(f\"\\n📊 Manuell {manual_mhqa.score:.0%} → BootstrapFewShot {r_mhqa_bs.optimized_score:.0%} → MIPROv2 {r_mhqa.optimized_score:.0%}\")"
    ]
   },
   {

notebooks/_analyze_csv.py

@@ -0,0 +1,22 @@
+import csv
+from collections import Counter
+
+with open('/Users/abossard/Desktop/projects/python-quart-vite-react/csv/data.csv', encoding='utf-8', errors='replace') as f:
+    reader = csv.DictReader(f)
+    rows = list(reader)
+
+print(f'Total rows: {len(rows)}')
+sample = rows[0]
+for col in ['Summary*', 'Category', 'Priority*', 'Assigned Group*+', 'Incident Type*']:
+    val = sample.get(col, 'NOT FOUND')
+    print(f'  {col}: {str(val)[:60]}')
+
+cats = Counter(r.get('Category', '') for r in rows)
+prios = Counter(r.get('Priority*', '') for r in rows)
+groups = Counter(r.get('Assigned Group*+', '') for r in rows)
+
+print(f'\nCategories: {dict(cats.most_common(5))}')
+print(f'Priorities: {dict(prios.most_common(5))}')
+print(f'\nGroups ({len(groups)}):')
+for g, c in groups.most_common(25):
+    print(f'  {c:4d} {g}')

notebooks/dspy_tasks/actions.py

@@ -84,20 +84,32 @@ def run_on_examples(examples, instructions: str, signature_class, metric_fn) ->
     )
 
 
-def run_optimization(task_id: str, optimizer: str = "BootstrapFewShot", *, max_eval: Optional[int] = None) -> OptimizationResult:
-    """Optimize a task's prompt, then evaluate. Takes 10-60 seconds."""
+def run_optimization(task_id: str, optimizer: str = "BootstrapFewShot", *, max_eval: Optional[int] = None, instructions: Optional[str] = None) -> OptimizationResult:
+    """Optimize a task's prompt, then evaluate. Takes 10-60 seconds.
+    
+    If instructions is provided, it's used as the starting prompt (instead of zero-shot).
+    """
     task = get_task(task_id)
     trainset, devset = task.split_examples()
     if max_eval:
         devset = devset[:max_eval]
 
-    # Baseline
-    module = task.make_module()
+    # Build module — with custom instructions if provided
+    def _make_module_with_instructions():
+        if instructions:
+            CustomSig = _make_signature(task.signature_class, instructions)
+            if task.module_type == "ChainOfThought":
+                return dspy.ChainOfThought(CustomSig)
+            return dspy.Predict(CustomSig)
+        return task.make_module()
+
+    # Baseline (with the given instructions, not zero-shot)
+    module = _make_module_with_instructions()
     baseline_results = _evaluate_examples(module, devset, task.metric_fn)
     baseline_score = _mean([r["score"] for r in baseline_results])
 
-    # Optimize
-    module_fresh = task.make_module()
+    # Optimize (starting from the same instructions)
+    module_fresh = _make_module_with_instructions()
     start = time.time()
     if optimizer == "MIPROv2":
         opt = dspy.MIPROv2(metric=task.metric_fn, auto="light")
@@ -119,8 +131,8 @@ def run_optimization(task_id: str, optimizer: str = "BootstrapFewShot", *, max_e
         optimized_score=opt_score,
         improvement=improvement,
         improvement_pct=(improvement / max(baseline_score, 0.01)) * 100,
-        prompt_before="(zero-shot)",
-        prompt_after=str(optimized_module.dump_state()) if hasattr(optimized_module, 'dump_state') else "(optimized)",
+        prompt_before=instructions or "(zero-shot)",
+        prompt_after=_format_optimized_prompt(optimized_module),
         trial_scores=[baseline_score, opt_score],
         elapsed_seconds=round(opt_elapsed, 2),
         llm_calls=len(baseline_results) + len(opt_results),
@@ -138,6 +150,39 @@ def _current_model() -> str:
     return str(getattr(lm, 'model', '?')) if lm else '?'
 
 
+def _format_optimized_prompt(module) -> str:
+    """Format an optimized DSPy module's state into a human-readable string."""
+    if not hasattr(module, 'dump_state'):
+        return "(optimized)"
+    
+    state = module.dump_state()
+    parts = []
+    
+    for predictor_name, predictor_state in state.items():
+        # Extract instructions
+        sig = predictor_state.get('signature', {})
+        instructions = sig.get('instructions', '')
+        if instructions:
+            parts.append("━━━ INSTRUCTIONS ━━━")
+            parts.append(instructions)
+        
+        # Extract demos (few-shot examples)
+        demos = predictor_state.get('demos', [])
+        if demos:
+            parts.append(f"\n━━━ FEW-SHOT EXAMPLES ({len(demos)}) ━━━")
+            for i, demo in enumerate(demos):
+                parts.append(f"\n  Example {i+1}:")
+                for key, value in demo.items():
+                    if key == 'augmented':
+                        continue
+                    val_str = str(value)
+                    if len(val_str) > 200:
+                        val_str = val_str[:200] + "..."
+                    parts.append(f"    {key}: {val_str}")
+    
+    return "\n".join(parts) if parts else str(state)
+
+
 def _make_signature(base_sig, instructions: str):
     fields = {}
     for name, field_info in base_sig.model_fields.items():