Add data processing scripts for ticket analysis and routing

- Implemented _check_fields.py to analyze fields in the CSV data and print useful content.
- Created _check_more_fields.py to explore additional fields of interest and their fill rates.
- Developed _curate_data.py to curate and score tickets for training, focusing on informative content.
- Added _import_csv.py to import CSV data, analyze incident types, and generate new tickets for underrepresented groups.
- Introduced _predict_group.py to evaluate predictive power of various fields for ticket assignment.
- Built _rebuild_data.py to create a balanced dataset for ticket routing, ensuring representation across groups.

Author: abossard

notebooks/_check_fields.py

@@ -0,0 +1,46 @@
+import csv
+
+with open('/Users/abossard/Desktop/projects/python-quart-vite-react/csv/data.csv', encoding='utf-8', errors='replace') as f:
+    reader = csv.DictReader(f)
+    rows = list(reader)
+
+# Check which fields have useful content for the first ticket
+sample = rows[0]
+useful_fields = []
+for key, val in sample.items():
+    val = str(val).strip()
+    if val and len(val) > 5 and len(val) < 500:
+        useful_fields.append((key, val[:120]))
+
+print(f"Fields with content (5-500 chars) in first row:")
+for key, val in useful_fields[:40]:
+    print(f"  {key:50s} | {val}")
+
+# Check Notes and Resolution fields specifically
+print(f"\n━━━ Key content fields ━━━")
+for field in ['Summary*', 'Notes', 'Resolution', 'Operational Categorization Tier 1+', 
+              'Operational Categorization Tier 2', 'Operational Categorization Tier 3',
+              'Product Categorization Tier 1', 'Product Categorization Tier 2',
+              'Product Categorization Tier 3', 'Service*+', 'CI+', 'Incident Type*',
+              'Category', 'Short Description']:
+    val = str(sample.get(field, '')).strip()
+    print(f"  {field:50s} | {val[:150]}")
+
+# Check a few more rows for Notes content
+print(f"\n━━━ Notes samples ━━━")
+for r in rows[:5]:
+    summary = str(r.get('Summary*', '')).strip()[:40]
+    notes = str(r.get('Notes', '')).strip()[:100]
+    resolution = str(r.get('Resolution', '')).strip()[:100]
+    short_desc = str(r.get('Short Description', '')).strip()[:100]
+    op_cat1 = str(r.get('Operational Categorization Tier 1+', '')).strip()
+    op_cat2 = str(r.get('Operational Categorization Tier 2', '')).strip()
+    service = str(r.get('Service*+', '')).strip()[:60]
+    product = str(r.get('Product Categorization Tier 1', '')).strip()
+    print(f"\n  Summary: {summary}")
+    print(f"  Notes: {notes}")
+    print(f"  Resolution: {resolution}")
+    print(f"  Short Desc: {short_desc}")
+    print(f"  OpCat: {op_cat1} > {op_cat2}")
+    print(f"  Service: {service}")
+    print(f"  Product: {product}")

notebooks/_check_more_fields.py

@@ -0,0 +1,56 @@
+import csv
+from collections import Counter
+
+with open('/Users/abossard/Desktop/projects/python-quart-vite-react/csv/data.csv', encoding='utf-8', errors='replace') as f:
+    reader = csv.DictReader(f)
+    rows = list(reader)
+
+# Fields we already use: Summary*, Notes, Service*+, Product Categorization Tier 3, Priority*, Assigned Group*+, Incident Type*
+# What else could be interesting?
+
+interesting = [
+    # Organization context
+    ('Company', 'Which department/org filed it'),
+    ('Organization', 'Sub-org of the requester'),
+    ('Support Organization', 'Support org handling it'),
+    
+    # Categorization
+    ('Operational Categorization Tier 1+', 'OpCat level 1'),
+    ('Operational Categorization Tier 2', 'OpCat level 2'),
+    ('Operational Categorization Tier 3', 'OpCat level 3'),
+    ('Product Categorization Tier 1', 'Product level 1'),
+    ('Product Categorization Tier 2', 'Product level 2'),
+    ('Product Categorization Tier 3', 'Product level 3'),
+    
+    # Service info
+    ('Service*+', 'Service name'),
+    ('CI+', 'Configuration Item'),
+    ('CI Name', 'CI readable name'),
+    
+    # Resolution (could teach what worked)
+    ('Resolution', 'How it was resolved'),
+    ('Resolution Categorization Tier 1', 'Resolution cat'),
+    
+    # Impact/urgency
+    ('Urgency*', 'How urgent'),
+    ('Impact*', 'How impactful'),
+    
+    # Source
+    ('Reported Source', 'How was it reported'),
+    
+    # Status
+    ('Status*', 'Current status'),
+    ('Status Reason', 'Why this status'),
+]
+
+print("Field analysis (first 20 rows sampled):\n")
+for field, desc in interesting:
+    values = [str(r.get(field, '')).strip() for r in rows[:100] if str(r.get(field, '')).strip()]
+    unique = len(set(values))
+    if values:
+        top = Counter(values).most_common(3)
+        sample = values[0][:60]
+        fill_rate = len(values) / min(100, len(rows))
+        print(f"  {field:45s} | {fill_rate:3.0%} filled | {unique:3d} unique | {desc}")
+        print(f"    Top: {", ".join(f"{v[:30]}({c})" for v,c in top)}")
+        print()

notebooks/_curate_data.py

@@ -0,0 +1,143 @@
+import csv, json
+from collections import Counter
+
+with open('/Users/abossard/Desktop/projects/python-quart-vite-react/csv/data.csv', encoding='utf-8', errors='replace') as f:
+    reader = csv.DictReader(f)
+    rows = list(reader)
+
+TYPE_MAP = {
+    'User Service Request': 'Service Request',
+    'Infrastructure Event': 'Event NO Customer Impact',
+    'User Service Restoration': 'Failure',
+}
+
+TARGET_GROUPS = [
+    'SDE - Service Desk', 'OFC - Office & Collaboration', 'PRM - Premium Support',
+    'CDC - Client Design & Standard SW Integration', 'OUM - Incident', 'Helpdesk',
+    'Service-Center IKT', 'Service-Center IKT Bestellungen', 'Smartcard Office',
+    'CBCD - Container Basierte Cloud Dienste', 'OPC - Applikationen', 'DevOps - Rein',
+    'BVX - ePortal Service Line', 'IOM - Input / Output Mgmt.', 'OPM - DLC Dispatching',
+    'O-SDK', 'ESTV-RSS-Stammdaten', 'FIB - BIT Store Bollwerk', 'Immobilien BAZG',
+    'Bedarfsmanagement', 'Güter und Ausrüstung',
+]
+
+def build_summary(row):
+    """Build a concise but informative summary."""
+    parts = []
+    summary = str(row.get('Summary*', '')).strip()
+    if summary:
+        parts.append(summary)
+    
+    # Add the key detail from notes (not the whole blob)
+    notes = str(row.get('Notes', '')).strip()
+    if notes and summary and notes.startswith(summary):
+        notes = notes[len(summary):].strip()
+    if notes and len(notes) > 5:
+        # Take first sentence/clause only
+        for sep in ['. ', '  ', '\n']:
+            if sep in notes:
+                notes = notes[:notes.index(sep)]
+                break
+        if len(notes) > 120:
+            notes = notes[:120] + '...'
+        if notes:
+            parts.append(notes)
+    
+    # Company (strong routing signal — BAZG tickets go to specific groups)
+    company = str(row.get('Company', '')).strip()
+    if company and len(company) > 2:
+        parts.append(f'[Firma: {company}]')
+    
+    # Service name (strongest predictor — 85% uncertainty reduction)
+    service = str(row.get('Service*+', '')).strip()
+    if service and len(service) > 2:
+        parts.append(f'[Service: {service}]')
+    
+    # Product domain (Workplace vs ESTV vs Customer facing)
+    prod2 = str(row.get('Product Categorization Tier 2', '')).strip()
+    if prod2 and len(prod2) > 2:
+        parts.append(f'[Bereich: {prod2}]')
+    
+    return ' — '.join(parts) if len(parts) > 1 else (parts[0] if parts else '')
+
+def score_ticket(row):
+    """Score how informative/interesting a ticket is for training."""
+    score = 0
+    notes = str(row.get('Notes', '')).strip()
+    summary = str(row.get('Summary*', '')).strip()
+    service = str(row.get('Service*+', '')).strip()
+    
+    if len(notes) > 20: score += 2  # Has real description
+    if len(summary) > 10: score += 1  # Descriptive title
+    if service: score += 1  # Has service context
+    if any(kw in notes.lower() for kw in ['fehler', 'problem', 'kann nicht', 'geht nicht', 'funktioniert nicht', 'störung', 'ausfall', 'gesperrt', 'defekt']):
+        score += 2  # Has error keywords
+    if any(kw in notes.lower() for kw in ['bestell', 'anfrage', 'freischalt', 'zugang', 'berechtigung', 'neu', 'install']):
+        score += 1  # Has request keywords
+    return score
+
+# Collect and score all matching tickets
+all_tickets = []
+for row in rows:
+    group = str(row.get('Assigned Group*+', '')).strip()
+    if group not in TARGET_GROUPS:
+        continue
+    
+    rich_summary = build_summary(row)
+    if not rich_summary or len(rich_summary) < 8:
+        continue
+    
+    # Use Operational Categorization Tier 1 as the real category (100% filled, accurate)
+    op_cat = str(row.get('Operational Categorization Tier 1+', '')).strip()
+    if op_cat in ('Service Request', 'Failure', 'Event NO Customer Impact'):
+        category = op_cat
+    else:
+        inc_type = str(row.get('Incident Type*', '')).strip()
+        category = TYPE_MAP.get(inc_type, 'Service Request')
+    priority = str(row.get('Priority*', 'Standard')).strip()
+    if priority not in ('Standard', 'Medium', 'High'):
+        priority = 'Standard'
+    
+    all_tickets.append({
+        'summary': rich_summary,
+        'category': category,
+        'priority': priority,
+        'assigned_group': group,
+        '_score': score_ticket(row),
+    })
+
+# Pick the BEST tickets per group: 4 per group max, sorted by score
+final_tickets = []
+seen = set()
+for group in TARGET_GROUPS:
+    group_tickets = sorted(
+        [t for t in all_tickets if t['assigned_group'] == group],
+        key=lambda t: t['_score'],
+        reverse=True
+    )
+    count = 0
+    for t in group_tickets:
+        if count >= 4:
+            break
+        # Skip near-duplicates
+        short = t['summary'][:30]
+        if short in seen:
+            continue
+        seen.add(short)
+        ticket = {k: v for k, v in t.items() if not k.startswith('_')}
+        final_tickets.append(ticket)
+        count += 1
+
+with open('datasets/ticket_routing.json', 'w') as f:
+    json.dump(final_tickets, f, indent=2, ensure_ascii=False)
+
+print(f'Total: {len(final_tickets)} curated tickets')
+final_groups = Counter(t['assigned_group'] for t in final_tickets)
+for g, c in sorted(final_groups.items(), key=lambda x: -x[1]):
+    print(f'  {c} {g}')
+
+print(f'\n━━━ Samples ━━━')
+import random
+for t in random.sample(final_tickets, min(8, len(final_tickets))):
+    print(f'\n  [{t["assigned_group"]}] {t["category"]} / {t["priority"]}')
+    print(f'  {t["summary"][:140]}')

notebooks/_import_csv.py

@@ -0,0 +1,80 @@
+import csv, json
+from collections import Counter
+
+with open('/Users/abossard/Desktop/projects/python-quart-vite-react/csv/data.csv', encoding='utf-8', errors='replace') as f:
+    reader = csv.DictReader(f)
+    rows = list(reader)
+
+# Check Incident Type as category proxy
+inc_types = Counter(r.get('Incident Type*', '') for r in rows)
+print(f'Incident Types: {dict(inc_types.most_common(10))}')
+
+# Map Incident Type to our categories
+TYPE_MAP = {
+    'User Service Request': 'Service Request',
+    'Infrastructure Event': 'Event NO Customer Impact',
+    'User Service Restoration': 'Failure',
+}
+
+# Get the groups that exist in our current dataset
+with open('datasets/ticket_routing.json') as f:
+    existing = json.load(f)
+existing_groups = set(d['assigned_group'] for d in existing)
+print(f'\nExisting groups in JSON: {len(existing_groups)}')
+
+# Find CSV rows that match our groups
+matching = [r for r in rows if r.get('Assigned Group*+', '').strip() in existing_groups]
+print(f'CSV rows matching existing groups: {len(matching)}')
+
+# For each group, count how many CSV rows we have
+groups = Counter(r.get('Assigned Group*+', '').strip() for r in matching)
+print(f'\nCSV data per existing group:')
+for g in sorted(existing_groups):
+    csv_count = groups.get(g, 0)
+    existing_count = sum(1 for d in existing if d['assigned_group'] == g)
+    print(f'  {csv_count:4d} CSV | {existing_count:2d} JSON | {g}')
+
+# Generate new JSON from CSV - take up to 5 per group for underrepresented ones
+new_tickets = []
+for group in existing_groups:
+    group_rows = [r for r in rows if r.get('Assigned Group*+', '').strip() == group]
+    existing_count = sum(1 for d in existing if d['assigned_group'] == group)
+    need = max(0, 5 - existing_count)  # Want at least 5 per group
+    
+    for r in group_rows[:need]:
+        summary = r.get('Summary*', '').strip()
+        if not summary or len(summary) < 3:
+            continue
+        inc_type = r.get('Incident Type*', '').strip()
+        category = TYPE_MAP.get(inc_type, 'Service Request')
+        priority = r.get('Priority*', 'Standard').strip()
+        if priority not in ('Standard', 'Medium', 'High'):
+            priority = 'Standard'
+        
+        # Skip if already in existing
+        if any(d['summary'] == summary for d in existing):
+            continue
+        
+        new_tickets.append({
+            'summary': summary,
+            'category': category,
+            'priority': priority,
+            'assigned_group': group,
+        })
+
+print(f'\nNew tickets to add: {len(new_tickets)}')
+for t in new_tickets[:10]:
+    print(f'  {t["assigned_group"]:40s} | {t["category"]:20s} | {t["priority"]:10s} | {t["summary"][:50]}')
+if len(new_tickets) > 10:
+    print(f'  ... and {len(new_tickets) - 10} more')
+
+# Write updated dataset
+existing.extend(new_tickets)
+with open('datasets/ticket_routing.json', 'w') as f:
+    json.dump(existing, f, indent=2, ensure_ascii=False)
+print(f'\nUpdated dataset: {len(existing)} tickets (was {len(existing) - len(new_tickets)})')
+
+# Final stats
+final_groups = Counter(d['assigned_group'] for d in existing)
+single = sum(1 for c in final_groups.values() if c == 1)
+print(f'Groups with only 1 example: {single} (was 14)')