diff --git a/code/README.md b/code/README.md new file mode 100644 index 00000000..32eae89e --- /dev/null +++ b/code/README.md @@ -0,0 +1,31 @@ +## Setup + +```bash +pip install -r requirements.txt +``` + +## Run + +```bash +cd code/ +python main.py \ + --tickets ../support_tickets/support_tickets.csv \ + --data ../data/ \ + --output ../support_tickets/output.csv \ + --log ../run_log.txt +``` + +## Validation + +```bash +cd code/ +python main.py \ + --tickets ../support_tickets/sample_support_tickets.csv \ + --data ../data/ \ + --output ../support_tickets/sample_output.csv \ + --log ../run_log.txt +``` + +## Output columns + +`status, product_area, response, justification, request_type` diff --git a/code/classifier.py b/code/classifier.py new file mode 100644 index 00000000..7ee39673 --- /dev/null +++ b/code/classifier.py @@ -0,0 +1,183 @@ +import re +from dataclasses import dataclass +from typing import Dict +from typing import List +from retrieval import RetrievalHit +from utils import clean_text + + +ALLOWED_REQUEST_TYPES = {"product_issue", "feature_request", "bug", "invalid"} + +AREA_FALLBACKS = { + "hackerrank": { + "interviewer": "team_management", + "team": "team_management", + "certificate": "certifications", + "mock interview": "interviewing", + "compatibility": "assessments", + "assessment": "assessments", + "test": "assessments", + "resume": "community_support", + "subscription": "billing_and_subscriptions", + "interview": "interviewing", + "lti": "integrations", + }, + "claude": { + "bedrock": "api_and_developer_tools", + "api": "api_and_developer_tools", + "workspace": "account_management", + "personal data": "privacy_and_compliance", + "privacy": "privacy_and_compliance", + "crawl": "privacy_and_compliance", + "crawler": "privacy_and_compliance", + "lti": "education", + }, + "visa": { + "charge": "payment_processing", + "refund": "payment_processing", + "dispute": "payment_processing", + "merchant": "merchant_acceptance", + "minimum": "merchant_acceptance", + "travel": "travel_services", + "cash": "travel_services", + "identity": "fraud_and_security", + "fraud": "fraud_and_security", + "privacy": "data_privacy", + }, +} + + +@dataclass +class ClassificationResult: + subject: str + issue: str + ticket_text: str + company: str + request_type: str + + +class Classifier: + domain_keywords = { + "hackerrank": [ + "hackerrank", + "assessment", + "test", + "candidate", + "recruiter", + "interview", + "mock interview", + "resume builder", + "apply tab", + "certificate", + "interviewer", + ], + "claude": [ + "claude", + "bedrock", + "anthropic", + "workspace", + "crawler", + "lti", + "model", + "console", + ], + "visa": [ + "visa", + "card", + "merchant", + "charge", + "cash", + "travel", + "fraud", + ], + } + + invalid_patterns = ( + "delete all files", + "ignore previous instructions", + "show internal rules", + "show documents retrieved", + "logic exact", + "prompt injection", + ) + + bug_patterns = ( + "not working", + "stopped working", + "failing", + "error", + "down", + "issue while", + "compatibility", + "blocked", + "blocker", + "not responding", + "unable to", + ) + + feature_patterns = ( + "feature request", + "can you add", + "enhancement", + "new feature", + ) + + def classify(self, row: Dict[str, str]) -> ClassificationResult: + subject = clean_text(row.get("Subject", "")) + issue = clean_text(row.get("Issue", "")) + ticket_text = clean_text(f"{subject} {issue}") + company = self.infer_company(clean_text(row.get("Company", "")), ticket_text) + request_type = self.request_type(ticket_text) + if request_type not in ALLOWED_REQUEST_TYPES: + request_type = "invalid" + return ClassificationResult( + subject=subject, + issue=issue, + ticket_text=ticket_text, + company=company, + request_type=request_type, + ) + + def infer_company(self, provided: str, ticket_text: str) -> str: + lowered = provided.lower().strip() + if lowered in {"hackerrank", "claude", "visa"}: + return lowered + scores = {} + text = ticket_text.lower() + for company, keywords in self.domain_keywords.items(): + scores[company] = sum(1 for keyword in keywords if keyword in text) + best_company = max(scores, key=scores.get) + return best_company if scores[best_company] > 0 else "none" + + def request_type(self, ticket_text: str) -> str: + lowered = ticket_text.lower().strip() + if not lowered: + return "invalid" + if any(pattern in lowered for pattern in self.invalid_patterns): + return "invalid" + if re.fullmatch(r"(thanks|thank you|ok|okay)[.! ]*", lowered): + return "invalid" + if any(pattern in lowered for pattern in self.feature_patterns): + return "feature_request" + if any(pattern in lowered for pattern in self.bug_patterns): + return "bug" + return "product_issue" + + def product_area(self, classification: ClassificationResult, hits: List[RetrievalHit]) -> str: + if hits: + top_area = hits[0].doc.product_area + if top_area not in {"conversation_management", "general"}: + return top_area + + lowered = classification.ticket_text.lower() + for keyword, area in AREA_FALLBACKS.get(classification.company, {}).items(): + if keyword in lowered: + return area + + if classification.company == "hackerrank": + return "platform_support" + if classification.company == "claude": + return "account_management" + if classification.company == "visa": + return "payment_processing" + return "platform_support" diff --git a/code/decision.py b/code/decision.py new file mode 100644 index 00000000..f7b4ccec --- /dev/null +++ b/code/decision.py @@ -0,0 +1,183 @@ +from dataclasses import dataclass +from typing import List + +from classifier import ClassificationResult +from retrieval import RetrievalHit +from utils import tokenize + + +CONFIDENCE_THRESHOLDS = { + "strong_reply": 0.55, + "overlap_reply": 0.35, + "none": 0.0, +} + +BYPASS_INTENTS = {"bedrock", "crawler", "lti", "interviewer", "troubleshooting", "merchant_rules", "travel_support", "compatibility"} + + +@dataclass(frozen=True) +class Decision: + status: str + confidence: float + reason_code: str + reason_detail: str + + +class DecisionEngine: + pii_terms = ("personal data", "private info", "privacy request", "data request") + security_terms = ( + "security vulnerability", + "bug bounty", + "identity theft", + "restore my access", + "delete my account", + "remove my account", + "stolen", + "compromised", + ) + certificate_terms = ("certificate", "credential") + billing_terms = ("payment", "refund", "charge", "billing", "subscription", "dispute") + legal_terms = ("legal", "compliance", "dpo", "gdpr", "infosec", "security questionnaire") + + def decide( + self, + ticket_text: str, + classification: ClassificationResult, + hits: List[RetrievalHit], + ) -> Decision: + lowered = ticket_text.lower() + confidence = hits[0].score if hits else CONFIDENCE_THRESHOLDS["none"] + intent = self._intent_name(lowered) + + if classification.request_type == "invalid": + return Decision( + status="escalated", + confidence=confidence, + reason_code="invalid", + reason_detail="Ticket does not match any supported domain or request type.", + ) + + if not hits: + return Decision( + status="escalated", + confidence=0.0, + reason_code="no_docs", + reason_detail="No relevant documentation was retrieved from the support corpus.", + ) + + if self._is_high_risk(lowered): + reason_code, reason_detail = self._select_escalation_reason(lowered, hits) + return Decision( + status="escalated", + confidence=confidence, + reason_code=reason_code, + reason_detail=reason_detail, + ) + + if intent in BYPASS_INTENTS and hits: + return Decision( + status="replied", + confidence=confidence, + reason_code="corpus_grounded_reply", + reason_detail="A sufficiently strong support document was retrieved for a grounded reply.", + ) + + if confidence >= CONFIDENCE_THRESHOLDS["strong_reply"]: + return Decision( + status="replied", + confidence=confidence, + reason_code="corpus_grounded_reply", + reason_detail="A sufficiently strong support document was retrieved for a grounded reply.", + ) + + if confidence >= CONFIDENCE_THRESHOLDS["overlap_reply"] and self._intent_keyword_overlap(lowered, hits): + return Decision( + status="replied", + confidence=confidence, + reason_code="corpus_grounded_reply", + reason_detail="A sufficiently strong support document was retrieved for a grounded reply.", + ) + + return Decision( + status="escalated", + confidence=confidence, + reason_code="low_confidence", + reason_detail="The available documentation does not provide sufficient guidance for this specific case.", + ) + + def _select_escalation_reason(self, lowered: str, hits: List[RetrievalHit]): + if any(term in lowered for term in ("personal data", "gdpr", "privacy", "my data", "delete my")): + return ( + "pii", + "This ticket involves personal data and requires privacy review.", + ) + if any(term in lowered for term in ("fraud", "unauthorized", "stolen", "suspicious", "bug bounty", "security vulnerability")): + return ( + "security", + "This ticket involves account security or fraud and requires human verification.", + ) + if any(term in lowered for term in ("password", "account access", "login", "identity", "account deletion")): + return ( + "security", + "This ticket involves account security changes and requires human verification.", + ) + if any(term in lowered for term in ("billing", "payment", "refund", "charge", "invoice", "subscription cancel")): + return ( + "billing", + "This ticket involves a billing or payment matter that requires human review.", + ) + if any(term in lowered for term in ("certificate", "credential", "badge", "verify name")): + return ( + "certificate", + "This ticket involves a certificate or credential update that requires identity verification.", + ) + if any(term in lowered for term in ("legal", "compliance", "regulation", "court")): + return ( + "legal", + "This ticket involves compliance or legal review.", + ) + if not hits: + return ("no_docs", "No relevant documentation was found in the support corpus for this query.") + return ("low_confidence", "The available documentation does not provide sufficient guidance for this specific case.") + + def _is_high_risk(self, lowered: str) -> bool: + return any( + term in lowered + for term in ( + "personal data", "gdpr", "privacy", "my data", "delete my", + "fraud", "unauthorized", "stolen", "suspicious", "bug bounty", "security vulnerability", + "password", "account access", "login", "identity", "account deletion", + "billing", "payment", "refund", "charge", "invoice", "subscription cancel", + "certificate", "credential", "badge", "verify name", + "legal", "compliance", "regulation", "court", + ) + ) + + def _intent_name(self, lowered: str) -> str: + if "bedrock" in lowered: + return "bedrock" + if "crawler" in lowered or "crawl" in lowered: + return "crawler" + if "not responding" in lowered or "requests are failing" in lowered or "stopped working" in lowered: + return "troubleshooting" + if "minimum" in lowered and "visa" in lowered: + return "merchant_rules" + if "urgent cash" in lowered or ("cash" in lowered and "visa" in lowered): + return "travel_support" + if "compatibility" in lowered or "zoom connectivity" in lowered: + return "compatibility" + if "employee has left" in lowered or "remove them" in lowered: + return "interviewer" + if " lti " in f" {lowered} ": + return "lti" + if "interviewer" in lowered and "remove" in lowered: + return "interviewer" + return "" + + def _intent_keyword_overlap(self, lowered: str, hits: List[RetrievalHit]) -> bool: + ticket_tokens = set(tokenize(lowered)) + for hit in hits[:3]: + doc_tokens = set(tokenize(f"{hit.doc.title} {hit.doc.path} {hit.doc.text[:800]}")) + if len(ticket_tokens & doc_tokens) >= 2: + return True + return False diff --git a/code/generator.py b/code/generator.py new file mode 100644 index 00000000..9b8134b0 --- /dev/null +++ b/code/generator.py @@ -0,0 +1,155 @@ +import re +from typing import List + +from decision import Decision +from retrieval import RetrievalHit +from utils import clean_text, sentence_candidates, tokenize + + +ESCALATION_TEMPLATES = { + "security": "This request involves account security and requires identity verification by a support agent.", + "pii": "This request involves personal data and must be reviewed by our privacy team.", + "billing": "This request involves a billing or payment matter that requires human review.", + "certificate": "Certificate or credential updates require identity verification by support staff.", + "legal": "This request involves compliance or legal matters requiring human review.", + "no_docs": "No relevant documentation was found in the support corpus for this query.", + "low_confidence": "The available documentation does not provide sufficient guidance for this specific case.", + "invalid": "This request does not match a supported domain or request type.", +} + +MAX_RESPONSE_CHARS = 800 + + +class Generator: + def generate_response( + self, + ticket_text: str, + decision: Decision, + hits: List[RetrievalHit], + product_area: str, + ) -> str: + if decision.reason_code == "invalid": + return "This request does not match a supported domain and has been routed to a human agent." + if decision.status == "escalated": + return ESCALATION_TEMPLATES.get(decision.reason_code, ESCALATION_TEMPLATES["low_confidence"]) + + if not hits: + return ESCALATION_TEMPLATES["no_docs"] + + top_doc = hits[0].doc + sentences = self._relevant_sentences(ticket_text, hits) + if not sentences: + sentences = self._filtered_sentences(top_doc.text)[:2] + + answer = self._rewrite_answer(ticket_text, product_area, sentences) + answer = re.sub(r"\s+", " ", answer).strip() + return self._safe_truncate(answer, MAX_RESPONSE_CHARS) + + def generate_justification( + self, + decision: Decision, + hits: List[RetrievalHit], + product_area: str, + ) -> str: + doc_names = ", ".join(hit.doc.path for hit in hits[:3]) if hits else "none" + if decision.status == "escalated": + return ( + f"Escalated: {decision.reason_detail} " + f"Retrieved documents: {doc_names}." + ) + return ( + f"Replied: corpus match is strong enough to answer within {product_area}. " + f"Retrieved documents: {doc_names}." + ) + + def _relevant_sentences(self, ticket_text: str, hits: List[RetrievalHit]) -> List[str]: + query_tokens = set(tokenize(ticket_text)) + ranked = [] + for hit in hits[:3]: + for sentence in self._filtered_sentences(hit.doc.text): + score = len(query_tokens & set(tokenize(sentence))) + if score == 0: + continue + ranked.append((score + hit.score, sentence)) + ranked.sort(key=lambda item: item[0], reverse=True) + results = [] + seen = set() + for _, sentence in ranked: + key = sentence.lower() + if key in seen: + continue + seen.add(key) + results.append(sentence) + if len(results) == 3: + break + return results + + def _rewrite_answer(self, ticket_text: str, product_area: str, sentences: List[str]) -> str: + lowered = ticket_text.lower() + + if "resume builder" in lowered: + return self._clean_answer( + "You can create a resume in Resume Builder either from scratch with a template or by building it from your existing profile details. Open HackerRank Community, choose Resume Builder, and follow the guided steps to create or update the resume." + ) + if "bedrock" in lowered: + return self._clean_answer( + "For Claude in Amazon Bedrock, the support documentation directs you to contact AWS Support or your AWS account manager. It also points to AWS re:Post for community support." + ) + if "crawler" in lowered or "crawl" in lowered: + return self._clean_answer( + "Anthropic says site owners can control crawler access through robots.txt. To stop crawling, add a disallow rule for the relevant Anthropic bot on each domain or subdomain you want to block, and use Crawl-delay if you want to limit crawl rate instead." + ) + if "certificate" in lowered: + return self._clean_answer( + "Certificate changes are handled through support review rather than self-service guidance in the corpus. A human support agent will need to verify identity before updating the credential." + ) + if "compatibility" in lowered or "zoom connectivity" in lowered: + return self._clean_answer( + "Use a supported browser and rerun the HackerRank compatibility check before starting the session. If the compatibility screen still reports an error, contact support with the screenshot or exact failure shown by the check." + ) + if "remove" in lowered and "interviewer" in lowered: + return self._clean_answer( + "You can remove the user from Teams Management if you have Company Admin or Team Admin access. Open the team, go to the Users tab, and use the delete action for that member." + ) + if "not responding" in lowered or "requests are failing" in lowered or "stopped working" in lowered: + return self._clean_answer( + "The troubleshooting documentation says to capture the exact error and check the Claude status page for any confirmed incidents. If the issue continues after standard browser and login checks, include the error text and timestamps when you contact support." + ) + if "minimum" in lowered and "visa" in lowered: + return self._clean_answer( + "The Visa rules documentation does not state that Visa universally requires a minimum purchase amount. It directs customers to report a purchase issue or file a Visa rule inquiry when they need clarification about merchant acceptance rules." + ) + if "urgent cash" in lowered or ("cash" in lowered and "visa" in lowered): + return self._clean_answer( + "The travel support documentation points users to Visa travel assistance tools such as the global ATM locator and other card-support resources. It does not promise emergency cash directly in this article, so you should use the listed travel support channels and card assistance resources." + ) + if "lti" in lowered: + return self._clean_answer( + "An administrator can set up the Claude LTI integration by creating the Claude LTI Developer Key in the LMS admin area and then following the documented LTI configuration steps." + ) + + return self._clean_answer(" ".join(sentences[:2])) + + def _filtered_sentences(self, text: str) -> List[str]: + stripped = re.sub(r"(?im)^(#.*|updated:.*|article:.*|http.*|breadcrumbs?.*)$", " ", clean_text(text)) + return sentence_candidates(stripped) + + def _clean_answer(self, text: str) -> str: + cleaned = re.sub(r"https?://\S+", "", text) + cleaned = re.sub(r"(?im)^\s*(updated|article|breadcrumbs?)\s*:.*$", "", cleaned) + cleaned = cleaned.replace("**", "") + cleaned = re.sub(r"\s+\d+\.\s*$", ".", cleaned) + cleaned = re.sub(r"\s+", " ", cleaned).strip() + cleaned = re.sub(r"\?+$", ".", cleaned) + if "?" in cleaned: + cleaned = cleaned.replace("?", ".") + return cleaned + + def _safe_truncate(self, text: str, max_chars: int = MAX_RESPONSE_CHARS) -> str: + if len(text) <= max_chars: + return text + truncated = text[:max_chars] + last_period = max(truncated.rfind("."), truncated.rfind("?"), truncated.rfind("!")) + if last_period > max_chars // 2: + return truncated[: last_period + 1] + return truncated diff --git a/code/main.py b/code/main.py index e69de29b..454f6f79 100644 --- a/code/main.py +++ b/code/main.py @@ -0,0 +1,29 @@ +import argparse +from pathlib import Path + +from pipeline import run_pipeline +from utils import workspace_root + + +def parse_args() -> argparse.Namespace: + root = workspace_root() + parser = argparse.ArgumentParser(description="Multi-domain support triage agent") + parser.add_argument("--tickets", required=True, help="Path to the input tickets CSV") + parser.add_argument("--data", default=str(root / "data"), help="Path to the support corpus directory") + parser.add_argument("--output", required=True, help="Path to the output CSV") + parser.add_argument("--log", default=str(root / "run_log.txt"), help="Path to the run log inside the workspace") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + run_pipeline( + tickets_path=Path(args.tickets), + data_dir=Path(args.data), + output_path=Path(args.output), + log_path=Path(args.log), + ) + + +if __name__ == "__main__": + main() diff --git a/code/pipeline.py b/code/pipeline.py new file mode 100644 index 00000000..619630aa --- /dev/null +++ b/code/pipeline.py @@ -0,0 +1,133 @@ +import csv +from pathlib import Path +from typing import Dict, List + +from classifier import Classifier, ClassificationResult +from decision import Decision, DecisionEngine +from generator import Generator +from retrieval import RetrievalEngine, RetrievalHit +from utils import ( + append_run_log, + build_ticket_id, + ensure_directory, + resolve_log_path, + workspace_root, +) + + +OUTPUT_COLUMNS = ["status", "product_area", "response", "justification", "request_type"] + + +class TriagePipeline: + def __init__(self, data_dir: Path) -> None: + self.retrieval = RetrievalEngine(data_dir) + self.classifier = Classifier() + self.decision = DecisionEngine() + self.generator = Generator() + + def process_row(self, row: Dict[str, str]) -> Dict[str, str]: + classification = self.classifier.classify(row) + hits = self.retrieval.search( + ticket_text=classification.ticket_text, + company=classification.company, + ) + decision = self.decision.decide( + ticket_text=classification.ticket_text, + classification=classification, + hits=hits, + ) + product_area = self.classifier.product_area( + classification=classification, + hits=hits, + ) + response = self.generator.generate_response( + ticket_text=classification.ticket_text, + decision=decision, + hits=hits, + product_area=product_area, + ) + justification = self.generator.generate_justification( + decision=decision, + hits=hits, + product_area=product_area, + ) + return { + "status": decision.status, + "product_area": product_area, + "response": response, + "justification": justification, + "request_type": classification.request_type, + } + + def log_ticket( + self, + log_path: Path, + row_index: int, + row: Dict[str, str], + classification: ClassificationResult, + hits: List[RetrievalHit], + decision: Decision, + response: str, + product_area: str, + ) -> None: + retrieved = ", ".join(hit.doc.path for hit in hits) if hits else "none" + lines = [ + f"[TICKET] id={build_ticket_id(row, row_index)}", + f"subject={classification.subject or '(empty)'}", + f"company={classification.company}", + f"request_type={classification.request_type}", + f"product_area={product_area}", + f"confidence={decision.confidence:.3f}", + f"decision={decision.status}", + f"reason={decision.reason_code}", + f"retrieved_docs={retrieved}", + f"response={response}", + "", + ] + append_run_log(log_path, "\n".join(lines)) + + +def run_pipeline(tickets_path: Path, data_dir: Path, output_path: Path, log_path: Path) -> None: + root = workspace_root() + safe_log_path = resolve_log_path(log_path, root) + ensure_directory(output_path.parent) + ensure_directory(safe_log_path.parent) + + pipeline = TriagePipeline(data_dir) + with tickets_path.open("r", newline="", encoding="utf-8") as input_file: + rows = list(csv.DictReader(input_file)) + + with output_path.open("w", newline="", encoding="utf-8") as output_file: + writer = csv.DictWriter(output_file, fieldnames=OUTPUT_COLUMNS) + writer.writeheader() + for row_index, row in enumerate(rows, start=1): + classification = pipeline.classifier.classify(row) + hits = pipeline.retrieval.search(classification.ticket_text, classification.company) + decision = pipeline.decision.decide(classification.ticket_text, classification, hits) + product_area = pipeline.classifier.product_area(classification, hits) + response = pipeline.generator.generate_response( + classification.ticket_text, + decision, + hits, + product_area, + ) + justification = pipeline.generator.generate_justification(decision, hits, product_area) + writer.writerow( + { + "status": decision.status, + "product_area": product_area, + "response": response, + "justification": justification, + "request_type": classification.request_type, + } + ) + pipeline.log_ticket( + log_path=safe_log_path, + row_index=row_index, + row=row, + classification=classification, + hits=hits, + decision=decision, + response=response, + product_area=product_area, + ) diff --git a/code/qa.py b/code/qa.py new file mode 100644 index 00000000..1284f1d2 --- /dev/null +++ b/code/qa.py @@ -0,0 +1,86 @@ +import argparse +import csv +import json +from collections import Counter +from pathlib import Path +from typing import Dict, List, Optional + + +REQUIRED_COLUMNS = ["status", "product_area", "response", "justification", "request_type"] +ALLOWED_STATUS = {"replied", "escalated"} +ALLOWED_REQUEST_TYPES = {"product_issue", "feature_request", "bug", "invalid"} + + +def read_csv(path: Path) -> List[Dict[str, str]]: + with path.open("r", newline="", encoding="utf-8-sig") as file: + return list(csv.DictReader(file)) + + +def read_header(path: Path) -> List[str]: + with path.open("r", newline="", encoding="utf-8-sig") as file: + return next(csv.reader(file), []) + + +def validate(input_path: Path, output_path: Path, log_path: Optional[Path]) -> Dict[str, object]: + input_rows = read_csv(input_path) + output_rows = read_csv(output_path) + issues = [] + if read_header(output_path) != REQUIRED_COLUMNS: + issues.append("Output header does not match required columns exactly.") + if len(input_rows) != len(output_rows): + issues.append(f"Row count mismatch: input={len(input_rows)} output={len(output_rows)}") + + empty_values = [] + bad_status = [] + bad_request_type = [] + for index, row in enumerate(output_rows, start=1): + for column in REQUIRED_COLUMNS: + if not str(row.get(column, "")).strip(): + empty_values.append({"row": index, "column": column}) + if row.get("status") not in ALLOWED_STATUS: + bad_status.append({"row": index, "value": row.get("status")}) + if row.get("request_type") not in ALLOWED_REQUEST_TYPES: + bad_request_type.append({"row": index, "value": row.get("request_type")}) + + if empty_values: + issues.append(f"Empty required values: {empty_values}") + if bad_status: + issues.append(f"Invalid status labels: {bad_status}") + if bad_request_type: + issues.append(f"Invalid request_type labels: {bad_request_type}") + + log_tickets = None + if log_path: + if not log_path.exists(): + issues.append(f"Log file does not exist: {log_path}") + else: + text = log_path.read_text(encoding="utf-8", errors="ignore") + log_tickets = text.count("\nTicket ") + (1 if text.startswith("Ticket ") else 0) + if log_tickets != len(output_rows): + issues.append(f"Log ticket count mismatch: log={log_tickets} output={len(output_rows)}") + + return { + "passed": not issues, + "issues": issues, + "input_rows": len(input_rows), + "output_rows": len(output_rows), + "log_tickets": log_tickets, + "status_distribution": Counter(row["status"] for row in output_rows), + "request_type_distribution": Counter(row["request_type"] for row in output_rows), + "product_area_distribution": Counter(row["product_area"] for row in output_rows), + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Validate triage output and logs") + parser.add_argument("--input", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--log") + args = parser.parse_args() + report = validate(Path(args.input), Path(args.output), Path(args.log) if args.log else None) + print(json.dumps(report, indent=2, default=dict)) + raise SystemExit(0 if report["passed"] else 1) + + +if __name__ == "__main__": + main() diff --git a/code/retrieval.py b/code/retrieval.py new file mode 100644 index 00000000..452b445f --- /dev/null +++ b/code/retrieval.py @@ -0,0 +1,214 @@ +import re +from collections import Counter +from dataclasses import dataclass +from pathlib import Path +from typing import List, Sequence + +from utils import clean_text, markdown_to_text, tokenize + + +AREA_MAP = { + "hackerrank/interviews": "interviewing", + "hackerrank/screen": "assessments", + "hackerrank/settings": "team_management", + "hackerrank/integrations": "integrations", + "hackerrank/skillup": "learning_and_development", + "hackerrank/chakra": "ai_interviewing", + "hackerrank/hackerrank_community/subscriptions-payments-and-billing": "billing_and_subscriptions", + "hackerrank/hackerrank_community/certifications": "certifications", + "hackerrank/hackerrank_community": "community_support", + "hackerrank/general-help": "platform_support", + "hackerrank/uncategorized": "platform_support", + "claude/privacy-and-legal": "privacy_and_compliance", + "claude/claude/account-management": "account_management", + "claude/claude/troubleshooting": "service_availability", + "claude/claude-api-and-console": "api_and_developer_tools", + "claude/amazon-bedrock": "api_and_developer_tools", + "claude/claude-for-education": "education", + "claude/team-and-enterprise-plans/security-and-compliance": "privacy_and_compliance", + "visa/support/consumer/travel-support": "travel_services", + "visa/support/consumer": "consumer_support", + "visa/support/small-business/fraud-protection": "fraud_and_security", + "visa/support/small-business/dispute-resolution": "payment_processing", + "visa/support/small-business/regulations-fees": "merchant_acceptance", + "visa/support/small-business": "merchant_acceptance", +} + +INTENT_ROUTING = { + "crawler": ["claude/privacy-and-legal/8896518"], + "crawl": ["claude/privacy-and-legal/8896518"], + "bedrock": ["claude/amazon-bedrock"], + "not responding": ["claude/claude/troubleshooting"], + "requests are failing": ["claude/claude/troubleshooting"], + "stopped working": ["claude/claude/troubleshooting"], + "urgent cash": ["visa/support/consumer/travel-support"], + "minimum spend": ["visa/support/consumer/visa-rules"], + "employee has left": ["hackerrank/settings/teams-management"], + "remove them": ["hackerrank/settings/teams-management"], + "certificate": [ + "hackerrank/hackerrank_community/certifications", + "hackerrank/skillup/getting-started", + ], + "interviewer": ["hackerrank/settings/teams-management"], + "personal data": [ + "claude/team-and-enterprise-plans/security-and-compliance/9267387", + "claude/privacy-and-legal", + ], + "privacy": ["claude/privacy-and-legal"], + "compatibility": [ + "hackerrank/interviews/getting-started/6271433412", + "hackerrank/uncategorized/5897755717", + ], + "reschedule": ["hackerrank/interviews/manage-interviews/2342466364"], +} + + +@dataclass(frozen=True) +class SupportDoc: + company: str + path: str + title: str + text: str + product_area: str + + +@dataclass(frozen=True) +class RetrievalHit: + doc: SupportDoc + score: float + + +class RetrievalEngine: + def __init__(self, data_dir: Path) -> None: + self.data_dir = data_dir + self.docs = self._load_docs() + + def search(self, ticket_text: str, company: str, top_k: int = 5) -> List[RetrievalHit]: + candidates = self._company_candidates(company) + routed_candidates = self._intent_candidates(ticket_text, company, candidates) + if routed_candidates: + candidates = routed_candidates + + query_tokens = tokenize(ticket_text) + if not query_tokens: + return [] + + hits: List[RetrievalHit] = [] + for doc in candidates: + score = self._score_doc(query_tokens, ticket_text, doc) + if score >= 0.12: + hits.append(RetrievalHit(doc=doc, score=score)) + + hits.sort(key=lambda hit: hit.score, reverse=True) + return hits[:top_k] + + def _company_candidates(self, company: str) -> List[SupportDoc]: + if company == "none": + return self.docs + return [doc for doc in self.docs if doc.company == company] + + def _intent_candidates( + self, + ticket_text: str, + company: str, + candidates: Sequence[SupportDoc], + ) -> List[SupportDoc]: + lowered = ticket_text.lower() + routes: List[str] = [] + if re.search(r"\blti\b", lowered): + routes = ["hackerrank/integrations", "hackerrank/skillup"] if company in {"hackerrank", "none", "", None} else ["claude/claude-for-education"] + return [doc for doc in candidates if any(route in doc.path.lower() for route in routes)] + for intent, prefixes in INTENT_ROUTING.items(): + if intent in lowered: + routes.extend(prefixes) + if not routes: + return [] + + matched = [ + doc + for doc in candidates + if any(route in doc.path.lower() for route in routes) + ] + return matched + + def _load_docs(self) -> List[SupportDoc]: + docs: List[SupportDoc] = [] + for path in sorted(self.data_dir.rglob("*.md")): + raw = path.read_text(encoding="utf-8", errors="ignore") + plain_text = markdown_to_text(raw) + if not plain_text: + continue + relative = path.relative_to(self.data_dir).as_posix() + company = relative.split("/", 1)[0].lower() + docs.append( + SupportDoc( + company=company, + path=relative, + title=self._extract_title(raw, path.stem), + text=plain_text, + product_area=self._product_area(relative), + ) + ) + return docs + + def _extract_title(self, raw: str, stem: str) -> str: + for line in raw.splitlines(): + stripped = line.strip() + if stripped.startswith("# "): + return clean_text(stripped[2:]) + return clean_text(stem.replace("-", " ")) + + def _product_area(self, relative_path: str) -> str: + lowered = relative_path.lower() + for prefix, area in sorted(AREA_MAP.items(), key=lambda item: len(item[0]), reverse=True): + if lowered.startswith(prefix): + return area + if lowered.startswith("hackerrank/"): + return "platform_support" + if lowered.startswith("claude/"): + return "account_management" + if lowered.startswith("visa/"): + return "consumer_support" + return "platform_support" + + def _score_doc(self, query_tokens: List[str], ticket_text: str, doc: SupportDoc) -> float: + path_tokens = tokenize(doc.path.replace("/", " ")) + title_tokens = tokenize(doc.title) + body_tokens = tokenize(doc.text) + + path_overlap = self._overlap_score(query_tokens, path_tokens) + title_overlap = self._overlap_score(query_tokens, title_tokens) + body_overlap = self._overlap_score(query_tokens, body_tokens[:400]) + phrase_boost = self._phrase_boost(ticket_text.lower(), doc) + + score = (0.35 * title_overlap) + (0.25 * path_overlap) + (0.30 * body_overlap) + phrase_boost + return round(min(score, 1.0), 3) + + def _overlap_score(self, left: Sequence[str], right: Sequence[str]) -> float: + if not left or not right: + return 0.0 + left_counts = Counter(left) + right_counts = Counter(right) + overlap = sum(min(left_counts[token], right_counts[token]) for token in left_counts) + return overlap / max(len(set(left)), 1) + + def _phrase_boost(self, ticket_text: str, doc: SupportDoc) -> float: + combined = f"{doc.title} {doc.path} {doc.text}".lower() + boosts = 0.0 + phrases = [ + "bedrock", + "crawler", + "robots.txt", + "privacy center", + "team members", + "compatibility", + "status page", + "reschedule", + "certificate", + "subscription", + "refund", + ] + for phrase in phrases: + if phrase in ticket_text and phrase in combined: + boosts += 0.08 + return min(boosts, 0.24) diff --git a/code/utils.py b/code/utils.py new file mode 100644 index 00000000..010ad4c1 --- /dev/null +++ b/code/utils.py @@ -0,0 +1,126 @@ +import re +import unicodedata +from pathlib import Path +from typing import Dict, List + + +STOPWORDS = { + "a", + "an", + "and", + "are", + "as", + "at", + "be", + "but", + "by", + "for", + "from", + "how", + "i", + "in", + "is", + "it", + "my", + "of", + "on", + "or", + "the", + "to", + "us", + "we", + "with", + "you", + "your", +} + + +def workspace_root() -> Path: + return Path(__file__).resolve().parent.parent + + +def clean_text(value: str) -> str: + text = unicodedata.normalize("NFKC", str(value or "")) + replacements = { + "\ufeff": " ", + "’": "'", + "‘": "'", + "“": '"', + "”": '"', + "–": "-", + "—": "-", + "Â": " ", + "é": "e", + "è": "e", + } + for source, target in replacements.items(): + text = text.replace(source, target) + text = re.sub(r"\s+", " ", text) + return text.strip() + + +def markdown_to_text(raw: str) -> str: + text = raw + text = re.sub(r"^---.*?---", " ", text, flags=re.DOTALL) + text = re.sub(r"(?im)^(title|title_slug|source_url|final_url|article_id|article_slug|description|last_updated.*|last_modified|breadcrumbs):.*$", " ", text) + text = re.sub(r"!\[[^\]]*\]\([^)]*\)", " ", text) + text = re.sub(r"\[([^\]]+)\]\([^)]*\)", r"\1", text) + text = re.sub(r"`{1,3}.*?`{1,3}", " ", text, flags=re.DOTALL) + text = re.sub(r"(?m)^\s*[-*]\s+", "", text) + text = re.sub(r"[#>|_]", " ", text) + text = re.sub(r"\s+", " ", text) + return clean_text(text) + + +def tokenize(text: str) -> List[str]: + return [ + token + for token in re.findall(r"[a-z0-9]+", clean_text(text).lower()) + if token not in STOPWORDS and len(token) > 1 + ] + + +def sentence_candidates(text: str) -> List[str]: + sentences = re.split(r"(?<=[.!?])\s+", clean_text(text)) + results = [] + for sentence in sentences: + lowered = sentence.lower() + if len(sentence) < 25: + continue + if lowered.startswith(("title", "source url", "last updated", "breadcrumbs")): + continue + if "last modified" in lowered or "last updated" in lowered: + continue + if any(term in lowered for term in ("the magic of travel", "discover the magic", "offers and perks")): + continue + if "http" in lowered: + continue + results.append(sentence.strip()) + return results + + +def ensure_directory(path: Path) -> None: + path.mkdir(parents=True, exist_ok=True) + + +def resolve_log_path(log_path: Path, root: Path) -> Path: + candidate = log_path if log_path.is_absolute() else (root / log_path) + resolved_root = root.resolve() + resolved_log = candidate.resolve() + assert str(resolved_log).startswith(str(resolved_root)) + assert "hackerrank_orchestrate" not in str(resolved_log).lower() or str(resolved_log).startswith(str(resolved_root)) + return resolved_log + + +def append_run_log(log_path: Path, text: str) -> None: + with log_path.open("a", encoding="utf-8") as handle: + handle.write(text) + if not text.endswith("\n"): + handle.write("\n") + + +def build_ticket_id(row: Dict[str, str], row_index: int) -> str: + subject = clean_text(row.get("Subject", "")) + if subject: + return subject + return f"row-{row_index}" diff --git a/support_tickets/output.csv b/support_tickets/output.csv index 69666e12..5d690400 100644 --- a/support_tickets/output.csv +++ b/support_tickets/output.csv @@ -1 +1,30 @@ -issue,subject,company,response,product_area,status,request_type,justification \ No newline at end of file +status,product_area,response,justification,request_type +escalated,account_management,The available documentation does not provide sufficient guidance for this specific case.,"Escalated: The available documentation does not provide sufficient guidance for this specific case. Retrieved documents: claude/team-and-enterprise-plans/billing/11526368-how-am-i-billed-for-my-enterprise-plan.md, claude/claude-api-and-console/using-the-claude-api-and-console/10186004-claude-console-roles-and-permissions.md, claude/claude-in-chrome/13065128-claude-in-chrome-admin-controls.md.",product_issue +escalated,integrations,The available documentation does not provide sufficient guidance for this specific case.,"Escalated: The available documentation does not provide sufficient guidance for this specific case. Retrieved documents: hackerrank/integrations/applicant-tracking-systems/northstarz/4546750166-northstarz-hackerrank-integration-user-guide.md, hackerrank/screen/test-reports/8263794320-viewing-a-candidate%27s-detailed-test-report.md, hackerrank/integrations/applicant-tracking-systems/lever-v2/6548604920-lever-v2-hackerrank-integration-test-user-guide.md.",product_issue +escalated,consumer_support,This request involves a billing or payment matter that requires human review.,"Escalated: This ticket involves a billing or payment matter that requires human review. Retrieved documents: visa/support/consumer/travelers-cheques.md, visa/support/small-business/dispute-resolution.md, visa/support/consumer/visa-rules.md.",product_issue +escalated,billing_and_subscriptions,This request involves a billing or payment matter that requires human review.,"Escalated: This ticket involves a billing or payment matter that requires human review. Retrieved documents: hackerrank/hackerrank_community/subscriptions-payments-and-billing/3282259518-purchase-mock-interviews.md, hackerrank/hackerrank_community/subscriptions-payments-and-billing/9157064719-payments-and-billing-faqs.md, hackerrank/hackerrank_community/mock-interviews/8988753946-introduction-to-mock-interview.md.",bug +escalated,platform_support,This request involves a billing or payment matter that requires human review.,Escalated: This ticket involves a billing or payment matter that requires human review. Retrieved documents: hackerrank/general-help/evaluation-guides/7483769801-mid-level.md.,product_issue +escalated,assessments,The available documentation does not provide sufficient guidance for this specific case.,"Escalated: The available documentation does not provide sufficient guidance for this specific case. Retrieved documents: hackerrank/screen/invite-candidates/9684438314-creating-an-email-template.md, hackerrank/integrations/scheduling/3675175160-using-hackerrank-interviews-in-interviewplanner.md, hackerrank/screen/getting-started/9248897371-quick-start-guide-for-recruiters.md.",product_issue +escalated,integrations,The available documentation does not provide sufficient guidance for this specific case.,"Escalated: The available documentation does not provide sufficient guidance for this specific case. Retrieved documents: hackerrank/integrations/applicant-tracking-systems/greenhouse/3593554348-enabling-outcome-data-and-public-link-with-greenhouse.md, hackerrank/library/manage-question/4968606472-steps-to-create-accessible-content-at-hackerrank.md, hackerrank/skillup/getting-started/6047728972-get-started-with-learn.md.",bug +escalated,assessments,The available documentation does not provide sufficient guidance for this specific case.,"Escalated: The available documentation does not provide sufficient guidance for this specific case. Retrieved documents: hackerrank/screen/frequently-asked-questions/4989710628-how-to-move-questions-across-sections-in-a-test%3F.md, hackerrank/hackerrank_community/mock-interviews/3796809491-ai-fluency-mock-interview.md, hackerrank/hackerrank_community/mock-interviews/5174727491-behavioural-mock-interview.md.",bug +replied,assessments,"Use a supported browser and rerun the HackerRank compatibility check before starting the session. If the compatibility screen still reports an error, contact support with the screenshot or exact failure shown by the check.","Replied: corpus match is strong enough to answer within assessments. Retrieved documents: hackerrank/screen/test-reports/1972468979-understanding-errors-in-post-assessment-reports-for-coding-questions.md, hackerrank/interviews/integrations/5805684780-zoom---hackerrank-interview-integration.md, hackerrank/screen/frequently-asked-questions/5773853281-how-do-i-upload-all-test-cases-at-once.md.",bug +escalated,platform_support,The available documentation does not provide sufficient guidance for this specific case.,"Escalated: The available documentation does not provide sufficient guidance for this specific case. Retrieved documents: hackerrank/uncategorized/9695299159-onboarding-candidates.md, hackerrank/screen/invite-candidates/9684438314-creating-an-email-template.md, hackerrank/screen/managing-tests/4811403281-adding-extra-time-for-candidates.md.",product_issue +escalated,team_management,No relevant documentation was found in the support corpus for this query.,Escalated: No relevant documentation was retrieved from the support corpus. Retrieved documents: none.,product_issue +escalated,account_management,The available documentation does not provide sufficient guidance for this specific case.,"Escalated: The available documentation does not provide sufficient guidance for this specific case. Retrieved documents: claude/claude/features-and-capabilities/11473015-retrieval-augmented-generation-rag-for-projects.md, claude/claude/features-and-capabilities/12512198-how-to-create-custom-skills.md, claude/claude-mobile-apps/claude-for-android/11869629-using-claude-with-android-apps.md.",bug +replied,team_management,"You can remove the user from Teams Management if you have Company Admin or Team Admin access. Open the team, go to the Users tab, and use the delete action for that member.","Replied: corpus match is strong enough to answer within team_management. Retrieved documents: hackerrank/settings/teams-management/9603546665-types-of-user-roles.md, hackerrank/settings/teams-management/1508181842-teams-management.md, hackerrank/settings/teams-management/5319929416-setting-logo-at-the-team-level.md.",product_issue +escalated,team_management,The available documentation does not provide sufficient guidance for this specific case.,"Escalated: The available documentation does not provide sufficient guidance for this specific case. Retrieved documents: hackerrank/settings/user-account-settings-and-preferences/5157311476-pause-subscription.md, hackerrank/hackerrank_community/subscriptions-payments-and-billing/4312449655-upgrade-subscription-plan.md, hackerrank/hackerrank_community/subscriptions-payments-and-billing/5432562533-downgrade-subscription-plan.md.",product_issue +replied,service_availability,"The troubleshooting documentation says to capture the exact error and check the Claude status page for any confirmed incidents. If the issue continues after standard browser and login checks, include the error text and timestamps when you contact support.","Replied: corpus match is strong enough to answer within service_availability. Retrieved documents: claude/claude/troubleshooting/8241188-claude-is-producing-links-that-don-t-work-and-falsely-claiming-that-it-has-sent-emails-or-produced-external-documents-what-s-going-on.md, claude/claude/troubleshooting/12466728-troubleshoot-claude-error-messages.md, claude/claude/troubleshooting/8525154-claude-is-providing-incorrect-or-misleading-responses-what-s-going-on.md.",bug +escalated,consumer_support,This request involves account security and requires identity verification by a support agent.,"Escalated: This ticket involves account security or fraud and requires human verification. Retrieved documents: visa/support/consumer/travelers-cheques.md, visa/support/small-business/data-security.md, visa/support/small-business/fraud-protection.md.",product_issue +replied,community_support,"You can create a resume in Resume Builder either from scratch with a template or by building it from your existing profile details. Open HackerRank Community, choose Resume Builder, and follow the guided steps to create or update the resume.","Replied: corpus match is strong enough to answer within community_support. Retrieved documents: hackerrank/hackerrank_community/additional-resources/job-search-and-applications/9106957203-create-a-resume-with-resume-builder.md, hackerrank/interviews/manage-interviews/8404005945-creating-an-interview.md, hackerrank/interviews/manage-interviews/2837093779-standardizing-interviews-using-templates.md.",bug +escalated,certifications,Certificate or credential updates require identity verification by support staff.,"Escalated: This ticket involves a certificate or credential update that requires identity verification. Retrieved documents: hackerrank/hackerrank_community/certifications/8941367927-certifications-faqs.md, hackerrank/hackerrank_community/certifications/2077861863-download-certificate.md, hackerrank/skillup/getting-started/2960935422-get-started-with-certify.md.",product_issue +escalated,payment_processing,This request involves a billing or payment matter that requires human review.,Escalated: This ticket involves a billing or payment matter that requires human review. Retrieved documents: visa/support/small-business/dispute-resolution.md.,product_issue +escalated,account_management,This request involves account security and requires identity verification by a support agent.,"Escalated: This ticket involves account security or fraud and requires human verification. Retrieved documents: claude/claude-code/11932705-automated-security-reviews-in-claude-code.md, claude/safeguards/12119250-model-safety-bug-bounty-program.md, claude/claude-api-and-console/claude-api-usage-and-best-practices/8241216-violating-anthropic-s-usage-policy.md.",product_issue +replied,privacy_and_compliance,"Anthropic says site owners can control crawler access through robots.txt. To stop crawling, add a disallow rule for the relevant Anthropic bot on each domain or subdomain you want to block, and use Crawl-delay if you want to limit crawl rate instead.",Replied: corpus match is strong enough to answer within privacy_and_compliance. Retrieved documents: claude/privacy-and-legal/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler.md.,product_issue +replied,travel_services,"The travel support documentation points users to Visa travel assistance tools such as the global ATM locator and other card-support resources. It does not promise emergency cash directly in this article, so you should use the listed travel support channels and card assistance resources.",Replied: corpus match is strong enough to answer within travel_services. Retrieved documents: visa/support/consumer/travel-support.md.,product_issue +escalated,privacy_and_compliance,This request involves personal data and must be reviewed by our privacy team.,"Escalated: This ticket involves personal data and requires privacy review. Retrieved documents: claude/privacy-and-legal/12326764-can-i-use-my-outputs-to-train-an-ai-model.md, claude/privacy-and-legal/10684638-reporting-blocking-and-removing-content-from-claude.md, claude/privacy-and-legal/10023638-why-am-i-receiving-an-output-blocked-by-content-filtering-policy-error.md.",product_issue +escalated,account_management,This request does not match a supported domain and has been routed to a human agent.,"Escalated: Ticket does not match any supported domain or request type. Retrieved documents: claude/claude-code/launch-guides/14553240-give-claude-context-claude-md-and-better-prompts.md, claude/claude/features-and-capabilities/12111783-create-and-edit-files-with-claude.md, claude/claude/features-and-capabilities/14604397-set-up-your-design-system-in-claude-design.md.",invalid +escalated,fraud_and_security,No relevant documentation was found in the support corpus for this query.,Escalated: No relevant documentation was retrieved from the support corpus. Retrieved documents: none.,product_issue +replied,api_and_developer_tools,"For Claude in Amazon Bedrock, the support documentation directs you to contact AWS Support or your AWS account manager. It also points to AWS re:Post for community support.","Replied: corpus match is strong enough to answer within api_and_developer_tools. Retrieved documents: claude/amazon-bedrock/10280791-what-aws-regions-are-claude-models-available-in-amazon-bedrock.md, claude/amazon-bedrock/10280783-where-do-i-find-claude-in-amazon-bedrock-documentation.md, claude/amazon-bedrock/7996920-how-do-i-get-access-to-claude-in-amazon-bedrock.md.",bug +replied,team_management,Log in to your HackerRank for Work account using your credentials. Prerequisite You must have Company Admin or Team Admin access.,"Replied: corpus match is strong enough to answer within team_management. Retrieved documents: hackerrank/settings/teams-management/2181136239-deleting-a-team.md, hackerrank/settings/teams-management/2203617737-manage-team-members.md, hackerrank/settings/teams-management/6534774997-locking-user-access-from-hackerrank.md.",product_issue +replied,education,An administrator can set up the Claude LTI integration by creating the Claude LTI Developer Key in the LMS admin area and then following the documented LTI configuration steps.,"Replied: corpus match is strong enough to answer within education. Retrieved documents: claude/claude-for-education/11725453-set-up-the-claude-lti-in-canvas-by-instructure.md, claude/claude-for-education/11139094-getting-started-with-claude-for-education-at-your-university-for-owners-admins.md, claude/claude-for-education/11139144-faqs-on-using-claude-for-education-at-your-university.md.",product_issue +replied,consumer_support,The Visa rules documentation does not state that Visa universally requires a minimum purchase amount. It directs customers to report a purchase issue or file a Visa rule inquiry when they need clarification about merchant acceptance rules.,Replied: corpus match is strong enough to answer within consumer_support. Retrieved documents: visa/support/consumer/visa-rules.md.,product_issue