Add note qc feature (#155)

# Summary
This is a heavy but cool one! This change adds a note QC workflow. It
allows users to add checks to notes with natural language then these
checks run on publish. For example, you can verify a note vs the
transcript.

<img width="290" height="276" alt="image"
src="https://github.com/user-attachments/assets/d9af6406-4532-44b4-94bf-5e6269c8653f"
/>

Checks can be added via the settings UI

<img width="2106" height="1242" alt="2026-05-14 14 34 31"
src="https://github.com/user-attachments/assets/75cae0f1-0f25-4c3f-b572-8cb2f89fd634"
/>

These checks run in the publish UI. They can update the note itself or
the other fields on the note.

The checks have access to tools to browse the prodtracking software. An
example of where this can be used is to find attachments to plug into
the note links, tos, or CCs,

<img width="2106" height="1242" alt="2026-05-14 14 35 23"
src="https://github.com/user-attachments/assets/842f918b-d411-4cc3-a271-e3aa8d7dd13b"
/>

## Testing
- [X] I have tested these changes locally
- [ X ] I have run all relevant automated tests
- [ X ] I have verified this does not break existing workflows
- [X] For changes that can be tested in UI, I have included screenshots
or gif animations of the changes.

## How I Tested

- Created a transcript missing info and the QC catches it
- Created a note mentioning a person who was not tagged
- Checked content of a note to ensure it contained the requested
information.

---------

Signed-off-by: James Spadafora <spadjv@gmail.com>

Author: jspada200

backend/requirements.txt

@@ -1,16 +1,17 @@
 fastapi==0.104.1
 uvicorn[standard]==0.24.0
-pydantic==2.5.0
-pydantic-settings==2.1.0
+pydantic==2.13.4
+pydantic-settings==2.8.1
+instructor==1.15.1
 pytest==7.4.3
 pytest-cov==4.1.0
 pytest-asyncio==0.21.1
 httpx==0.25.2
 shotgun_api3==3.9.2
 pymongo==4.10.1
 websockets==12.0
-openai==1.58.1
+openai==2.36.0
 google-auth==2.0.0
-requests==2.28.0
+requests==2.32.3
 python-multipart==0.0.9
 PyYAML==6.0.1

backend/src/dna/llm_providers/llm_provider_base.py

@@ -3,13 +3,94 @@
 Abstract base class for LLM providers and factory function.
 """
 
+import json
+import logging
 import os
-from typing import Optional
+from typing import Any, Awaitable, Callable, Optional, TypeVar
 
+import instructor
 from openai import AsyncOpenAI
+from pydantic import BaseModel
 
 from dna.prompts.generate_note_prompt import GENERATE_NOTE_PROMPT
 
+logger = logging.getLogger(__name__)
+
+T = TypeVar("T", bound=BaseModel)
+
+DEFAULT_MAX_TOOL_RESULT_CHARS = 50_000
+
+
+def _truncate_tool_result(content: str, max_chars: int) -> str:
+    if len(content) <= max_chars:
+        return content
+    suffix = "\n...[truncated]"
+    return content[: max_chars - len(suffix)] + suffix
+
+
+def _safe_parse_tool_arguments(
+    raw: str | None,
+) -> tuple[dict[str, Any] | None, str | None]:
+    text = (raw or "").strip() or "{}"
+    try:
+        parsed = json.loads(text)
+    except json.JSONDecodeError as exc:
+        err = json.dumps(
+            {
+                "error": "Invalid JSON in tool arguments.",
+                "detail": str(exc),
+            }
+        )
+        return None, err
+    if not isinstance(parsed, dict):
+        return None, json.dumps({"error": "Tool arguments must be a JSON object."})
+    return parsed, None
+
+
+def _assistant_message_with_tool_calls(
+    msg: Any, tool_calls: list[Any]
+) -> dict[str, Any]:
+    return {
+        "role": "assistant",
+        "content": msg.content,
+        "tool_calls": [
+            {
+                "id": tc.id,
+                "type": getattr(tc, "type", "function") or "function",
+                "function": {
+                    "name": tc.function.name,
+                    "arguments": tc.function.arguments or "{}",
+                },
+            }
+            for tc in tool_calls
+        ],
+    }
+
+
+async def _append_tool_use_round(
+    messages: list[dict[str, Any]],
+    msg: Any,
+    tool_calls: list[Any],
+    tool_executor: Callable[[str, dict[str, Any]], Awaitable[str]],
+    max_tool_result_chars: int,
+) -> None:
+    messages.append(_assistant_message_with_tool_calls(msg, tool_calls))
+    for tc in tool_calls:
+        args, parse_error = _safe_parse_tool_arguments(tc.function.arguments)
+        if parse_error is not None:
+            result = parse_error
+        else:
+            assert args is not None
+            result = await tool_executor(tc.function.name, args)
+        result = _truncate_tool_result(result, max_tool_result_chars)
+        messages.append(
+            {
+                "role": "tool",
+                "tool_call_id": tc.id,
+                "content": result,
+            }
+        )
+
 
 class LLMProviderBase:
     """Abstract base class for LLM providers."""
@@ -119,6 +200,135 @@ async def generate_note(
 
         return response.choices[0].message.content or ""
 
+    async def generate_with_tools(
+        self,
+        system_prompt: str,
+        user_message: str,
+        tools: list[dict[str, Any]],
+        tool_executor: Callable[[str, dict[str, Any]], Awaitable[str]],
+        max_iterations: int = 5,
+        temperature: float = 0.2,
+        max_tool_result_chars: int = DEFAULT_MAX_TOOL_RESULT_CHARS,
+    ) -> str:
+        """Run an agentic loop: LLM may call tools until it returns final text."""
+        messages: list[dict[str, Any]] = [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_message},
+        ]
+        last_text = ""
+        for _ in range(max_iterations):
+            response = await self.client.chat.completions.create(
+                model=self.model,
+                messages=messages,
+                tools=tools,
+                tool_choice="auto",
+                temperature=temperature,
+                max_tokens=2048,
+            )
+            choice = response.choices[0]
+            msg = choice.message
+            last_text = msg.content or ""
+            tool_calls = getattr(msg, "tool_calls", None) or []
+            if tool_calls:
+                await _append_tool_use_round(
+                    messages,
+                    msg,
+                    tool_calls,
+                    tool_executor,
+                    max_tool_result_chars,
+                )
+            else:
+                return last_text
+        return last_text
+
+    async def generate_structured_with_tools(
+        self,
+        system_prompt: str,
+        user_message: str,
+        tools: list[dict[str, Any]],
+        tool_executor: Callable[[str, dict[str, Any]], Awaitable[str]],
+        response_model: type[T],
+        max_iterations: int = 5,
+        temperature: float = 0.2,
+        max_tool_result_chars: int = DEFAULT_MAX_TOOL_RESULT_CHARS,
+    ) -> T:
+        """Tool-use phase then instructor-validated structured extraction."""
+        messages: list[dict[str, Any]] = [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_message},
+        ]
+        hit_limit_with_tools = False
+        for i in range(max_iterations):
+            response = await self.client.chat.completions.create(
+                model=self.model,
+                messages=messages,
+                tools=tools,
+                tool_choice="auto",
+                temperature=temperature,
+                max_tokens=2048,
+            )
+            choice = response.choices[0]
+            msg = choice.message
+            tool_calls = getattr(msg, "tool_calls", None) or []
+            if tool_calls:
+                await _append_tool_use_round(
+                    messages,
+                    msg,
+                    tool_calls,
+                    tool_executor,
+                    max_tool_result_chars,
+                )
+                if i == max_iterations - 1:
+                    hit_limit_with_tools = True
+            else:
+                messages.append(
+                    {"role": "assistant", "content": msg.content or ""},
+                )
+                hit_limit_with_tools = False
+                break
+
+        if hit_limit_with_tools:
+            logger.warning(
+                "Tool-use loop reached max_iterations=%s with pending tool rounds; "
+                "requesting a final assistant message with tool_choice=none before "
+                "structured extraction.",
+                max_iterations,
+            )
+            response = await self.client.chat.completions.create(
+                model=self.model,
+                messages=messages,
+                tools=tools,
+                tool_choice="none",
+                temperature=temperature,
+                max_tokens=2048,
+            )
+            final_msg = response.choices[0].message
+            messages.append(
+                {"role": "assistant", "content": final_msg.content or ""},
+            )
+
+        extraction_messages = list(messages)
+        extraction_messages.append(
+            {
+                "role": "user",
+                "content": (
+                    "Provide your final quality-check result for this draft and check. "
+                    "Fill every required field in the structured response schema."
+                ),
+            }
+        )
+        instructor_client = instructor.from_openai(
+            self.client,
+            mode=instructor.Mode.JSON,
+        )
+        return await instructor_client.chat.completions.create(
+            model=self.model,
+            messages=extraction_messages,
+            response_model=response_model,
+            temperature=temperature,
+            max_tokens=2048,
+        )
+
 
 def get_llm_provider() -> LLMProviderBase:
     """Factory function to get the configured LLM provider."""

backend/src/dna/models/__init__.py

@@ -27,6 +27,18 @@
     PlaylistMetadata,
     PlaylistMetadataUpdate,
 )
+from dna.models.qc_check import (
+    DEFAULT_ACTION_ITEM_CHECK,
+    NoteQCAttributeSuggestion,
+    NoteQCCheck,
+    NoteQCCheckCreate,
+    NoteQCCheckUpdate,
+    NoteQCLLMOutput,
+    NoteQCResult,
+    NoteQCSeverity,
+    RunQCChecksRequest,
+    RunQCChecksResponse,
+)
 from dna.models.requests import (
     CreateNoteRequest,
     EntityLink,
@@ -103,4 +115,14 @@
     "UserSettings",
     "UserSettingsUpdate",
     "UserSettingsResponse",
+    "NoteQCSeverity",
+    "NoteQCCheckCreate",
+    "NoteQCCheckUpdate",
+    "NoteQCCheck",
+    "NoteQCAttributeSuggestion",
+    "NoteQCLLMOutput",
+    "NoteQCResult",
+    "RunQCChecksRequest",
+    "RunQCChecksResponse",
+    "DEFAULT_ACTION_ITEM_CHECK",
 ]

backend/src/dna/models/qc_check.py

@@ -0,0 +1,106 @@
+"""Models for user-defined note QC checks and run results."""
+
+from datetime import datetime
+from typing import Literal, Optional
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from dna.models.draft_note import DraftNoteLink
+
+NoteQCSeverity = Literal["warning", "error"]
+
+
+class NoteQCCheckCreate(BaseModel):
+    """Payload for creating a QC check."""
+
+    name: str
+    prompt: str
+    severity: NoteQCSeverity
+    enabled: bool = True
+
+
+class NoteQCCheckUpdate(BaseModel):
+    """Partial update for a QC check."""
+
+    name: Optional[str] = None
+    prompt: Optional[str] = None
+    severity: Optional[NoteQCSeverity] = None
+    enabled: Optional[bool] = None
+
+
+class NoteQCCheck(BaseModel):
+    """Stored QC check definition."""
+
+    model_config = ConfigDict(populate_by_name=True)
+
+    id: str = Field(alias="_id")
+    user_email: str
+    name: str
+    prompt: str
+    severity: NoteQCSeverity
+    enabled: bool = True
+    created_at: datetime
+    updated_at: datetime
+
+
+class NoteQCAttributeSuggestion(BaseModel):
+    """Suggested updates to draft note metadata."""
+
+    to: Optional[str] = None
+    cc: Optional[str] = None
+    subject: Optional[str] = None
+    version_status: Optional[str] = Field(
+        default=None,
+        description="Suggested version status (maps to draft version_status).",
+    )
+    links: Optional[list[DraftNoteLink]] = None
+
+
+class NoteQCLLMOutput(BaseModel):
+    """Structured QC verdict returned by the LLM (instructor-validated)."""
+
+    passed: bool
+    issue: Optional[str] = None
+    evidence: Optional[str] = None
+    note_suggestion: Optional[str] = Field(
+        default=None,
+        description="Full suggested note body when the check fails.",
+    )
+    attribute_suggestion: Optional[NoteQCAttributeSuggestion] = None
+
+
+class NoteQCResult(BaseModel):
+    """Outcome of running one check against a draft."""
+
+    check_id: str
+    check_name: str
+    severity: NoteQCSeverity
+    passed: bool
+    issue: Optional[str] = None
+    evidence: Optional[str] = None
+    note_suggestion: Optional[str] = None
+    attribute_suggestion: Optional[NoteQCAttributeSuggestion] = None
+
+
+class RunQCChecksRequest(BaseModel):
+    """Body for run-qc-checks; playlist/version are path params."""
+
+    user_email: str
+
+
+class RunQCChecksResponse(BaseModel):
+    """QC results for one draft note."""
+
+    results: list[NoteQCResult]
+
+
+DEFAULT_ACTION_ITEM_CHECK = NoteQCCheckCreate(
+    name="Action Item Check",
+    prompt=(
+        "Review the transcript and note below. "
+        "If the transcript mentions an action item, task, or decision that is NOT "
+        "reflected in the note, report it. Otherwise respond with passed=true."
+    ),
+    severity="warning",
+    enabled=True,
+)