feat(documents): enhance contract comparison with clause frequency analysis (#750)

Author: larryro

examples/workflows/contract-comparison/config.json

@@ -10,11 +10,45 @@
 
 import asyncio
 import logging
+import re
 
 from playwright.async_api import Browser, Page, async_playwright
 
 logger = logging.getLogger(__name__)
 
+_ATX_HEADING_RE = re.compile(r"^#{1,6}\s", re.MULTILINE)
+
+
+def _normalize_markdown_headings(text: str) -> str:
+    """Ensure blank lines before ATX headings for reliable parsing.
+
+    Skips content inside fenced code blocks (``` or ~~~).
+    """
+    lines = text.split("\n")
+    result: list[str] = []
+    in_fence = False
+    fence_marker = ""
+
+    for line in lines:
+        stripped = line.strip()
+
+        if not in_fence:
+            if stripped.startswith("```") or stripped.startswith("~~~"):
+                in_fence = True
+                fence_marker = stripped[:3]
+        elif stripped == fence_marker or (stripped.startswith(fence_marker) and stripped.rstrip("`~") == ""):
+            in_fence = False
+            result.append(line)
+            continue
+
+        if not in_fence and _ATX_HEADING_RE.match(line) and result and result[-1].strip():
+            result.append("")
+
+        result.append(line)
+
+    return "\n".join(result)
+
+
 # Default HTML template for rendering content
 # Uses Noto fonts for multi-language support:
 # - Noto Sans: Latin (English, German, French, Spanish), Cyrillic (Russian)
@@ -150,15 +184,15 @@ async def markdown_to_html(self, markdown: str) -> str:
         """Convert markdown to HTML using Python-Markdown."""
         import markdown as md
 
-        # Convert markdown to HTML with common extensions
+        normalized = _normalize_markdown_headings(markdown)
+
         html = md.markdown(
-            markdown,
+            normalized,
             extensions=[
                 "tables",
                 "fenced_code",
                 "codehilite",
                 "toc",
-                "nl2br",
             ],
         )
         return html

services/crawler/app/services/base_converter.py

@@ -0,0 +1,88 @@
+"""Tests for _normalize_markdown_headings in base_converter."""
+
+import pytest
+
+from app.services.base_converter import _normalize_markdown_headings
+
+
+class TestNormalizeMarkdownHeadings:
+    def test_heading_after_table_row(self):
+        md = "| a | b |\n|---|---|\n| c | d |\n### Heading"
+        result = _normalize_markdown_headings(md)
+        assert "| c | d |\n\n### Heading" in result
+
+    def test_heading_after_paragraph(self):
+        md = "Some text\n### Heading"
+        result = _normalize_markdown_headings(md)
+        assert result == "Some text\n\n### Heading"
+
+    def test_heading_already_has_blank_line(self):
+        md = "Some text\n\n### Heading"
+        result = _normalize_markdown_headings(md)
+        assert result == "Some text\n\n### Heading"
+
+    def test_heading_at_start_of_document(self):
+        md = "# Title\nSome text"
+        result = _normalize_markdown_headings(md)
+        assert result == "# Title\nSome text"
+
+    def test_consecutive_headings(self):
+        md = "## Section\n### Subsection"
+        result = _normalize_markdown_headings(md)
+        assert result == "## Section\n\n### Subsection"
+
+    def test_hash_inside_fenced_code_block(self):
+        md = "```python\n# comment\ndef foo():\n    pass\n```"
+        result = _normalize_markdown_headings(md)
+        assert result == md
+
+    def test_hash_inside_tilde_fence(self):
+        md = "~~~\n# not a heading\n~~~"
+        result = _normalize_markdown_headings(md)
+        assert result == md
+
+    def test_heading_after_code_block(self):
+        md = "```\ncode\n```\n## Heading"
+        result = _normalize_markdown_headings(md)
+        assert "```\n\n## Heading" in result
+
+    def test_hash_in_middle_of_line_unchanged(self):
+        md = "Use the # symbol for headings"
+        result = _normalize_markdown_headings(md)
+        assert result == md
+
+    def test_multiple_headings_with_tables(self):
+        md = "### Section A\n| h1 | h2 |\n|---|---|\n| c1 | c2 |\n### Section B\n| h3 | h4 |\n|---|---|\n| c3 | c4 |"
+        result = _normalize_markdown_headings(md)
+        assert "| c2 |\n\n### Section B" in result
+
+    def test_empty_input(self):
+        assert _normalize_markdown_headings("") == ""
+
+    def test_no_headings(self):
+        md = "Just some text\nAnother line"
+        result = _normalize_markdown_headings(md)
+        assert result == md
+
+
+class TestMarkdownToHtmlHeadings:
+    """Integration tests verifying headings produce correct HTML tags."""
+
+    @pytest.mark.asyncio
+    async def test_heading_after_table_produces_h3(self):
+        from app.services.base_converter import BaseConverterService
+
+        converter = BaseConverterService()
+        md = "| a | b |\n|---|---|\n| c | d |\n### Heading"
+        html = await converter.markdown_to_html(md)
+        assert "<h3" in html
+        assert "Heading</h3>" in html
+
+    @pytest.mark.asyncio
+    async def test_heading_not_swallowed_by_table(self):
+        from app.services.base_converter import BaseConverterService
+
+        converter = BaseConverterService()
+        md = "### Section A\n\n| h1 | h2 |\n|---|---|\n| c1 | c2 |\n### Section B\n\nSome text."
+        html = await converter.markdown_to_html(md)
+        assert html.count("</h3>") == 2

services/crawler/tests/test_markdown_normalize.py

@@ -6,7 +6,7 @@ import {
   CheckCircle,
   XCircle,
   Loader2,
-  RefreshCw,
+  PenLine,
   ChevronDown,
   ChevronRight,
   Copy,
@@ -159,7 +159,7 @@ function WorkflowUpdateApprovalCardComponent({
       {/* Header */}
       <HStack gap={2} align="start" justify="between" className="mb-2">
         <HStack gap={2}>
-          <RefreshCw className="text-primary size-4 shrink-0" />
+          <PenLine className="text-primary size-4 shrink-0" />
           <div>
             <Text as="div" variant="label">
               {metadata.workflowName}

services/platform/app/features/chat/components/workflow-update-approval-card.tsx

@@ -97,11 +97,14 @@ USE THIS TOOL TO:
 
 DO NOT USE THIS TOOL FOR:
 • Semantic/content search — use rag_search instead
-• Reading indexed document content — use document_retrieve with the document ID instead
-• Extracting data from uploaded files — use pdf, docx, txt, excel, image, or pptx tools instead
+• Reading indexed document content — use document_retrieve with the document ID (id) instead
+• Extracting data from uploaded files — use pdf, docx, txt, excel, image, or pptx tools with the file ID (fileId) instead
 
 RESPONSE FIELDS:
-• documents: Array of {id, title, extension, folderPath, teamId, createdAt (Unix ms UTC), sizeBytes}
+• documents: Array of {id, fileId, title, extension, folderPath, teamId, createdAt (Unix ms UTC), sizeBytes}
+  - id: The document ID (Convex record ID). Use with document_retrieve to read indexed content.
+  - fileId: The storage file ID. Use with file extraction tools (pdf, docx, txt, excel, image, pptx) and any tool that operates on stored files.
+  - IMPORTANT: id and fileId are DIFFERENT identifiers for different purposes. File extraction tools (pdf, docx, excel, image, etc.) require fileId, NOT id. document_retrieve requires id, NOT fileId.
 • totalCount: Total matching documents (number), or null if the scan limit was reached and the true count is unknown — this does NOT mean zero results.
 • hasMore: Whether more results are available
 • cursor: Pass to next call to get the next page

services/platform/convex/agent_tools/documents/document_list_tool.ts

@@ -19,7 +19,7 @@ export const documentRetrieveArgs = z
       .string()
       .min(1)
       .describe(
-        'The Convex document ID to retrieve content from. Get IDs from document_list.',
+        'The Convex document ID (the "id" field from document_list, NOT "fileId"). Use fileId with file extraction tools instead.',
       ),
     chunkStart: z
       .number()

services/platform/convex/agent_tools/documents/document_retrieve_tool.ts

@@ -105,7 +105,7 @@ function mapChangeBlock(block: RagChangeBlock): ChangeBlock {
 }
 
 /**
- * Compare two documents via the RAG service's deterministic diff endpoint.
+ * Compare two documents by ID via the RAG service's deterministic diff endpoint.
  */
 export async function fetchDocumentComparison(
   ragServiceUrl: string,
@@ -155,28 +155,7 @@ export async function fetchDocumentComparison(
     }
 
     const result = await fetchJson<RagCompareResponse>(response);
-
-    return {
-      baseDocument: {
-        documentId: result.base_document.document_id,
-        title: result.base_document.title,
-      },
-      comparisonDocument: {
-        documentId: result.comparison_document.document_id,
-        title: result.comparison_document.title,
-      },
-      changeBlocks: result.change_blocks.map(mapChangeBlock),
-      stats: {
-        totalParagraphsBase: result.stats.total_paragraphs_base,
-        totalParagraphsComparison: result.stats.total_paragraphs_comparison,
-        unchanged: result.stats.unchanged,
-        modified: result.stats.modified,
-        added: result.stats.added,
-        deleted: result.stats.deleted,
-        highDivergence: result.stats.high_divergence,
-      },
-      truncated: result.truncated,
-    };
+    return mapRagResponse(result);
   } catch (error) {
     clearTimeout(timeoutId);
 
@@ -190,3 +169,102 @@ export async function fetchDocumentComparison(
     throw error;
   }
 }
+
+function mapRagResponse(result: RagCompareResponse): DocumentComparisonResult {
+  return {
+    baseDocument: {
+      documentId: result.base_document.document_id,
+      title: result.base_document.title,
+    },
+    comparisonDocument: {
+      documentId: result.comparison_document.document_id,
+      title: result.comparison_document.title,
+    },
+    changeBlocks: result.change_blocks.map(mapChangeBlock),
+    stats: {
+      totalParagraphsBase: result.stats.total_paragraphs_base,
+      totalParagraphsComparison: result.stats.total_paragraphs_comparison,
+      unchanged: result.stats.unchanged,
+      modified: result.stats.modified,
+      added: result.stats.added,
+      deleted: result.stats.deleted,
+      highDivergence: result.stats.high_divergence,
+    },
+    truncated: result.truncated,
+  };
+}
+
+/**
+ * Compare two files by URL via the RAG service — no pre-indexing required.
+ *
+ * Downloads both files and uploads them to the RAG service in a single
+ * function to avoid passing large Blobs through function parameters.
+ */
+export async function fetchDocumentComparisonByUrls(
+  ragServiceUrl: string,
+  baseFileUrl: string,
+  baseFileName: string,
+  comparisonFileUrl: string,
+  comparisonFileName: string,
+  maxChanges?: number,
+): Promise<DocumentComparisonResult> {
+  const [baseResponse, compResponse] = await Promise.all([
+    fetch(baseFileUrl),
+    fetch(comparisonFileUrl),
+  ]);
+  if (!baseResponse.ok) {
+    throw new Error(`Failed to download base file: ${baseResponse.status}`);
+  }
+  if (!compResponse.ok) {
+    throw new Error(
+      `Failed to download comparison file: ${compResponse.status}`,
+    );
+  }
+
+  const [baseBlob, compBlob] = await Promise.all([
+    baseResponse.blob(),
+    compResponse.blob(),
+  ]);
+
+  const url = `${ragServiceUrl}/api/v1/documents/compare-files`;
+
+  const formData = new FormData();
+  formData.append('base_file', baseBlob, baseFileName);
+  formData.append('comparison_file', compBlob, comparisonFileName);
+  if (maxChanges != null) {
+    formData.append('max_changes', String(maxChanges));
+  }
+
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
+
+  try {
+    const response = await fetch(url, {
+      method: 'POST',
+      body: formData,
+      signal: controller.signal,
+    });
+    clearTimeout(timeoutId);
+
+    if (!response.ok) {
+      const errorText = await response.text().catch(() => '');
+      throw new Error(
+        `RAG service error (${response.status}): ${errorText || 'Unknown error'}`,
+      );
+    }
+
+    const result = await fetchJson<RagCompareResponse>(response);
+    return mapRagResponse(result);
+  } catch (error) {
+    clearTimeout(timeoutId);
+
+    if (error instanceof Error && error.name === 'AbortError') {
+      throw new Error(
+        `RAG service timed out after ${FETCH_TIMEOUT_MS / 1000}s while comparing files.`,
+        { cause: error },
+      );
+    }
+
+    throw error;
+  }
+}

services/platform/convex/agent_tools/documents/helpers/fetch_document_comparison.ts

@@ -100,7 +100,7 @@ export function createBoundIntegrationTool(
             requiresApproval: true,
             approvalId: result.approvalId,
             approvalCreated: true,
-            approvalMessage: `APPROVAL CREATED SUCCESSFULLY: An approval card (ID: ${result.approvalId}) has been created for "${result.operationTitle || args.operation}" on ${integrationName}. The user must approve or reject this operation in the chat UI before it will be executed.`,
+            approvalMessage: `APPROVAL CREATED SUCCESSFULLY: An approval card (ID: ${result.approvalId}) has been created below your message for "${result.operationTitle || args.operation}" on ${integrationName}. The user must approve or reject this operation before it will be executed.`,
             data: {
               approvalId: result.approvalId,
               operationName: result.operationName,
@@ -147,7 +147,7 @@ function buildDescription(
   lines.push(
     '',
     `Usage: { operation: "<operation_name>", params: { ... } }`,
-    'Write operations create approval cards for user review.',
+    'Write operations create approval cards below your message for user review.',
   );
 
   return lines.join('\n');

services/platform/convex/agent_tools/integrations/create_bound_integration_tool.ts

@@ -49,7 +49,7 @@ Steps:
 1. First call integration_introspect(operation="xxx") to get required parameters
 2. Then call this tool with ALL required params filled in
 
-Write operations create approval cards. Use integration_batch for multiple parallel reads.`,
+Write operations create approval cards below your message. Use integration_batch for multiple parallel reads.`,
 
     args: integrationArgs,
 
@@ -122,7 +122,7 @@ Write operations create approval cards. Use integration_batch for multiple paral
             requiresApproval: true,
             approvalId: approvalResult.approvalId,
             approvalCreated: true,
-            approvalMessage: `APPROVAL CREATED SUCCESSFULLY: An approval card (ID: ${approvalResult.approvalId}) has been created for "${approvalResult.operationTitle || args.operation}" on ${args.integrationName}. The user must approve or reject this operation in the chat UI before it will be executed.`,
+            approvalMessage: `APPROVAL CREATED SUCCESSFULLY: An approval card (ID: ${approvalResult.approvalId}) has been created below your message for "${approvalResult.operationTitle || args.operation}" on ${args.integrationName}. The user must approve or reject this operation before it will be executed.`,
             data: {
               approvalId: approvalResult.approvalId,
               operationName: approvalResult.operationName,

services/platform/convex/agent_tools/integrations/integration_tool.ts

@@ -152,8 +152,8 @@ export function createBoundWorkflowTool(
           requiresApproval: true,
           approvalId,
           approvalCreated: true,
-          approvalMessage: `APPROVAL CREATED SUCCESSFULLY: An approval card (ID: ${approvalId}) has been created to run workflow "${resolvedWf.name}". The user must approve this in the chat UI before execution begins.`,
-          message: `Workflow "${resolvedWf.name}" is ready to run. An approval card has been created. The workflow will start once the user approves it.`,
+          approvalMessage: `APPROVAL CREATED SUCCESSFULLY: An approval card (ID: ${approvalId}) has been created below your message to run workflow "${resolvedWf.name}". The user must approve this before execution begins.`,
+          message: `Workflow "${resolvedWf.name}" is ready to run. An approval card has been created below. The workflow will start once the user approves it.`,
         };
       } catch (error) {
         return {