feat(ai): add chunk formatting capabilities

packages/super-editor/src/components/toolbar/AIWriter.vue

@@ -1,6 +1,6 @@
 <script setup>
 import { computed, ref, onMounted, onUnmounted, nextTick } from 'vue';
-import { writeStreaming, rewriteStreaming } from './ai-helpers';
+import { writeStreaming, rewriteStreaming, formatDocument } from './ai-helpers';
 import { TextSelection } from 'prosemirror-state';
 import edit from '@harbour-enterprises/common/icons/edit-regular.svg?raw';
 import paperPlane from '@harbour-enterprises/common/icons/paper-plane-regular.svg?raw';
@@ -140,6 +140,8 @@ const isError = ref('');
 const promptText = ref('');
 const textProcessingStarted = ref(false);
 const previousText = ref('');
+const isFormatting = ref(false);
+const pendingFormatting = ref(false);
 
 // Computed property to check if editor is in suggesting mode
 const isInSuggestingMode = computed(() => {
@@ -159,7 +161,7 @@ const getDocumentXml = () => {
 };
 
 // Handler for processing text chunks from the stream
-const handleTextChunk = (text) => {
+const handleTextChunk = async (text) => {
   try {
     // Remove the loader node when we start receiving text
     props.editor.commands.removeAiNode('aiLoaderNode');
@@ -193,7 +195,6 @@ const handleTextChunk = (text) => {
 
       // Mark as processed
       textProcessingStarted.value = true;
-
     }
 
     // If the text is null, undefined or empty, don't process it
@@ -204,7 +205,7 @@ const handleTextChunk = (text) => {
     // Convert to string in case it's not already a string
     const textStr = String(text);
 
-    // Wrap the content in a span with our animation class and unique ID
+    // Wrap the raw content in a span with our animation class and unique ID
     const wrappedContent = {
       type: 'text',
       marks: [{
@@ -217,9 +218,15 @@ const handleTextChunk = (text) => {
       text: textStr
     };
 
-    // Insert the new content
+    // Insert the raw content with animation mark
     props.editor.commands.insertContent(wrappedContent);
 
+    // Prevent race conditions - do not call formatDocument if we are already formatting
+    pendingFormatting.value = true;
+    if (!isFormatting.value) {
+      await runSafeFormat();
+    }
+
     // Hide the AI Writer after content is received
     props.handleClose();
 
@@ -228,13 +235,60 @@ const handleTextChunk = (text) => {
   }
 };
 
-// Handler for when the stream is done
-const handleDone = () => {
+/**
+ * Run formatting when it's safe (no other formatting is in progress)
+ * Recursively call itself if we are still needing to process raw requests from
+ * pendingFormatting.value
+ */
+const runSafeFormat = async () => {
+  if (isFormatting.value) return;
+
+  try {
+    isFormatting.value = true;
+    pendingFormatting.value = false;
+
+    await nextTick();
+
+    formatDocument(props.editor);
+
+    // Check if more formatting requests arrived while we were formatting
+    if (pendingFormatting.value) {
+      pendingFormatting.value = false;
+      await runSafeFormat();
+    }
+  } finally {
+    isFormatting.value = false;
+  }
+};
+
+/**
+ * Handler for when the stream is done
+ * 
+ * We need to make sure we're not currently running any formatting before our final call
+ * 
+ * We can do this by using a short recursive polling system to wait.
+ */
+const handleDone = async () => {
+  if (pendingFormatting.value || isFormatting.value) {
+    pendingFormatting.value = true;
+    await new Promise(resolve => {
+      const checkFormatting = () => {
+        if (!isFormatting.value && !pendingFormatting.value) {
+          resolve();
+        } else {
+          setTimeout(checkFormatting, 100);
+        }
+      };
+      checkFormatting();
+    });
+  }
+
+  await runSafeFormat();
+
   // If we are done we can remove the animation mark
   // We need to wait for the animation to finish before removing the mark
   setTimeout(() => {
     props.editor.commands.removeAiMark('aiAnimationMark');
-    // Remove the highlight when we're done
     emitAiHighlight('remove');
   }, 1000);
 };
@@ -245,7 +299,7 @@ const handleSubmit = async () => {
   isError.value = '';
   textProcessingStarted.value = false;
   previousText.value = '';
-  isLoading.value = true;  // Set loading to true before starting the operation
+  isLoading.value = true;
 
   try {
     // Close the AI Writer immediately and transition to loading states
@@ -329,7 +383,6 @@ const handleInput = (event) => {
   <div class="ai-writer prosemirror-isolated" ref="aiWriterRef" @mousedown.stop>
     <div class="ai-user-input-field">
       <span class="ai-textarea-icon" v-html="edit"></span>
-      <!-- Replace contenteditable div with textarea -->
       <textarea
         ref="editableRef"
         class="ai-textarea"

packages/super-editor/src/components/toolbar/ai-helpers.js

@@ -31,7 +31,7 @@ const SYSTEM_PROMPT =
  */
 async function baseInsightsFetch(payload, options = {}) {
   const apiKey = options.apiKey;
-  
+
   // Use the provided endpoint from config, or fall back to the default
   const apiEndpoint = options.endpoint || DEFAULT_API_ENDPOINT;
 
@@ -295,3 +295,163 @@ export async function rewrite(text, prompt = '', options = {}) {
   const response = await baseInsightsFetch(payload, options.config || {});
   return returnNonStreamingJson(response);
 }
+
+/**
+ * Format registry to manage text formatting rules
+ * Each rule has a name, pattern, and transform function
+ * Extend this for more rules (e.g. italic, underline, etc.)
+ */
+const formatRegistry = {
+  rules: [
+    {
+      name: 'bold',
+      pattern: /\*\*(.*?)\*\*/g,
+      transform: (_match, content, _editor) => ({
+        type: 'text',
+        marks: [{ type: 'bold' }],
+        text: content,
+      }),
+    },
+  ],
+};
+
+/**
+ * Converts markdown-style formatting in the document text to ProseMirror's native formatting.
+ * Uses a node-aware approach that safely handles formatting across node boundaries.
+ * 
+ * This function processes the entire document content and applies formatting rules defined in formatRegistry.
+ * It handles cases where formatting markers (like **bold**) span across multiple text nodes by tracking
+ * node positions and boundaries. The function works from the end of the document to the start to avoid
+ * position shifts when making replacements.
+ * 
+ * @param {Object} editor - The ProseMirror editor instance containing the document state and view
+ */
+export function formatDocument(editor) {
+  try {
+    let doc = editor.state.doc;
+    const docText = doc.textContent || '';
+    if (!docText) return;
+
+    // Process each formatting rule
+    // Registry is defined above
+    formatRegistry.rules.forEach((rule) => {
+      rule.pattern.lastIndex = 0;
+      const matches = [];
+      let match;
+
+      while ((match = rule.pattern.exec(docText)) !== null) {
+        matches.push({
+          rule,
+          startPos: match.index,
+          endPos: match.index + match[0].length,
+          originalText: match[0],
+          contentText: match[1],
+        });
+      } 
+
+      // We may have 0, 1, or more matches for a single rule in a chunk of text
+      // Need to handle each match individually but preserve positions of the matches
+      // Process matches from end to start to avoid position shifts
+      matches.sort((a, b) => b.startPos - a.startPos);
+
+      for (const match of matches) {
+        const { startPos, endPos, originalText, contentText } = match;
+
+        try {
+          // Create transaction
+          let tr = editor.state.tr;
+          const replacement = rule.transform(originalText, contentText, editor);
+
+          // Gather nodes needed to replace the match
+          const nodesInRange = [];
+          doc.nodesBetween(startPos, Math.min(endPos, doc.content.size), (node, pos) => {
+            if (node.isText) {
+              nodesInRange.push({ node, pos });
+            }
+            return true;
+          });
+
+          if (nodesInRange.length > 0) {
+            // Try first to find match in a single node
+            // This is best case scenario and would skip the need to reconstruct across nodes
+            let foundExactMatch = false;
+            let actualStartPos = -1;
+            let actualEndPos = -1;
+
+            for (let i = 0; i < nodesInRange.length; i++) {
+              const nodeInfo = nodesInRange[i];
+              const nodeText = nodeInfo.node.text || '';
+              const nodePos = nodeInfo.pos;
+
+              // Check if this node contains the entire pattern
+              if (nodeText.includes(originalText)) {
+                const nodeMatchIndex = nodeText.indexOf(originalText);
+                actualStartPos = nodePos + nodeMatchIndex;
+                actualEndPos = actualStartPos + originalText.length;
+
+                foundExactMatch = true;
+                break;
+              }
+            }
+
+            // If we couldn't find the pattern in a single node, try reconstructing across nodes
+            if (!foundExactMatch) {
+              // Build text spanning multiple nodes
+              let combinedText = '';
+              let offsets = [];
+              // Start of first node 
+              // This acts as an anchor point for the relative position of characters in other nodes
+              let basePos = nodesInRange[0].pos;
+
+              // Build a mapping between combined text positions and actual document positions
+              for (const nodeInfo of nodesInRange) {
+                const nodeText = nodeInfo.node.text || '';
+                const relativePos = nodeInfo.pos - basePos;
+
+                // For each character in the node, record its position
+                for (let i = 0; i < nodeText.length; i++) {
+                  offsets.push(relativePos + i);
+                }
+
+                combinedText += nodeText;
+              }
+
+              const matchIndex = combinedText.indexOf(originalText);
+              if (matchIndex >= 0) {
+                // Use our offset map to find the actual position in the document
+                actualStartPos = basePos + offsets[matchIndex];
+                // The end position might be beyond the last recorded offset if it falls at a node boundary
+                const endIndex = matchIndex + originalText.length - 1;
+                actualEndPos = basePos + (offsets[endIndex] || 0) + 1;
+
+                foundExactMatch = true;
+              }
+            }
+
+            if (foundExactMatch) {
+              const marks = replacement.marks
+                ? replacement.marks.map((mark) => editor.schema.marks[mark.type].create(mark.attrs))
+                : [];
+
+              // PM transactions
+              tr = tr.delete(actualStartPos, actualEndPos);
+              tr = tr.insert(actualStartPos, editor.schema.text(replacement.text, marks));
+
+              if (tr.docChanged) {
+                editor.view.dispatch(tr);
+
+                // After making this change, we need to recalculate positions
+                // Get updated doc reference
+                doc = editor.state.doc;
+              }
+            }
+          }
+        } catch (error) {
+          console.error('Error processing match:', error);
+        }
+      }
+    });
+  } catch (error) {
+    console.error('Error formatting document:', error);
+  }
+}