Refactor analytics and add LLM translation utilities

Moved analytics modules from 'optional' to 'utilities' for better organization and updated all imports accordingly. Added shared translation utilities in 'utilities/translation/translationProcessor.ts' to support LLM-based translation with symbol preservation across AAC formats. Updated processors (gridset, obf, touchchat) to use these utilities, and added comprehensive documentation in 'utilities/translation/README.md'.

Author: willwade

scripts/analysis/analyze_obl_data.ts

@@ -1,6 +1,6 @@
 import * as fs from 'fs';
 import * as path from 'path';
-import { OblUtil } from '../../src/optional/analytics/index';
+import { OblUtil } from '../../src/utilities/analytics/index';
 
 /**
  * Script to bulk-analyze OBLA clinical data and extract utterances to a CSV.

scripts/translation/gemini-translate-gridset.js

@@ -103,6 +103,17 @@ async function main() {
   // Step 1: Extract symbol information
   console.log('STEP 1: Extracting symbol information from gridset...');
   const symbolInfo = processor.extractSymbolsForLLM(inputPath);
+
+  // NOTE: This script uses batch processing - sends ALL buttons in a single API call.
+  // For large vocabularies (1000+ buttons), consider chunking by page to avoid:
+  // - Hitting LLM context window limits
+  // - API rate limits
+  // - Better error recovery and progress tracking
+  //
+  // Example chunking approach:
+  // - Group buttons by page
+  // - Process each page separately
+  // - Combine results before applying to gridset
 
   console.log(`  Found ${symbolInfo.length} buttons with symbols`);
   console.log(`  Processing first ${Math.min(maxItems, symbolInfo.length)} items`);

src/core/baseProcessor.ts

@@ -1,3 +1,45 @@
+/**
+ * Base Processor for AAC File Formats
+ *
+ * This module provides base functionality for processing AAC (Augmentative and Alternative
+ * Communication) files across various formats (gridset, OBF, Snap, TouchChat, etc.).
+ *
+ * ## LLM-Based Translation with Symbol Preservation
+ *
+ * All processor formats support LLM-based translation that preserves symbol-to-word
+ * associations across languages. This is critical for AAC systems where visual symbols
+ * are attached to specific words.
+ *
+ * ### Usage Example:
+ *
+ * ```typescript
+ * import { extractAllButtonsForTranslation, createTranslationPrompt } from '../optional/translation/translationProcessor';
+ *
+ * // 1. Extract buttons from your format
+ * const buttons = extractAllButtonsForTranslation(myFormatButtons, (button) => ({
+ *   pageId: button.pageId,
+ *   pageName: button.pageName
+ * }));
+ *
+ * // 2. Create prompt for LLM
+ * const prompt = createTranslationPrompt(buttons, 'Spanish');
+ *
+ * // 3. Send to LLM (Gemini, GPT, etc.) and get response
+ * const llmResponse = await callLLMAPI(prompt);
+ *
+ * // 4. Apply translations to your format
+ * processor.processLLMTranslations(filePath, llmResponse, outputPath);
+ * ```
+ *
+ * ### Format-Specific Implementation:
+ *
+ * Each processor should implement:
+ * - `extractSymbolsForLLM()` - Uses extractAllButtonsForTranslation() utility
+ * - `processLLMTranslations()` - Applies translations using format-specific logic
+ *
+ * See `src/utilities/translation/translationProcessor.ts` for shared utilities.
+ */
+
 import { AACTree, AACButton, AACSemanticCategory } from './treeStructure';
 import { StringCasing, detectCasing, isNumericOrEmpty } from './stringCasing';
 import { ValidationResult } from '../validation/validationTypes';

src/index.ts

@@ -4,12 +4,12 @@ export * from './core/baseProcessor';
 export * from './core/stringCasing';
 export * from './processors';
 export * from './validation';
-export * as Analytics from './optional/analytics';
+export * as Analytics from './utilities/analytics';
 export {
   collectUnifiedHistory,
   listGrid3Users as listHistoryGrid3Users,
   listSnapUsers as listHistorySnapUsers,
-} from './optional/analytics/history';
+} from './utilities/analytics/history';
 
 import { BaseProcessor } from './core/baseProcessor';
 import { DotProcessor } from './processors/dotProcessor';

src/processors/gridsetProcessor.ts

@@ -18,6 +18,12 @@ import AdmZip from 'adm-zip';
 import fs from 'fs';
 import { XMLParser, XMLBuilder } from 'fast-xml-parser';
 import { resolveGrid3CellImage } from './gridset/resolver';
+import {
+  extractAllButtonsForTranslation,
+  validateTranslationResults,
+  type ButtonForTranslation,
+  type LLMLTranslationResult,
+} from '../utilities/translation/translationProcessor';
 import { getZipEntriesWithPassword, resolveGridsetPassword } from './gridset/password';
 import crypto from 'crypto';
 import zlib from 'zlib';
@@ -28,7 +34,7 @@ import { detectPluginCellType, Grid3CellType } from './gridset/pluginTypes';
 import { detectCommand } from './gridset/commands';
 import { type SymbolReference, parseSymbolReference } from './gridset/symbols';
 import { isSymbolLibraryReference } from './gridset/resolver';
-import { generateCloneId } from '../optional/analytics/utils/idGenerator';
+import { generateCloneId } from '../utilities/analytics/utils/idGenerator';
 import { translateWithSymbols, extractSymbolsFromButton } from './gridset/symbolAlignment';
 
 class GridsetProcessor extends BaseProcessor {
@@ -1245,122 +1251,56 @@ class GridsetProcessor extends BaseProcessor {
    * Extract symbol information from a gridset for LLM-based translation.
    * Returns a structured format showing which buttons have symbols and their context.
    *
+   * This method uses shared translation utilities that work across all AAC formats.
+   *
    * @param filePathOrBuffer - Path to gridset file or buffer
    * @returns Array of symbol information for LLM processing
    */
-  extractSymbolsForLLM(filePathOrBuffer: string | Buffer): Array<{
-    buttonId: string;
-    pageId: string;
-    pageName: string;
-    label: string;
-    message: string;
-    textToTranslate: string;
-    symbols: Array<{
-      text: string;
-      image?: string;
-      symbolLibrary?: string;
-      symbolPath?: string;
-    }>;
-  }> {
+  extractSymbolsForLLM(filePathOrBuffer: string | Buffer): ButtonForTranslation[] {
     const tree = this.loadIntoTree(filePathOrBuffer);
-    const symbolInfo: Array<{
-      buttonId: string;
-      pageId: string;
-      pageName: string;
-      label: string;
-      message: string;
-      textToTranslate: string;
-      symbols: Array<{
-        text: string;
-        image?: string;
-        symbolLibrary?: string;
-        symbolPath?: string;
-      }>;
-    }> = [];
 
+    // Collect all buttons from all pages
+    const allButtons: any[] = [];
     Object.values(tree.pages).forEach((page) => {
       page.buttons.forEach((button) => {
-        // Extract symbols from various sources
-        const symbols: Array<{
-          text: string;
-          image?: string;
-          symbolLibrary?: string;
-          symbolPath?: string;
-        }> = [];
-
-        // Check richText.symbols
-        if (button.semanticAction?.richText?.symbols) {
-          symbols.push(...button.semanticAction.richText.symbols);
-        }
-
-        // Check symbolLibrary + symbolPath
-        if (button.symbolLibrary && button.symbolPath) {
-          const text = button.label || button.message || '';
-          if (text) {
-            symbols.push({
-              text,
-              symbolLibrary: button.symbolLibrary,
-              symbolPath: button.symbolPath,
-            });
-          }
-        }
-
-        // Check image field for symbol reference
-        if (button.image && button.image.startsWith('[')) {
-          const text = button.label || button.message || '';
-          if (text) {
-            symbols.push({
-              text,
-              image: button.image,
-            });
-          }
-        }
-
-        // Only include buttons that have symbols
-        if (symbols.length > 0) {
-          const textToTranslate = button.message || button.label || '';
-          if (textToTranslate) {
-            symbolInfo.push({
-              buttonId: button.id,
-              pageId: page.id,
-              pageName: page.name || page.id,
-              label: button.label || '',
-              message: button.message || '',
-              textToTranslate,
-              symbols,
-            });
-          }
-        }
+        // Add page context to each button
+        (button as any).pageId = page.id;
+        (button as any).pageName = page.name || page.id;
+        allButtons.push(button);
       });
     });
 
-    return symbolInfo;
+    // Use shared utility to extract buttons with translation context
+    return extractAllButtonsForTranslation(allButtons, (button) => ({
+      pageId: button.pageId,
+      pageName: button.pageName,
+    }));
   }
 
   /**
    * Apply LLM translations with symbol information.
    * The LLM should provide translations with symbol attachments in the correct positions.
    *
+   * This method uses shared translation utilities that work across all AAC formats.
+   *
    * @param filePathOrBuffer - Path to gridset file or buffer
    * @param llmTranslations - Array of LLM translations with symbol info
    * @param outputPath - Where to save the translated gridset
+   * @param options - Translation options (e.g., allowPartial for testing)
    * @returns Buffer of the translated gridset
    */
   processLLMTranslations(
     filePathOrBuffer: string | Buffer,
-    llmTranslations: Array<{
-      buttonId: string;
-      translatedLabel?: string;
-      translatedMessage?: string;
-      symbols?: Array<{
-        text: string;
-        image?: string;
-      }>;
-    }>,
-    outputPath: string
+    llmTranslations: LLMLTranslationResult[],
+    outputPath: string,
+    options?: { allowPartial?: boolean }
   ): Buffer {
     const tree = this.loadIntoTree(filePathOrBuffer);
 
+    // Validate translations using shared utility
+    const buttonIds = Object.values(tree.pages).flatMap((page) => page.buttons.map((b) => b.id));
+    validateTranslationResults(llmTranslations, buttonIds, options);
+
     // Create a map for quick lookup
     const translationMap = new Map(llmTranslations.map((t) => [t.buttonId, t]));
 

src/processors/obfProcessor.ts

@@ -13,11 +13,17 @@ import {
   AACSemanticCategory,
   AACSemanticIntent,
 } from '../core/treeStructure';
-import { generateCloneId } from '../optional/analytics/utils/idGenerator';
+import { generateCloneId } from '../utilities/analytics/utils/idGenerator';
 import AdmZip from 'adm-zip';
 import fs from 'fs';
 import { ObfValidator } from '../validation/obfValidator';
 import { ValidationResult } from '../validation/validationTypes';
+import {
+  extractAllButtonsForTranslation,
+  validateTranslationResults,
+  type ButtonForTranslation,
+  type LLMLTranslationResult,
+} from '../utilities/translation/translationProcessor';
 
 const OBF_FORMAT_VERSION = 'open-board-0.1';
 
@@ -496,6 +502,102 @@ class ObfProcessor extends BaseProcessor {
   async validate(filePath: string): Promise<ValidationResult> {
     return ObfValidator.validateFile(filePath);
   }
+
+  /**
+   * Extract symbol information from an OBF/OBZ file for LLM-based translation.
+   * Returns a structured format showing which buttons have symbols and their context.
+   *
+   * This method uses shared translation utilities that work across all AAC formats.
+   *
+   * @param filePathOrBuffer - Path to OBF/OBZ file or buffer
+   * @returns Array of symbol information for LLM processing
+   */
+  extractSymbolsForLLM(filePathOrBuffer: string | Buffer): ButtonForTranslation[] {
+    const tree = this.loadIntoTree(filePathOrBuffer);
+
+    // Collect all buttons from all pages
+    const allButtons: any[] = [];
+    Object.values(tree.pages).forEach((page) => {
+      page.buttons.forEach((button) => {
+        // Add page context to each button
+        (button as any).pageId = page.id;
+        (button as any).pageName = page.name || page.id;
+        allButtons.push(button);
+      });
+    });
+
+    // Use shared utility to extract buttons with translation context
+    return extractAllButtonsForTranslation(allButtons, (button) => ({
+      pageId: button.pageId,
+      pageName: button.pageName,
+    }));
+  }
+
+  /**
+   * Apply LLM translations with symbol information.
+   * The LLM should provide translations with symbol attachments in the correct positions.
+   *
+   * This method uses shared translation utilities that work across all AAC formats.
+   *
+   * @param filePathOrBuffer - Path to OBF/OBZ file or buffer
+   * @param llmTranslations - Array of LLM translations with symbol info
+   * @param outputPath - Where to save the translated OBF/OBZ file
+   * @param options - Translation options (e.g., allowPartial for testing)
+   * @returns Buffer of the translated OBF/OBZ file
+   */
+  processLLMTranslations(
+    filePathOrBuffer: string | Buffer,
+    llmTranslations: LLMLTranslationResult[],
+    outputPath: string,
+    options?: { allowPartial?: boolean }
+  ): Buffer {
+    const tree = this.loadIntoTree(filePathOrBuffer);
+
+    // Validate translations using shared utility
+    const buttonIds = Object.values(tree.pages).flatMap((page) => page.buttons.map((b) => b.id));
+    validateTranslationResults(llmTranslations, buttonIds, options);
+
+    // Create a map for quick lookup
+    const translationMap = new Map(llmTranslations.map((t) => [t.buttonId, t]));
+
+    // Apply translations
+    Object.values(tree.pages).forEach((page) => {
+      page.buttons.forEach((button) => {
+        const translation = translationMap.get(button.id);
+        if (!translation) return;
+
+        // Apply label translation
+        if (translation.translatedLabel) {
+          button.label = translation.translatedLabel;
+        }
+
+        // Apply message translation (vocalization in OBF)
+        if (translation.translatedMessage) {
+          button.message = translation.translatedMessage;
+
+          // Update semantic action if symbols provided
+          if (translation.symbols && translation.symbols.length > 0) {
+            if (!button.semanticAction) {
+              button.semanticAction = {
+                category: AACSemanticCategory.COMMUNICATION,
+                intent: AACSemanticIntent.SPEAK_TEXT,
+                text: translation.translatedMessage,
+              };
+            }
+
+            button.semanticAction.richText = {
+              text: translation.translatedMessage,
+              symbols: translation.symbols,
+            };
+          }
+        }
+      });
+    });
+
+    // Save and return
+    this.saveFromTree(tree, outputPath);
+    return fs.readFileSync(outputPath);
+  }
 }
 
 export { ObfProcessor };

src/processors/snapProcessor.ts

@@ -13,7 +13,7 @@ import {
   AACSemanticCategory,
   AACSemanticIntent,
 } from '../core/treeStructure';
-import { generateCloneId } from '../optional/analytics/utils/idGenerator';
+import { generateCloneId } from '../utilities/analytics/utils/idGenerator';
 import Database from 'better-sqlite3';
 import path from 'path';
 import fs from 'fs';