improving validation for touchchat too

willwade · willwade · commit d9f54e284e15 · 2026-01-30T12:50:26.000Z
diff --git a/src/processors/touchchatProcessor.ts b/src/processors/touchchatProcessor.ts
@@ -653,20 +653,131 @@ class TouchChatProcessor extends BaseProcessor {
         'processTexts is only supported in Node.js environments for TouchChat files.'
       );
     }
-    // Load the tree, apply translations, and save to new file
+    /**
+     * TouchChat .ce files are ZIP archives containing a SQLite .c4v database.
+     * Rebuilding the database can drop tables/metadata/resources that we don't
+     * currently model in the tree, which can corrupt the file.
+     *
+     * For file paths, we preserve the original archive and update text in-place
+     * within the embedded SQLite database, ensuring assets and metadata remain intact.
+     */
+    if (typeof filePathOrBuffer === 'string') {
+      const fs = getFs();
+      const path = getPath();
+      const os = getOs();
+      const AdmZip = getNodeRequire()('adm-zip') as typeof import('adm-zip');
+
+      const inputPath = filePathOrBuffer;
+      const outputDir = path.dirname(outputPath);
+      if (!fs.existsSync(outputDir)) {
+        fs.mkdirSync(outputDir, { recursive: true });
+      }
+      if (fs.existsSync(outputPath)) {
+        fs.unlinkSync(outputPath);
+      }
+
+      const zip = new AdmZip(inputPath);
+      const entries = zip.getEntries();
+      const vocabEntry = entries.find((entry) => entry.entryName.endsWith('.c4v'));
+      if (!vocabEntry) {
+        throw new Error('No .c4v vocab DB found in TouchChat export');
+      }
+
+      const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'touchchat-translate-'));
+      const dbPath = path.join(tempDir, 'vocab.c4v');
+      try {
+        fs.writeFileSync(dbPath, vocabEntry.getData());
+
+        const Database = requireBetterSqlite3();
+        const db = new Database(dbPath, { readonly: false });
+        try {
+          const getColumns = (tableName: string): Set<string> => {
+            try {
+              const rows = db.prepare(`PRAGMA table_info(${tableName})`).all() as Array<{
+                name: string;
+              }>;
+              return new Set(rows.map((row) => row.name));
+            } catch {
+              return new Set();
+            }
+          };
+
+          const resourceColumns = getColumns('resources');
+          const pageColumns = getColumns('pages');
+          const buttonColumns = getColumns('buttons');
+
+          const updatePageResourceName = resourceColumns.has('name')
+            ? db.prepare(
+                'UPDATE resources SET name = ? WHERE name = ? AND id IN (SELECT resource_id FROM pages)'
+              )
+            : null;
+          const updatePageName = pageColumns.has('name')
+            ? db.prepare('UPDATE pages SET name = ? WHERE name = ?')
+            : null;
+          const updateButtonLabel = buttonColumns.has('label')
+            ? db.prepare('UPDATE buttons SET label = ? WHERE label = ?')
+            : null;
+          const updateButtonMessage = buttonColumns.has('message')
+            ? db.prepare('UPDATE buttons SET message = ? WHERE message = ?')
+            : null;
+
+          const entriesToUpdate = Array.from(translations.entries());
+          const applyUpdates = db.transaction(() => {
+            entriesToUpdate.forEach(([original, translated]) => {
+              if (!translated || translated === original) {
+                return;
+              }
+              if (updatePageResourceName) {
+                updatePageResourceName.run(translated, original);
+              }
+              if (updatePageName) {
+                updatePageName.run(translated, original);
+              }
+              if (updateButtonLabel) {
+                updateButtonLabel.run(translated, original);
+              }
+              if (updateButtonMessage) {
+                updateButtonMessage.run(translated, original);
+              }
+            });
+          });
+          applyUpdates();
+        } finally {
+          db.close();
+        }
+
+        const outputZip = new AdmZip();
+        entries.forEach((entry) => {
+          if (entry.entryName === vocabEntry.entryName) {
+            return;
+          }
+          const data = entry.isDirectory ? Buffer.alloc(0) : entry.getData();
+          outputZip.addFile(entry.entryName, data, entry.comment || '');
+        });
+        outputZip.addFile(vocabEntry.entryName, fs.readFileSync(dbPath));
+        outputZip.writeZip(outputPath);
+      } finally {
+        try {
+          fs.rmSync(tempDir, { recursive: true, force: true });
+        } catch {
+          // Best-effort cleanup
+        }
+      }
+
+      return fs.readFileSync(outputPath);
+    }
+
+    // Fallback for buffer inputs: rebuild from tree (may drop TouchChat metadata)
     const tree = await this.loadIntoTree(filePathOrBuffer);
 
-    // Apply translations to all text content
     Object.values(tree.pages).forEach((page) => {
-      // Translate page names
       if (page.name && translations.has(page.name)) {
         const translatedName = translations.get(page.name);
         if (translatedName !== undefined) {
           page.name = translatedName;
         }
       }
 
-      // Translate button labels and messages
       page.buttons.forEach((button) => {
         if (button.label && translations.has(button.label)) {
           const translatedLabel = translations.get(button.label);
@@ -683,7 +794,6 @@ class TouchChatProcessor extends BaseProcessor {
       });
     });
 
-    // Save the translated tree and return its content
     await this.saveFromTree(tree, outputPath);
     const fs = getFs();
     return fs.readFileSync(outputPath);
diff --git a/src/validation/touchChatValidator.ts b/src/validation/touchChatValidator.ts
@@ -5,10 +5,13 @@ import * as xml2js from 'xml2js';
 import { BaseValidator } from './baseValidator';
 import { ValidationResult } from './validationTypes';
 import { decodeText, getBasename, getFs, readBinaryFromInput, toUint8Array } from '../utils/io';
+import { openZipFromInput } from '../utils/zip';
+import { openSqliteDatabase } from '../utils/sqlite';
 
 /**
  * Validator for TouchChat files (.ce)
- * TouchChat files are XML-based
+ * TouchChat files are ZIP archives that contain a .c4v SQLite database.
+ * Some legacy exports may be XML, so we support both formats.
  */
 export class TouchChatValidator extends BaseValidator {
   constructor() {
@@ -34,6 +37,17 @@ export class TouchChatValidator extends BaseValidator {
       return true;
     }
 
+    // Try to parse as ZIP and check for .c4v database
+    try {
+      const { zip } = await openZipFromInput(content);
+      const entries = zip.listFiles();
+      if (entries.some((entry) => entry.toLowerCase().endsWith('.c4v'))) {
+        return true;
+      }
+    } catch {
+      // Fall back to XML detection
+    }
+
     // Try to parse as XML and check for TouchChat structure
     try {
       const contentStr = typeof content === 'string' ? content : decodeText(toUint8Array(content));
@@ -62,45 +76,48 @@ export class TouchChatValidator extends BaseValidator {
       }
     });
 
-    let xmlObj: any = null;
-    await this.add_check('xml_parse', 'valid XML', async () => {
-      try {
-        const parser = new xml2js.Parser();
-        const contentStr = decodeText(content);
-        xmlObj = await parser.parseStringPromise(contentStr);
-      } catch (e: any) {
-        this.err(`Failed to parse XML: ${e.message}`, true);
+    const zipped = await this.tryValidateZipSqlite(content);
+    if (!zipped) {
+      let xmlObj: any = null;
+      await this.add_check('xml_parse', 'valid XML', async () => {
+        try {
+          const parser = new xml2js.Parser();
+          const contentStr = decodeText(content);
+          xmlObj = await parser.parseStringPromise(contentStr);
+        } catch (e: any) {
+          this.err(`Failed to parse XML: ${e.message}`, true);
+        }
+      });
+
+      if (!xmlObj) {
+        return this.buildResult(filename, filesize, 'touchchat');
       }
-    });
 
-    if (!xmlObj) {
-      return this.buildResult(filename, filesize, 'touchchat');
-    }
+      await this.add_check('xml_structure', 'TouchChat root element', async () => {
+        // TouchChat can have different root elements
+        const hasValidRoot =
+          xmlObj.PageSet ||
+          xmlObj.Pageset ||
+          xmlObj.page ||
+          xmlObj.Page ||
+          xmlObj.pages ||
+          xmlObj.Pages;
 
-    await this.add_check('xml_structure', 'TouchChat root element', async () => {
-      // TouchChat can have different root elements
-      const hasValidRoot =
+        if (!hasValidRoot) {
+          this.err('file does not contain a recognized TouchChat structure');
+        }
+      });
+
+      const root =
         xmlObj.PageSet ||
         xmlObj.Pageset ||
         xmlObj.page ||
         xmlObj.Page ||
         xmlObj.pages ||
         xmlObj.Pages;
-
-      if (!hasValidRoot) {
-        this.err('file does not contain a recognized TouchChat structure');
+      if (root) {
+        await this.validateTouchChatStructure(root);
       }
-    });
-
-    const root =
-      xmlObj.PageSet ||
-      xmlObj.Pageset ||
-      xmlObj.page ||
-      xmlObj.Page ||
-      xmlObj.pages ||
-      xmlObj.Pages;
-    if (root) {
-      await this.validateTouchChatStructure(root);
     }
 
     return this.buildResult(filename, filesize, 'touchchat');
@@ -243,4 +260,113 @@ export class TouchChatValidator extends BaseValidator {
       }
     );
   }
+
+  private isSQLiteBuffer(content: Buffer | Uint8Array): boolean {
+    const header = 'SQLite format 3\u0000';
+    const bytes = content instanceof Uint8Array ? content : new Uint8Array(content);
+    if (bytes.length < header.length) {
+      return false;
+    }
+    for (let i = 0; i < header.length; i++) {
+      if (bytes[i] !== header.charCodeAt(i)) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  private async tryValidateZipSqlite(content: Buffer | Uint8Array): Promise<boolean> {
+    let usedZip = false;
+    await this.add_check('zip', 'TouchChat ZIP package', async () => {
+      try {
+        const { zip } = await openZipFromInput(content);
+        const entries = zip.listFiles();
+        const vocabEntry = entries.find((name) => name.toLowerCase().endsWith('.c4v'));
+        if (!vocabEntry) {
+          this.err('TouchChat package missing .c4v database', true);
+          return;
+        }
+        const dbBuffer = await zip.readFile(vocabEntry);
+        if (!this.isSQLiteBuffer(dbBuffer)) {
+          this.err('TouchChat .c4v is not a valid SQLite database', true);
+          return;
+        }
+        usedZip = true;
+        await this.validateSqliteStructure(dbBuffer);
+      } catch (e: any) {
+        this.err(`file is not a valid TouchChat ZIP package: ${e.message}`, true);
+      }
+    });
+    return usedZip;
+  }
+
+  private async validateSqliteStructure(content: Buffer | Uint8Array): Promise<void> {
+    await this.add_check('sqlite', 'valid TouchChat SQLite database', async () => {
+      let cleanup: (() => void) | undefined;
+      try {
+        const result = await openSqliteDatabase(content, { readonly: true });
+        const db = result.db;
+        cleanup = result.cleanup;
+
+        const tableRows = db
+          .prepare("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name")
+          .all() as Array<{ name: string }>;
+        const tables = new Set(tableRows.map((row) => row.name));
+
+        const requiredTables = [
+          'resources',
+          'pages',
+          'buttons',
+          'button_boxes',
+          'button_box_cells',
+          'button_box_instances',
+        ];
+        const missingTables = requiredTables.filter((t) => !tables.has(t));
+        if (missingTables.length > 0) {
+          this.err(`Missing required TouchChat tables: ${missingTables.join(', ')}`);
+        }
+
+        const resourcesCols = new Set(
+          db
+            .prepare('PRAGMA table_info(resources)')
+            .all()
+            .map((row: any) => row.name)
+        );
+        if (!resourcesCols.has('id') || !resourcesCols.has('name')) {
+          this.err('resources table missing id/name columns');
+        }
+
+        const pagesCols = new Set(
+          db
+            .prepare('PRAGMA table_info(pages)')
+            .all()
+            .map((row: any) => row.name)
+        );
+        if (!pagesCols.has('id') || !pagesCols.has('resource_id')) {
+          this.err('pages table missing id/resource_id columns');
+        }
+
+        const buttonsCols = new Set(
+          db
+            .prepare('PRAGMA table_info(buttons)')
+            .all()
+            .map((row: any) => row.name)
+        );
+        if (!buttonsCols.has('id') || !buttonsCols.has('resource_id')) {
+          this.err('buttons table missing id/resource_id columns');
+        }
+
+        const pageCount = db.prepare('SELECT COUNT(*) as c FROM pages').get() as { c: number };
+        if (!pageCount || pageCount.c === 0) {
+          this.warn('TouchChat database has no pages');
+        }
+      } catch (e: any) {
+        this.err(`TouchChat database validation failed: ${e.message}`, true);
+      } finally {
+        if (cleanup) {
+          cleanup();
+        }
+      }
+    });
+  }
 }
