Merge pull request Expensify#64389 from Expensify/Rory-DecodeUnicode

[No QA] Decode unicode

Author: luacmartins

.github/actions/javascript/proposalPoliceComment/index.js

@@ -450,6 +450,7 @@
         "positionMillis",
         "Postharvest",
         "Postproduction",
+        "Précédent",
         "precheck",
         "Precheck",
         "prescribers",

cspell.json

@@ -329,7 +329,7 @@
     "memfs": "^4.6.0",
     "nitro-codegen": "0.25.2",
     "onchange": "^7.1.0",
-    "openai": "^4.47.2",
+    "openai": "5.5.1",
     "patch-package": "^8.1.0-canary.1",
     "peggy": "^4.0.3",
     "portfinder": "^1.0.28",

package-lock.json

@@ -8,9 +8,10 @@ import fs from 'fs';
 import path from 'path';
 import type {TemplateExpression} from 'typescript';
 import ts from 'typescript';
+import decodeUnicode from '@libs/StringUtils/decodeUnicode';
 import dedent from '@libs/StringUtils/dedent';
 import hashStr from '@libs/StringUtils/hash';
-import {isTranslationTargetLocale, TRANSLATION_TARGET_LOCALES} from '@src/CONST/LOCALES';
+import {isTranslationTargetLocale, LOCALES, TRANSLATION_TARGET_LOCALES} from '@src/CONST/LOCALES';
 import type {TranslationTargetLocale} from '@src/CONST/LOCALES';
 import CLI from './utils/CLI';
 import Prettier from './utils/Prettier';
@@ -112,7 +113,7 @@ class TranslationGenerator {
 
             // Generate translated TypeScript code
             const printer = ts.createPrinter();
-            const translatedCode = printer.printFile(transformedSourceFile);
+            const translatedCode = decodeUnicode(printer.printFile(transformedSourceFile));
 
             // Write to file
             const outputPath = path.join(this.languagesDir, `${targetLanguage}.ts`);
@@ -182,9 +183,15 @@ class TranslationGenerator {
             return false;
         }
 
-        // Only translate string literals if they contain alphabet characters
+        // Don't translate object keys
+        if (ts.isComputedPropertyName(node.parent)) {
+            return false;
+        }
+
+        // Only translate string literals if they contain at least one real letter
         if (ts.isStringLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node)) {
-            return /[a-zA-Z]/.test(node.text);
+            // \p{L} matches a-z, à-ö, Α-Ω, Ж, 文, …  – but NOT digits, emoji, punctuation, etc.
+            return /\p{L}/u.test(node.text);
         }
 
         // Only translate a template expression if it contains alphabet characters outside the spans
@@ -441,7 +448,7 @@ async function main(): Promise<void> {
             // By default, generate translations for all supported languages. Can be overridden with the --locales flag
             locales: {
                 description: 'Locales to generate translations for.',
-                default: TRANSLATION_TARGET_LOCALES,
+                default: Object.values(TRANSLATION_TARGET_LOCALES).filter((locale) => locale !== LOCALES.ES),
                 parse: (val: string): TranslationTargetLocale[] => {
                     const rawLocales = val.split(',');
                     const validatedLocales: TranslationTargetLocale[] = [];

package.json

@@ -76,7 +76,7 @@ class OpenAIUtils {
         let count = 0;
         while (!response && count < OpenAIUtils.MAX_POLL_COUNT) {
             // await thread run completion
-            threadRun = await this.client.beta.threads.runs.retrieve(threadRun.thread_id, threadRun.id);
+            threadRun = await this.client.beta.threads.runs.retrieve(threadRun.thread_id, {thread_id: threadRun.id});
             if (threadRun.status !== 'completed') {
                 count++;
                 await new Promise((resolve) => {
@@ -110,7 +110,7 @@ class OpenAIUtils {
         // Handle known/predictable API errors
         if (error instanceof OpenAI.APIError) {
             // Only retry 429 (rate limit) or 5xx errors
-            const status = error.status;
+            const status = error.status as number;
             return !!status && (status === 429 || status >= 500);
         }
 

scripts/generateTranslations.ts

@@ -9,7 +9,7 @@ class ChatGPTTranslator extends Translator {
     /**
      * The maximum number of times we'll retry a successful translation request in the event of hallucinations.
      */
-    private static readonly MAX_RETRIES: number = 2;
+    private static readonly MAX_RETRIES: number = 4;
 
     /**
      * OpenAI API client to perform translations.
@@ -36,6 +36,9 @@ class ChatGPTTranslator extends Translator {
                 });
 
                 if (this.validateTemplatePlaceholders(text, result)) {
+                    if (attempt > 0) {
+                        console.log(`🙃 Translation succeeded after ${attempt + 1} attempts`);
+                    }
                     console.log(`🧠 Translated "${text}" to ${targetLang}: "${result}"`);
                     return result;
                 }