f

Author: carlospolop

scripts/translator.py

@@ -10,6 +10,7 @@
 import concurrent.futures
 from tqdm import tqdm #pip3 install tqdm
 import traceback
+import re
 
 
 MASTER_BRANCH = "master"
@@ -85,6 +86,10 @@ def _get_encoding_for_model(model: str):
         print(f"Tokenizer for model {model} not found. Falling back to {FINAL_TOKENIZER_FALLBACK}.")
         return tiktoken.get_encoding(FINAL_TOKENIZER_FALLBACK)
 
+def _fix_translated_shortcodes(text: str) -> str:
+    """Keep mdBook shortcode attribute names valid after translation."""
+    return re.sub(r'(\{\{#tab\s+)(?!name=)[^=\s}]+=', r'\1name=', text)
+
 def reportTokens(prompt, model):
     encoding = _get_encoding_for_model(model)
     # print number of tokens in light gray, with first 50 characters of prompt in green. if truncated, show that it is truncated
@@ -263,6 +268,7 @@ def translate_text(language, text, file_path, model, cont=0, slpitted=False, cli
     response_message = response_message.replace("bypassy", "bypasses") # PL translations translates that from time to time
     response_message = response_message.replace("Bypassy", "Bypasses")
     response_message = response_message.replace("-privec.md", "-privesc.md") # PL translations translates that from time to time
+    response_message = _fix_translated_shortcodes(response_message)
 
     # Sometimes chatgpt modified the number of "#" at the beginning of the text, so we need to fix that. This is specially important for the first line of the MD that mucst have only 1 "#"
     cont2 = 0