feat: parallel

Author: alterxyz

.github/workflows/translate.yml

@@ -28,16 +28,48 @@ jobs:
       - name: Get changed markdown files
         id: changed-files
         run: |
-          # Get the list of changed files between the current and previous commit
+          # Get the list of newly added files between the current and previous commit
           # We filter for .md and .mdx files that are inside the language directories
-          files=$(git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep -E '^(en|zh-hans|ja-jp|plugin-dev-en|plugin-dev-zh|plugin-dev-ja)/.*(\.md|\.mdx)$' || true)
-          if [[ -z "$files" ]]; then
-            echo "No markdown files to translate."
+          # Only include added (A) files, skip modified (M) and deleted (D) files
+          
+          # Try different approaches to get the diff
+          if [[ -n "${{ github.event.before }}" ]]; then
+            echo "Using github.event.before: ${{ github.event.before }}"
+            files=$(git diff --name-status ${{ github.event.before }} ${{ github.sha }} | grep -E '^A\s+' | cut -f2 | grep -E '^(en|en-us|zh-hans|ja-jp|plugin-dev-en|plugin-dev-zh|plugin-dev-ja|versions)/.*(\.md|\.mdx)$' || true)
+          else
+            echo "github.event.before is empty, using HEAD~1"
+            files=$(git diff --name-status HEAD~1 HEAD | grep -E '^A\s+' | cut -f2 | grep -E '^(en|en-us|zh-hans|ja-jp|plugin-dev-en|plugin-dev-zh|plugin-dev-ja|versions)/.*(\.md|\.mdx)$' || true)
+          fi
+          
+          echo "Detected files (Added only):"
+          echo "$files"
+          
+          # Filter out files that don't actually exist
+          existing_files=""
+          if [[ -n "$files" ]]; then
+            while IFS= read -r file; do
+              if [[ -n "$file" && -f "$file" ]]; then
+                if [[ -z "$existing_files" ]]; then
+                  existing_files="$file"
+                else
+                  existing_files="$existing_files"$'\n'"$file"
+                fi
+              else
+                echo "Skipping non-existent file: $file"
+              fi
+            done <<< "$files"
+          fi
+          
+          echo "Final files to translate:"
+          echo "$existing_files"
+          
+          if [[ -z "$existing_files" ]]; then
+            echo "No new markdown files to translate."
             echo "files=" >> $GITHUB_OUTPUT
           else
             # The script expects absolute paths, but we run it from the root, so relative is fine.
             echo "files<<EOF" >> $GITHUB_OUTPUT
-            echo "$files" >> $GITHUB_OUTPUT
+            echo "$existing_files" >> $GITHUB_OUTPUT
             echo "EOF" >> $GITHUB_OUTPUT
           fi
 
@@ -49,12 +81,35 @@ jobs:
           echo "Files to translate:"
           echo "${{ steps.changed-files.outputs.files }}"
           
-          echo "${{ steps.changed-files.outputs.files }}" | while IFS= read -r file; do
+          # Create temporary file list
+          echo "${{ steps.changed-files.outputs.files }}" > /tmp/files_to_translate.txt
+          
+          # Start all translation processes in parallel
+          pids=()
+          while IFS= read -r file; do
             if [[ -n "$file" ]]; then
-              echo "Translating $file..."
-              python tools/translate/main.py "$file" "$DIFY_API_KEY"
+              echo "Starting translation for $file..."
+              python tools/translate/main.py "$file" "$DIFY_API_KEY" &
+              pids+=($!)
+            fi
+          done < /tmp/files_to_translate.txt
+          
+          # Wait for all background processes to complete
+          echo "Waiting for ${#pids[@]} translation processes to complete..."
+          failed=0
+          for pid in "${pids[@]}"; do
+            if ! wait "$pid"; then
+              echo "Translation process $pid failed"
+              failed=1
             fi
           done
+          
+          if [ $failed -eq 1 ]; then
+            echo "Some translations failed"
+            exit 1
+          fi
+          
+          echo "All translations completed successfully"
 
       - name: Commit and push changes
         run: |

en/test.mdx

@@ -0,0 +1,5 @@
+---
+title: "test1"
+---
+
+Here is test 1

tools/translate/main.py

@@ -14,6 +14,21 @@
         "English": "plugin-dev-en",
         "Chinese": "plugin-dev-zh",
         "Japanese": "plugin-dev-ja"
+    },
+    "version_28x": {
+        "English": "versions/2-8-x/en-us",
+        "Chinese": "versions/2-8-x/zh-cn",
+        "Japanese": "versions/2-8-x/ja-jp"
+    },
+    "version_30x": {
+        "English": "versions/3-0-x/en-us",
+        "Chinese": "versions/3-0-x/zh-cn",
+        "Japanese": "versions/3-0-x/ja-jp"
+    },
+    "version_31x": {
+        "English": "versions/3-1-x/en-us",
+        "Chinese": "versions/3-1-x/zh-cn",
+        "Japanese": "versions/3-1-x/ja-jp"
     }
 }
 
@@ -109,12 +124,23 @@ def determine_doc_type_and_language(file_path):
     Determine document type and current language based on file path
     Returns (doc_type, current_language, language_name)
     """
-    path_parts = file_path.split(os.sep)
+    # Normalize path separators
+    normalized_path = file_path.replace(os.sep, '/')
 
+    # Collect all possible matches and find the longest one
+    matches = []
     for doc_type, languages in docs_structure.items():
         for lang_name, lang_code in languages.items():
-            if lang_code in path_parts:
-                return doc_type, lang_code, lang_name
+            # Normalize lang_code path separators too
+            normalized_lang_code = lang_code.replace(os.sep, '/')
+            if normalized_lang_code in normalized_path:
+                matches.append((len(normalized_lang_code), doc_type, lang_code, lang_name))
+    
+    # Return the match with the longest lang_code (most specific)
+    if matches:
+        matches.sort(reverse=True)  # Sort by length descending
+        _, doc_type, lang_code, lang_name = matches[0]
+        return doc_type, lang_code, lang_name
 
     return None, None, None
 

versions/3-1-x/zh-cn/test31xzh.mdx

@@ -0,0 +1,5 @@
+---
+title: "test1"
+---
+
+Here is test 31xzh