Refactor Git processing: move batch processing functionality to a dedicated module and update references

Author: marcarl

exporters/git/__init__.py

@@ -10,6 +10,7 @@
     clone_target_repository_to_temp
 )
 from .generate_init_commit_for_doc import generate_init_commit_for_document
+from .batch_processor import process_files_with_git_batch
 
 __all__ = [
     'prepare_git_branch', 
@@ -19,5 +20,6 @@
     'configure_git_remote', 
     'push_to_target_repository',
     'clone_target_repository_to_temp',
-    'generate_init_commit_for_document'
+    'generate_init_commit_for_document',
+    'process_files_with_git_batch'
 ]

exporters/git/batch_processor.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""
+Batch processing functionality for git exports.
+
+This module handles batch processing of multiple SFS documents to git repository.
+"""
+
+import os
+import shutil
+import subprocess
+from datetime import datetime
+from pathlib import Path
+import random
+import json
+
+from exporters.git import clone_target_repository_to_temp
+from exporters.git.git_utils import GIT_TIMEOUT
+
+
+def process_files_with_git_batch(json_files, output_dir, output_modes, year_folder, verbose, git_branch, predocs, apply_links):
+    """Process files with git batch workflow."""
+    # Clone target repository once for all documents
+    repo_dir, original_cwd = clone_target_repository_to_temp(verbose=verbose)
+    if repo_dir is None:
+        print("Fel: Kunde inte klona target repository, faller tillbaka på lokal bearbetning")
+        # Fallback to normal processing
+        from sfs_processor import make_document
+        for json_file in json_files:
+            try:
+                with open(json_file, 'r', encoding='utf-8') as f:
+                    data = json.load(f)
+            except (json.JSONDecodeError, FileNotFoundError) as e:
+                print(f"Fel vid läsning av {json_file}: {e}")
+                continue
+            make_document(data, output_dir, output_modes, year_folder, verbose, git_branch, predocs, apply_links)
+        return
+
+    try:
+        # Change to cloned repository directory
+        os.chdir(repo_dir)
+
+        # Create unique branch name for this batch
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        random_suffix = random.randint(1000, 9999)
+        unique_branch = f"{git_branch}_batch_{timestamp}_{random_suffix}"
+
+        # Create and checkout new branch directly
+        try:
+            subprocess.run(['git', 'checkout', '-b', unique_branch],
+                         check=True, capture_output=True, timeout=GIT_TIMEOUT)
+            if verbose:
+                print(f"Skapade och bytte till branch '{unique_branch}' för batch-commits")
+        except subprocess.CalledProcessError as e:
+            print(f"Fel: Kunde inte skapa git branch: {e}")
+            return
+
+        # Process each JSON file
+        from sfs_processor import make_document
+        for json_file in json_files:
+            # Use absolute path since we changed working directory
+            abs_json_file = Path(original_cwd) / json_file
+            try:
+                with open(abs_json_file, 'r', encoding='utf-8') as f:
+                    data = json.load(f)
+            except (json.JSONDecodeError, FileNotFoundError) as e:
+                print(f"Fel vid läsning av {abs_json_file}: {e}")
+                continue
+
+            # Create documents in the cloned repository
+            make_document(data, output_dir, output_modes, year_folder, verbose, git_branch, predocs, apply_links)
+
+        # Push all commits to target repository
+        if verbose:
+            print(f"Pushar batch till target repository...")
+
+        subprocess.run(['git', 'push', 'origin', unique_branch],
+                     check=True, capture_output=True, timeout=GIT_TIMEOUT)
+
+        print(f"Batch pushad till target repository som branch '{unique_branch}'")
+
+    except subprocess.CalledProcessError as e:
+        print(f"Fel vid git batch processing: {e}")
+        if hasattr(e, 'stderr') and e.stderr:
+            print(f"Git stderr: {e.stderr.decode('utf-8', errors='replace')}")
+    except Exception as e:
+        print(f"Oväntat fel vid git batch processing: {e}")
+    finally:
+        # Always change back to original directory
+        os.chdir(original_cwd)
+        # Clean up temporary directory
+        try:
+            shutil.rmtree(repo_dir.parent)
+        except Exception as e:
+            if verbose:
+                print(f"Varning: Kunde inte rensa temporär katalog: {e}")

sfs_processor.py

@@ -560,88 +560,6 @@ def create_ignored_markdown_content(data: Dict[str, Any], reason: str) -> str:
     return markdown_body
 
 
-def _process_files_with_git_batch(json_files, output_dir, output_modes, year_folder, verbose, git_branch, predocs, apply_links):
-    """Process files with git batch workflow."""
-    import shutil
-    import subprocess
-    from datetime import datetime
-    import random
-    from exporters.git import clone_target_repository_to_temp
-    from exporters.git.git_utils import GIT_TIMEOUT
-    
-    # Clone target repository once for all documents
-    repo_dir, original_cwd = clone_target_repository_to_temp(verbose=verbose)
-    if repo_dir is None:
-        print("Fel: Kunde inte klona target repository, faller tillbaka på lokal bearbetning")
-        # Fallback to normal processing
-        for json_file in json_files:
-            try:
-                with open(json_file, 'r', encoding='utf-8') as f:
-                    data = json.load(f)
-            except (json.JSONDecodeError, FileNotFoundError) as e:
-                print(f"Fel vid läsning av {json_file}: {e}")
-                continue
-            make_document(data, output_dir, output_modes, year_folder, verbose, git_branch, predocs, apply_links)
-        return
-    
-    try:
-        # Change to cloned repository directory
-        os.chdir(repo_dir)
-        
-        # Create unique branch name for this batch
-        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-        random_suffix = random.randint(1000, 9999)
-        unique_branch = f"{git_branch}_batch_{timestamp}_{random_suffix}"
-        
-        # Create and checkout new branch directly
-        try:
-            subprocess.run(['git', 'checkout', '-b', unique_branch], 
-                         check=True, capture_output=True, timeout=GIT_TIMEOUT)
-            if verbose:
-                print(f"Skapade och bytte till branch '{unique_branch}' för batch-commits")
-        except subprocess.CalledProcessError as e:
-            print(f"Fel: Kunde inte skapa git branch: {e}")
-            return
-        
-        # Process each JSON file
-        for json_file in json_files:
-            # Use absolute path since we changed working directory
-            abs_json_file = Path(original_cwd) / json_file
-            try:
-                with open(abs_json_file, 'r', encoding='utf-8') as f:
-                    data = json.load(f)
-            except (json.JSONDecodeError, FileNotFoundError) as e:
-                print(f"Fel vid läsning av {abs_json_file}: {e}")
-                continue
-
-            # Create documents in the cloned repository
-            make_document(data, output_dir, output_modes, year_folder, verbose, git_branch, predocs, apply_links)
-        
-        # Push all commits to target repository
-        if verbose:
-            print(f"Pushar batch till target repository...")
-        
-        subprocess.run(['git', 'push', 'origin', unique_branch], 
-                     check=True, capture_output=True, timeout=GIT_TIMEOUT)
-        
-        print(f"Batch pushad till target repository som branch '{unique_branch}'")
-        
-    except subprocess.CalledProcessError as e:
-        print(f"Fel vid git batch processing: {e}")
-        if hasattr(e, 'stderr') and e.stderr:
-            print(f"Git stderr: {e.stderr.decode('utf-8', errors='replace')}")
-    except Exception as e:
-        print(f"Oväntat fel vid git batch processing: {e}")
-    finally:
-        # Always change back to original directory
-        os.chdir(original_cwd)
-        # Clean up temporary directory
-        try:
-            shutil.rmtree(repo_dir.parent)
-        except Exception as e:
-            if verbose:
-                print(f"Varning: Kunde inte rensa temporär katalog: {e}")
-
 
 def main():
     """Main function to process all JSON files in the json directory."""
@@ -733,7 +651,8 @@ def main():
 
     # Handle git mode with batch processing
     if "git" in output_modes:
-        _process_files_with_git_batch(json_files, output_dir, output_modes, args.year_folder, args.verbose, args.git_branch, args.predocs, args.apply_links)
+        from exporters.git import process_files_with_git_batch
+        process_files_with_git_batch(json_files, output_dir, output_modes, args.year_folder, args.verbose, args.git_branch, args.predocs, args.apply_links)
     else:
         # Convert each JSON file normally
         for json_file in json_files: