Refactor batch processing in git exports: enhance batch handling, improve branch checkout logic, and streamline file processing

Author: marcarl

exporters/git/batch_processor.py

@@ -6,20 +6,19 @@
 """
 
 import os
-import shutil
-import subprocess
 from datetime import datetime
 from pathlib import Path
 import random
 import json
 
 from exporters.git import clone_target_repository_to_temp
-from exporters.git.git_utils import GIT_TIMEOUT
+from exporters.git.git_utils import checkout_branch, push_to_target_repository
+from sfs_processor import make_document
 
 
-def process_files_with_git_batch(json_files, output_dir, verbose, predocs):
-    """Process files with git batch workflow."""
-    # Clone target repository once for all documents
+def process_files_with_git_batch(json_files, output_dir, verbose, predocs, batch_size=10):
+    """Process files with git batch workflow, using same branch but pushing after each batch."""
+    # Clone target repository once for all batches
     repo_dir, original_cwd = clone_target_repository_to_temp(verbose=verbose)
     if repo_dir is None:
         raise RuntimeError("Failed to clone target repository")
@@ -28,60 +27,75 @@ def process_files_with_git_batch(json_files, output_dir, verbose, predocs):
         # Change to cloned repository directory
         os.chdir(repo_dir)
 
-        # Create unique branch name for this batch
+        # Create unique branch name for this entire operation
         timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
         random_suffix = random.randint(1000, 9999)
         unique_branch = f"batch_{timestamp}_{random_suffix}"
 
-        # Create and checkout new branch directly
-        try:
-            subprocess.run(['git', 'checkout', '-b', unique_branch],
-                         check=True, capture_output=True, timeout=GIT_TIMEOUT)
-            if verbose:
-                print(f"Skapade och bytte till branch '{unique_branch}' för batch-commits")
-        except subprocess.CalledProcessError as e:
-            print(f"Fel: Kunde inte skapa git branch: {e}")
+        # Create and checkout new branch
+        if not checkout_branch(unique_branch, create_if_missing=True, verbose=verbose):
+            print(f"Fel: Kunde inte skapa git branch: {unique_branch}")
             return
 
-        # Process each JSON file
-        from sfs_processor import make_document
-        for json_file in json_files:
-            # Use absolute path since we changed working directory
-            abs_json_file = Path(original_cwd) / json_file
-            try:
-                with open(abs_json_file, 'r', encoding='utf-8') as f:
-                    data = json.load(f)
-            except (json.JSONDecodeError, FileNotFoundError) as e:
-                print(f"Fel vid läsning av {abs_json_file}: {e}")
-                continue
-
-            # Create documents in the cloned repository AND save to original output directory
-            # First convert to absolute path since we changed working directory
-            original_output_dir = Path(original_cwd) / Path(output_dir).name if not Path(output_dir).is_absolute() else Path(output_dir)
-            make_document(data, original_output_dir, ["git"], True, verbose, True, predocs, True)
-
-        # Push all commits to target repository
-        if verbose:
-            print(f"Pushar batch till target repository...")
-
-        from exporters.git.git_utils import push_to_target_repository
-        if push_to_target_repository(unique_branch, 'origin', verbose):
-            print(f"Batch pushad till target repository som branch '{unique_branch}'")
+        # Split files into batches
+        total_files = len(json_files)
+        if total_files > batch_size:
+            print(f"Delar upp {total_files} filer i batcher om {batch_size} filer var")
+            batches = [json_files[i:i + batch_size] for i in range(0, total_files, batch_size)]
+            print(f"Skapade {len(batches)} batcher")
+            
+            # Process each batch in the same repository and branch, pushing after each
+            for i, batch in enumerate(batches, 1):
+                print(f"\nBearbetar batch {i}/{len(batches)} ({len(batch)} filer)...")
+                _process_batch_files(batch, output_dir, verbose, predocs, original_cwd, i, len(batches))
+                
+                # Push after each batch
+                print(f"Pushar batch {i}/{len(batches)} till target repository...")
+                if push_to_target_repository(unique_branch, 'origin', verbose):
+                    print(f"Batch {i}/{len(batches)} pushad till target repository som branch '{unique_branch}'")
+                else:
+                    print(f"Misslyckades med att pusha batch {i}/{len(batches)} till target repository")
         else:
-            print(f"Misslyckades med att pusha batch till target repository")
+            print(f"Bearbetar {total_files} filer i en enda batch...")
+            _process_batch_files(json_files, output_dir, verbose, predocs, original_cwd, 1, 1)
+            
+            # Push the single batch
+            print(f"Pushar alla {total_files} filer till target repository...")
+            if push_to_target_repository(unique_branch, 'origin', verbose):
+                print(f"Alla {total_files} filer pushade till target repository som branch '{unique_branch}'")
+            else:
+                print(f"Misslyckades med att pusha till target repository")
 
-    except subprocess.CalledProcessError as e:
-        print(f"Fel vid git batch processing: {e}")
-        if hasattr(e, 'stderr') and e.stderr:
-            print(f"Git stderr: {e.stderr.decode('utf-8', errors='replace')}")
     except Exception as e:
         print(f"Oväntat fel vid git batch processing: {e}")
     finally:
         # Always change back to original directory
         os.chdir(original_cwd)
-        # Clean up temporary directory
+
+
+def _process_batch_files(json_files, output_dir, verbose, predocs, original_cwd, batch_num, total_batches):
+    """Process batch files in the current repository without creating new branches."""
+    # Process each JSON file in the current git repository
+    from sfs_processor import make_document
+    for json_file in json_files:
+        # Use absolute path since we changed working directory
+        abs_json_file = Path(original_cwd) / json_file
         try:
-            shutil.rmtree(repo_dir.parent)
-        except Exception as e:
-            if verbose:
-                print(f"Varning: Kunde inte rensa temporär katalog: {e}")
+            with open(abs_json_file, 'r', encoding='utf-8') as f:
+                data = json.load(f)
+        except (json.JSONDecodeError, FileNotFoundError) as e:
+            print(f"Fel vid läsning av {abs_json_file}: {e}")
+            continue
+
+        # Create documents in the cloned repository AND save to original output directory
+        # First convert to absolute path since we changed working directory
+        if not Path(output_dir).is_absolute():
+            original_output_dir = Path(original_cwd) / Path(output_dir).name
+        else:
+            original_output_dir = Path(output_dir)
+        make_document(data, original_output_dir, ["git"], True, verbose, True, predocs, True)
+    
+    if verbose:
+        print(f"Batch {batch_num}/{total_batches} bearbetad ({len(json_files)} filer)")
+
+

exporters/git/git_utils.py

@@ -359,6 +359,49 @@ def stage_file(file_path: str, verbose: bool = False) -> bool:
         return False
 
 
+def checkout_branch(branch_name: str, create_if_missing: bool = True, verbose: bool = False) -> bool:
+    """
+    Checkout to a git branch, optionally creating it if it doesn't exist.
+    
+    Args:
+        branch_name: Name of the branch to checkout
+        create_if_missing: If True, create the branch if it doesn't exist
+        verbose: Enable verbose output
+        
+    Returns:
+        bool: True if checkout was successful, False otherwise
+    """
+    try:
+        # Try to checkout the branch first
+        result = subprocess.run(['git', 'checkout', branch_name],
+                              capture_output=True, timeout=GIT_TIMEOUT)
+        
+        if result.returncode == 0:
+            if verbose:
+                print(f"Bytte till branch '{branch_name}'")
+            return True
+        elif create_if_missing:
+            # Branch doesn't exist, create it
+            subprocess.run(['git', 'checkout', '-b', branch_name],
+                          check=True, capture_output=True, timeout=GIT_TIMEOUT)
+            if verbose:
+                print(f"Skapade och bytte till branch '{branch_name}'")
+            return True
+        else:
+            if verbose:
+                print(f"Branch '{branch_name}' finns inte och create_if_missing=False")
+            return False
+            
+    except subprocess.CalledProcessError as e:
+        print(f"Fel vid checkout av branch '{branch_name}': {e}")
+        if hasattr(e, 'stderr') and e.stderr:
+            print(f"Git stderr: {e.stderr.decode('utf-8', errors='replace')}")
+        return False
+    except FileNotFoundError:
+        print("Varning: Git hittades inte.")
+        return False
+
+
 def create_commit_with_date(message: str, date: str, verbose: bool = False) -> bool:
     """
     Create a git commit with a specified date.

sfs_processor.py

@@ -23,7 +23,6 @@
 from typing import Dict, Any, List, Optional
 
 from downloaders.riksdagen_api import fetch_predocs_details, format_predocs_for_frontmatter
-from exporters.git.git_utils import GIT_TIMEOUT
 from formatters.format_sfs_text import (
     format_sfs_text_as_markdown,
     parse_logical_sections,
@@ -35,14 +34,13 @@
 from formatters.add_pdf_url_to_frontmatter import generate_pdf_url
 from formatters.frontmatter_manager import add_ikraft_datum_to_frontmatter
 from temporal.title_temporal import title_temporal
-from temporal.amendments import process_markdown_amendments, extract_amendments
+from temporal.amendments import extract_amendments
 from temporal.apply_temporal import apply_temporal
 from exporters.git import create_init_git_commit
 from util.yaml_utils import format_yaml_value
 from util.datetime_utils import format_datetime
 from util.file_utils import filter_json_files, save_to_disk
 from formatters.predocs_parser import parse_predocs_string
-from formatters.table_converter import convert_tables_in_markdown
 
 
 def create_safe_filename(beteckning: str, preserve_section_tags: bool = False) -> str: