Enhance Git processing: implement batch processing for git commits etc.

marcarl · marcarl · commit 3f72c90d32cc · 2025-07-12T16:41:20.000+02:00
diff --git a/.github/workflows/fetch-sfs-workflow.yml b/.github/workflows/fetch-sfs-workflow.yml
@@ -42,9 +42,9 @@ jobs:
       env:
         PYTHONPATH: ${{ github.workspace }}
 
-    - name: Process JSON files to Markdown files
+    - name: Process JSON files to Markdown files with Selex tags
       run: |
-        python sfs_processor.py --input sfs_json --output SFS --formats md,git
+        python sfs_processor.py --input sfs_json --output SFS --formats md-markers
       env:
         PYTHONPATH: ${{ github.workspace }}
 
diff --git a/exporters/git/__init__.py b/exporters/git/__init__.py
@@ -1,21 +1,23 @@
 """Git export functionality for SFS documents."""
 
 from .git_utils import (
-    ensure_git_branch_for_commits, 
+    prepare_git_branch, 
     restore_original_branch, 
     remove_all_commits_on_branch,
     get_target_repository,
     configure_git_remote,
-    push_to_target_repository
+    push_to_target_repository,
+    clone_target_repository_to_temp
 )
 from .generate_init_commit_for_doc import generate_init_commit_for_document
 
 __all__ = [
-    'ensure_git_branch_for_commits', 
+    'prepare_git_branch', 
     'restore_original_branch', 
     'remove_all_commits_on_branch',
     'get_target_repository',
     'configure_git_remote', 
     'push_to_target_repository',
+    'clone_target_repository_to_temp',
     'generate_init_commit_for_document'
 ]
diff --git a/exporters/git/generate_init_commit_for_doc.py b/exporters/git/generate_init_commit_for_doc.py
@@ -2,96 +2,108 @@
 """
 Module for generating initial Git commits for SFS documents.
 
-This module handles the complete git workflow for creating initial commits
-for SFS documents, including branch management and cleanup.
+This module handles creating commits for SFS documents without managing
+the overall git workflow (branching, pushing, etc).
 """
 
+import os
+import re
+import subprocess
 from pathlib import Path
-from typing import Optional
 
-from exporters.git import ensure_git_branch_for_commits, restore_original_branch
-from exporters.git.generate_commits import create_init_git_commit
+from exporters.git.git_utils import GIT_TIMEOUT
 from util.file_utils import save_to_disk
 from formatters.format_sfs_text import clean_selex_tags
-from util.datetime_utils import format_datetime
+from util.datetime_utils import format_datetime, format_datetime_for_git
 
 
 def generate_init_commit_for_document(
     data: dict,
     output_file: Path,
     markdown_content: str,
-    git_branch: str,
-    preserve_section_tags: bool = False,
     verbose: bool = False
 ) -> str:
     """
-    Generate initial git commit for an SFS document with proper branch handling.
-    
-    This function handles the complete git workflow:
-    1. Creates a separate git branch for commits
-    2. Creates the initial commit with document metadata
-    3. Restores the original branch
-    4. Writes the final file content
-    
+    Generate initial git commit for an SFS document.
+
+    This function handles creating commits for individual documents.
+    It assumes we're already in a git repository and on the correct branch.
+
     Args:
         data: JSON data containing document information
-        output_file: Path to the output markdown file
+        output_file: Path to the output markdown file (for local reference)
         markdown_content: The markdown content to commit and save
-        git_branch: Branch name to use for git commits
-        preserve_section_tags: Whether to preserve <section> tags in final output
         verbose: Enable verbose output
-        
+
     Returns:
-        str: The final markdown content (cleaned if preserve_section_tags is False)
+        str: The final markdown content (cleaned, without selex tags)
     """
     # Extract document metadata
-    beteckning = data.get('beteckning', 'Unknown')
-    rubrik = data.get('rubrik', '')
-    utfardad_datum = format_datetime(data.get('fulltext', {}).get('utfardadDateTime'))
-    
-    # Ensure commits are made in a different branch
-    original_branch, commit_branch = ensure_git_branch_for_commits(
-        git_branch, 
-        remove_all_commits_first=True, 
-        verbose=verbose
-    )
+    beteckning = data.get('beteckning')
+    if not beteckning:
+        raise ValueError("Beteckning saknas i dokumentdata")
     
-    # Ensure branch creation was successful
-    if original_branch is None or commit_branch is None:
-        raise RuntimeError(f"Misslyckades att skapa git branch för {beteckning}")
+    rubrik = data.get('rubrik')
+    if not rubrik:
+        raise ValueError("Rubrik saknas i dokumentdata")
     
-    try:
-        # Only create main commit if we have utfardad_datum
-        if utfardad_datum:
-            # Get förarbeten if available
-            register_data = data.get('register', {})
-            predocs = register_data.get('forarbeten')
-            
-            # Create initial git commit
-            success = create_init_git_commit(
-                output_file=output_file,
-                markdown_content=markdown_content,
-                beteckning=beteckning,
-                rubrik=rubrik,
-                utfardad_datum=utfardad_datum,
-                predocs=predocs,
-                verbose=verbose
-            )
-            
-            if not success:
-                print(f"Git-commit misslyckades för {beteckning}")
+    utfardad_datum = format_datetime(data.get('fulltext', {}).get('utfardadDateTime'))
+
+    # Prepare final content for local save (always clean selex tags in git mode)
+    final_content = clean_selex_tags(markdown_content)
+
+    # Save file locally for reference
+    save_to_disk(output_file, final_content)
+    print(f"Skapade dokument: {output_file}")
+
+    # Create git commit if we have utfardad_datum
+    if utfardad_datum:
+        # Extract year from beteckning for directory structure
+        year_match = re.search(r'(\d{4}):', beteckning)
+        if year_match:
+            year = year_match.group(1)
+            relative_path = Path(year) / output_file.name
         else:
-            # Write file if no utfardad_datum available
-            save_to_disk(output_file, markdown_content)
-            print(f"Skrev fil utan git-commit (inget utfärdandedatum): {output_file}")
+            relative_path = Path(output_file.name)
+
+        # Create directory structure if needed
+        target_file = Path.cwd() / relative_path
+        target_file.parent.mkdir(parents=True, exist_ok=True)
+
+        # Write the file (use clean content without selex tags for git)
+        clean_content = clean_selex_tags(markdown_content)
         
-    finally:
-        # Always restore original branch after git operations
-        restore_original_branch(original_branch)
-    
-    # Prepare final content for return
-    final_content = markdown_content
-    if not preserve_section_tags:
-        final_content = clean_selex_tags(final_content)
-    
+        with open(target_file, 'w', encoding='utf-8') as f:
+            f.write(clean_content)
+
+        # Stage the file
+        subprocess.run(['git', 'add', str(relative_path)],
+                     check=True, capture_output=True, timeout=GIT_TIMEOUT)
+
+        # Prepare commit message
+        commit_message = rubrik
+
+        # Add förarbeten if available
+        register_data = data.get('register', {})
+        predocs = register_data.get('forarbeten')
+        if predocs:
+            commit_message += (f"\n\nHar tillkommit i Svensk författningssamling "
+                             f"efter dessa förarbeten: {predocs}")
+
+        # Format date for git
+        commit_date = format_datetime_for_git(utfardad_datum)
+
+        # Create commit with specified date
+        env = {**os.environ, 'GIT_AUTHOR_DATE': commit_date, 'GIT_COMMITTER_DATE': commit_date}
+        subprocess.run([
+            'git', 'commit', '-m', commit_message
+        ], check=True, capture_output=True, env=env, timeout=GIT_TIMEOUT)
+
+        if verbose:
+            print(f"Git-commit skapad: '{commit_message}' daterad {commit_date}")
+
+    elif not utfardad_datum:
+        if verbose:
+            print(f"Hoppade över git-commit (inget utfärdandedatum): {beteckning}")
+
     return final_content
diff --git a/exporters/git/git_utils.py b/exporters/git/git_utils.py
@@ -3,6 +3,7 @@
 import os
 import subprocess
 from datetime import datetime
+from pathlib import Path
 from urllib.parse import urlparse
 
 # Git main branch name
@@ -15,7 +16,7 @@
 DEFAULT_TARGET_REPO = "https://github.com/se-lex/sfs.git"
 
 
-def ensure_git_branch_for_commits(git_branch, remove_all_commits_first=True, verbose=False):
+def prepare_git_branch(git_branch, remove_all_commits_first=True, verbose=False):
     """
     Ensures that git commits are made in a different branch than the current one.
     Creates a new branch if needed and switches to it.
@@ -229,6 +230,67 @@ def create_authenticated_url(repo_url: str, pat_token: str) -> str:
         return repo_url
 
 
+def clone_target_repository_to_temp(verbose: bool = False) -> tuple[Path, str]:
+    """
+    Clone target repository to a temporary directory.
+    
+    Args:
+        verbose: Enable verbose output
+        
+    Returns:
+        tuple[Path, str]: (repo_directory_path, original_cwd) or (None, None) if failed
+    """
+    import tempfile
+    
+    try:
+        # Get repository URL and PAT token
+        repo_url = get_target_repository()
+        pat_token = os.getenv('GIT_GITHUB_PAT')
+        
+        # Try to load PAT from .env file if not in environment
+        if not pat_token:
+            try:
+                from dotenv import load_dotenv
+                load_dotenv()
+                pat_token = os.getenv('GIT_GITHUB_PAT')
+            except ImportError:
+                pass  # dotenv not available
+        
+        # Create authenticated URL if PAT is available
+        if pat_token:
+            auth_url = create_authenticated_url(repo_url, pat_token)
+        else:
+            auth_url = repo_url
+            if verbose:
+                print("Varning: Ingen PAT token hittades, använder okrypterad URL")
+            
+        # Create temporary directory for cloning
+        temp_dir = tempfile.mkdtemp()
+        repo_dir = Path(temp_dir) / "target_repo"
+        
+        if verbose:
+            print(f"Klonar {repo_url} till temporär katalog...")
+        
+        # Clone the repository
+        subprocess.run([
+            'git', 'clone', auth_url, str(repo_dir)
+        ], check=True, capture_output=True, timeout=GIT_TIMEOUT)
+        
+        # Remember original directory
+        original_cwd = os.getcwd()
+        
+        return repo_dir, original_cwd
+        
+    except subprocess.CalledProcessError as e:
+        print(f"Fel vid kloning av target repository: {e}")
+        if hasattr(e, 'stderr') and e.stderr:
+            print(f"Git stderr: {e.stderr.decode('utf-8', errors='replace')}")
+        return None, None
+    except Exception as e:
+        print(f"Oväntat fel vid kloning av target repository: {e}")
+        return None, None
+
+
 def push_to_target_repository(branch_name: str, remote_name: str = 'target', verbose: bool = False) -> bool:
     """
     Push the specified branch to the target repository.
diff --git a/sfs_processor.py b/sfs_processor.py
