Refactor: Move to data/ directory structure with R2 backup

marcarl · marcarl · commit ff60339e432b · 2026-01-05T10:24:57.000+01:00
Changes:

- Reorganize file structure: data/sfs_json/ and data/md-markers/

- Add Cloudflare R2 backup for JSON files (redundancy)

- Make git commits configurable (enable_git_commit input)

- Add smart fallback in html-export (git first, R2 as backup)

- Remove gitignore exceptions for cleaner config

This provides better organization, permanent R2 backup, and flexibility

to disable git commits when only R2 storage is needed.
diff --git a/.github/workflows/fetch-sfs-workflow.yml b/.github/workflows/fetch-sfs-workflow.yml
@@ -11,6 +11,16 @@ on:
         required: false
         default: '30'
         type: string
+      enable_git_commit:
+        description: 'Aktivera git commits av markdown-filer'
+        required: false
+        default: true
+        type: boolean
+      branch_name:
+        description: 'Branch namn att använda (krävs om enable_git_commit är true)'
+        required: false
+        default: 'workflow-artifact-data'
+        type: string
 
 permissions:
   contents: write
@@ -38,29 +48,47 @@ jobs:
 
     - name: Fetch JSON from beta.rkrattsbaser.gov.se
       run: |
-        python downloaders/fetch_new_sfs_docs.py --days ${{ inputs.days || '1' }} --output sfs_json
+        python downloaders/fetch_new_sfs_docs.py --days ${{ inputs.days || '1' }} --output data/sfs_json
       env:
         PYTHONPATH: ${{ github.workspace }}
 
     - name: Process JSON files to Markdown files with Selex tags
       run: |
-        python sfs_processor.py --input sfs_json --output output/md --formats md-markers
+        python sfs_processor.py --input data/sfs_json --output data/md-markers --formats md-markers
       env:
         PYTHONPATH: ${{ github.workspace }}
 
+    - name: Upload JSON source files to Cloudflare R2 for permanent storage
+      run: |
+        aws configure set aws_access_key_id ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
+        aws configure set aws_secret_access_key ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
+        aws configure set region us-east-1
+        aws configure set output json
+
+        # Upload all JSON files to R2 for backup
+        aws s3 sync data/sfs_json/ s3://${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }}/sfs_json/ \
+          --endpoint-url https://${{ secrets.CLOUDFLARE_R2_ACCOUNT_ID }}.r2.cloudflarestorage.com \
+          --content-type "application/json" \
+          --exclude "*.md" \
+          --include "*.json"
+      env:
+        AWS_DEFAULT_REGION: us-east-1
+
     - name: Configure Git
+      if: inputs.enable_git_commit != 'false'
       run: |
         git config --local user.email "action@github.com"
         git config --local user.name "GitHub Action"
 
     - name: Commit and push changes
+      if: inputs.enable_git_commit != 'false'
       id: commit_changes
       run: |
         # Get current branch
         CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
-        
+
         # Create a new branch for commits if we have changes
-        git add output/md/ sfs_json/
+        git add data/md-markers/ data/sfs_json/
         if git diff --staged --quiet; then
           echo "Inga nya filer att committa"
           echo "has_changes=false" >> $GITHUB_OUTPUT
@@ -71,8 +99,14 @@ jobs:
           
           # Create and switch to the new branch
           git checkout -b "$COMMIT_BRANCH"
-          
-          git commit -m "Automatisk uppdatering av SFS-författningar $(date +'%Y-%m-%d')"
+
+          # Commit with multi-line message
+          git commit -m "Automatisk uppdatering av SFS-författningar $(date +'%Y-%m-%d')" \
+                     -m "" \
+                     -m "Inkluderar:" \
+                     -m "- Käll-JSON (data/sfs_json/)" \
+                     -m "- Markdown med selex-taggar (data/md-markers/)" \
+                     -m "- Backup till Cloudflare R2"
           echo "has_changes=true" >> $GITHUB_OUTPUT
           echo "commit_branch=${COMMIT_BRANCH}" >> $GITHUB_OUTPUT
           
@@ -113,16 +147,24 @@ jobs:
         fi
 
     - name: Trigger HTML export workflow
-      if: steps.commit_changes.outputs.has_changes == 'true' && steps.commit_changes.outputs.push_success == 'true'
+      if: (inputs.enable_git_commit != 'false' && steps.commit_changes.outputs.has_changes == 'true' && steps.commit_changes.outputs.push_success == 'true') || (inputs.enable_git_commit == 'false')
       uses: actions/github-script@v6
       with:
         github-token: ${{ secrets.GITHUB_TOKEN }}
         script: |
+          const enableGitCommit = '${{ inputs.enable_git_commit }}' !== 'false';
+          const sourceRef = enableGitCommit
+            ? '${{ steps.commit_changes.outputs.commit_branch }}'
+            : '${{ inputs.branch_name }}';
+
           github.rest.actions.createWorkflowDispatch({
             owner: context.repo.owner,
             repo: context.repo.repo,
             workflow_id: 'html-export-workflow.yml',
-            ref: context.ref
+            ref: sourceRef || context.ref,
+            inputs: {
+              source_ref: sourceRef || 'main'
+            }
           })
 
     - name: Report push failure
diff --git a/.github/workflows/html-export-workflow.yml b/.github/workflows/html-export-workflow.yml
@@ -45,20 +45,52 @@ jobs:
         python -m pip install --upgrade pip
         pip install -r requirements.txt
 
+    - name: Get JSON source files (from git or R2)
+      run: |
+        # Try to use JSON files from git first
+        if [ -d "data/sfs_json" ] && [ -n "$(ls -A data/sfs_json 2>/dev/null)" ]; then
+          echo "✅ Found $(find data/sfs_json -name '*.json' | wc -l) JSON files in git"
+          echo "Using JSON files from git checkout"
+        else
+          echo "⚠️  No JSON files in git, downloading from Cloudflare R2..."
+
+          # Configure AWS CLI for R2
+          aws configure set aws_access_key_id ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
+          aws configure set aws_secret_access_key ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
+          aws configure set region us-east-1
+          aws configure set output json
+
+          # Download all JSON files from R2
+          mkdir -p data/sfs_json
+          aws s3 sync s3://${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }}/sfs_json/ data/sfs_json/ \
+            --endpoint-url https://${{ secrets.CLOUDFLARE_R2_ACCOUNT_ID }}.r2.cloudflarestorage.com \
+            --exclude "*" \
+            --include "*.json"
+
+          # Verify download
+          if [ ! -d "data/sfs_json" ] || [ -z "$(ls -A data/sfs_json)" ]; then
+            echo "::error::Failed to download JSON files from R2"
+            exit 1
+          fi
+          echo "✅ Downloaded $(find data/sfs_json -name '*.json' | wc -l) JSON files from R2"
+        fi
+      env:
+        AWS_DEFAULT_REGION: us-east-1
+
     - name: Generate HTML export
       run: |
         if [ -n "${{ inputs.filter }}" ]; then
-          python sfs_processor.py --input sfs_json --output output/html --formats html --filter "${{ inputs.filter }}"
+          python sfs_processor.py --input data/sfs_json --output output/html --formats html --filter "${{ inputs.filter }}"
         else
-          python sfs_processor.py --input sfs_json --output output/html --formats html
+          python sfs_processor.py --input data/sfs_json --output output/html --formats html
         fi
       env:
         PYTHONPATH: ${{ github.workspace }}
 
     - name: Regenerate index pages for HTML export
       run: |
-        python exporters/html/populate_index_pages.py --input sfs_json --output index.html --limit 30
-        python exporters/html/populate_index_pages.py --input sfs_json --output latest.html --limit 10
+        python exporters/html/populate_index_pages.py --input data/sfs_json --output index.html --limit 30
+        python exporters/html/populate_index_pages.py --input data/sfs_json --output latest.html --limit 10
       env:
         PYTHONPATH: ${{ github.workspace }}
 
diff --git a/.github/workflows/upcoming-changes-workflow.yml b/.github/workflows/upcoming-changes-workflow.yml
@@ -32,7 +32,7 @@ jobs:
     - name: Run upcoming changes script
       run: |
         # Kör scriptet på markdown-katalogen för att uppdatera kommande.yaml
-        python temporal/upcoming_changes.py output/md/
+        python temporal/upcoming_changes.py data/md-markers/
       env:
         PYTHONPATH: ${{ github.workspace }}
 
@@ -42,7 +42,7 @@ jobs:
         # Detta gör att vi kan hämta ikapp missade dagar om jobbet misslyckats
         eval $(python temporal/get_temporal_date_range.py)
         echo "Bearbetar temporal commits från $FROM_DATE till $TO_DATE"
-        python scripts/temporal_commits_batch.py output/md/ --from-date "$FROM_DATE" --to-date "$TO_DATE" --verbose
+        python scripts/temporal_commits_batch.py data/md-markers/ --from-date "$FROM_DATE" --to-date "$TO_DATE" --verbose
       env:
         PYTHONPATH: ${{ github.workspace }}
         GIT_GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.gitignore b/.gitignore
@@ -12,6 +12,11 @@ ___pycache__
 htmlcov/
 .pytest_cache/
 
-# Ignore output files
+# Ignore output files (generated, not committed)
 logs/
-output/
+output/
+
+# Data directory contains source files committed to git:
+# - data/sfs_json/     = Käll-JSON från API
+# - data/md-markers/   = Markdown med selex-taggar
+# (nothing to ignore here, committed to git + backed up to R2)
