Implement automated ingestion with dual-path routing and multi-service CI/CD

Author: BLMgithub

.dockerignore

@@ -1,20 +1,26 @@
-.git
-.github
-.vscode
+.git/
+.gitignore
+.gitattributes
+.github/
 
-runtime
-__pycache__
-*.pyc
-*.pyo
-*.pyd
+.venv/
+venv/
+env/
 
-.pytest_cache
-.ruff_cache
+__pycache__/
+*.py[cod]
+*$py.class
+.pytest_cache/
+.coverage
+.coveragerc
 
-tests
-docs
+.vscode/
+.idea/
+pyrightconfig.json
 
-data/published
-runtime
-data/run_artifact
-data/raw
+data/
+assets/
+docs/
+tests/
+
+dev-requirements.txt

.github/workflows/cd-extract.yml

@@ -0,0 +1,64 @@
+name: Deploy Data Extract Job
+
+# Only run when merged changes happen in data_extract folder
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'data_extract/**'
+
+permissions:
+  contents: 'read'
+  id-token: 'write'
+
+# Service configurations in GCP
+env:
+  REGION: us-east1
+  REPO_NAME: operations-artifacts
+  IMAGE_NAME: extractor
+  JOB_NAME: drive-extractor
+
+jobs:
+  build-and-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      # Handshake to GCP
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/auth@v2
+        with:
+          workload_identity_provider: ${{ secrets.WIF_PROVIDER }}
+          service_account: ${{ secrets.DEPLOYER_SA_EMAIL }}
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v2
+
+      # Handshake to Artifact registry
+      - name: Configure Docker Auth
+        run: gcloud auth configure-docker ${{ env.REGION }}-docker.pkg.dev --quiet
+
+
+      - name: Build and Push Docker Image
+        run: |
+          # Full image path using the Git Commit Hash (github.sha)
+          IMAGE_PATH="${{ env.REGION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/${{ env.REPO_NAME }}/${{ env.IMAGE_NAME }}:${{ github.sha }}"
+          
+          # Build from root to ensure COPY commands find the files
+          docker build -f data_extract/Dockerfile -t $IMAGE_PATH .
+          
+          # Push to Artifact Registry
+          docker push $IMAGE_PATH
+
+      
+      - name: Deploy to Cloud Run Job
+        run: |
+          IMAGE_PATH="${{ env.REGION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/${{ env.REPO_NAME }}/${{ env.IMAGE_NAME }}:${{ github.sha }}"
+          
+          gcloud run jobs deploy ${{ env.JOB_NAME }} \
+            --image $IMAGE_PATH \
+            --region ${{ env.REGION }} \
+            --service-account ${{ secrets.EXTRACTOR_SA_EMAIL }}

.github/workflows/cd-pipeline.yml

@@ -0,0 +1,64 @@
+name: Deploy Data Pipeline Job
+
+# Only run when merged changes happen in data_pipeline folder
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'data_pipeline/**'
+
+permissions:
+  contents: 'read'
+  id-token: 'write'
+
+# Service configurations in GCP
+env:
+  REGION: us-east1
+  REPO_NAME: operations-artifacts
+  IMAGE_NAME: pipeline
+  JOB_NAME: operations-pipeline
+
+jobs:
+  build-and-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      # Handshake to GCP
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/auth@v2
+        with:
+          workload_identity_provider: ${{ secrets.WIF_PROVIDER }}
+          service_account: ${{ secrets.DEPLOYER_SA_EMAIL }}
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v2
+
+      # Handshake to Artifact registry
+      - name: Configure Docker Auth
+        run: gcloud auth configure-docker ${{ env.REGION }}-docker.pkg.dev --quiet
+
+
+      - name: Build and Push Docker Image
+        run: |
+          # Full image path using the Git Commit Hash (github.sha)
+          IMAGE_PATH="${{ env.REGION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/${{ env.REPO_NAME }}/${{ env.IMAGE_NAME }}:${{ github.sha }}"
+          
+          # Build from root to ensure COPY commands find the files
+          docker build -f data_pipeline/Dockerfile -t $IMAGE_PATH .
+          
+          # Push to Artifact Registry
+          docker push $IMAGE_PATH
+
+      
+      - name: Deploy to Cloud Run Job
+        run: |
+          IMAGE_PATH="${{ env.REGION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/${{ env.REPO_NAME }}/${{ env.IMAGE_NAME }}:${{ github.sha }}"
+          
+          gcloud run jobs deploy ${{ env.JOB_NAME }} \
+            --image $IMAGE_PATH \
+            --region ${{ env.REGION }} \
+            --service-account ${{ secrets.PIPELINE_SA_EMAIL }}

.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-name: CI
+name: CI - Code Quality & Tests
 
 on:
   pull_request:
@@ -19,21 +19,21 @@ jobs:
         with:
           python-version: "3.11"
 
-      - name: Install dependencies
+      - name: Install Dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          pip install -r dev-requirements.txt
 
       - name: Set PYTHONPATH
         run: echo "PYTHONPATH=$PWD" >> $GITHUB_ENV
 
-      - name: Check formatting
+      - name: Check Code Formatting
         run: black --check .
 
-      - name: Lint with ruff
+      - name: Ruff Linting
         run: ruff check .
 
-      - name: Run tests with coverage enforcement
+      - name: Run Tests with Coverage
         run: |
           pytest \
             --cov=data_pipeline \

.vscode/settings.json

@@ -4,11 +4,6 @@
     ],
     "python.testing.unittestEnabled": false,
     "python.testing.pytestEnabled": true,
-    "python-envs.pythonProjects": [
-        {
-            "path": "",
-            "envManager": "ms-python.python:conda",
-            "packageManager": "ms-python.python:conda"
-        }
-    ]
+    "python-envs.defaultEnvManager": "ms-python.python:conda",
+    "python-envs.defaultPackageManager": "ms-python.python:conda"
 }

Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.11-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+WORKDIR /app
+
+COPY data_extract/requirements.txt .
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY data_extract/ ./data_extract/
+
+CMD ["python", "-m", "data_extract.run_extract"]