feat: implement multi-service CI/CD with Workload Identity Federation

Author: BLMgithub

.github/workflows/cd-extract.yml

@@ -0,0 +1,64 @@
+name: Deploy Data Extract Job
+
+# Only run when merged changes happen in data_extract folder
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'data_extract/**'
+
+permissions:
+  contents: 'read'
+  id-token: 'write'
+
+# Service configurations in GCP
+env:
+  REGION: us-east1
+  REPO_NAME: operations-artifacts
+  IMAGE_NAME: extractor
+  JOB_NAME: drive-extractor
+
+jobs:
+  build-and-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      # Handshake to GCP
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/auth@v2
+        with:
+          workload_identity_provider: ${{ secrets.WIF_PROVIDER }}
+          service_account: ${{ secrets.DEPLOYER_SA_EMAIL }}
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v2
+
+      # Handshake to Artifact registry
+      - name: Configure Docker Auth
+        run: gcloud auth configure-docker ${{ env.REGION }}-docker.pkg.dev --quiet
+
+
+      - name: Build and Push Docker Image
+        run: |
+          # Full image path using the Git Commit Hash (github.sha)
+          IMAGE_PATH="${{ env.REGION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/${{ env.REPO_NAME }}/${{ env.IMAGE_NAME }}:${{ github.sha }}"
+          
+          # Build from root to ensure COPY commands find the files
+          docker build -f data_extract/Dockerfile -t $IMAGE_PATH .
+          
+          # Push to Artifact Registry
+          docker push $IMAGE_PATH
+
+      
+      - name: Deploy to Cloud Run Job
+        run: |
+          IMAGE_PATH="${{ env.REGION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/${{ env.REPO_NAME }}/${{ env.IMAGE_NAME }}:${{ github.sha }}"
+          
+          gcloud run jobs deploy ${{ env.JOB_NAME }} \
+            --image $IMAGE_PATH \
+            --region ${{ env.REGION }} \
+            --service-account ${{ secrets.EXTRACTOR_SA_EMAIL }}

.github/workflows/cd-pipeline.yml

@@ -0,0 +1,64 @@
+name: Deploy Data Pipeline Job
+
+# Only run when merged changes happen in data_pipeline folder
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'data_pipeline/**'
+
+permissions:
+  contents: 'read'
+  id-token: 'write'
+
+# Service configurations in GCP
+env:
+  REGION: us-east1
+  REPO_NAME: operations-artifacts
+  IMAGE_NAME: pipeline
+  JOB_NAME: operations-pipeline
+
+jobs:
+  build-and-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      # Handshake to GCP
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/auth@v2
+        with:
+          workload_identity_provider: ${{ secrets.WIF_PROVIDER }}
+          service_account: ${{ secrets.DEPLOYER_SA_EMAIL }}
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v2
+
+      # Handshake to Artifact registry
+      - name: Configure Docker Auth
+        run: gcloud auth configure-docker ${{ env.REGION }}-docker.pkg.dev --quiet
+
+
+      - name: Build and Push Docker Image
+        run: |
+          # Full image path using the Git Commit Hash (github.sha)
+          IMAGE_PATH="${{ env.REGION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/${{ env.REPO_NAME }}/${{ env.IMAGE_NAME }}:${{ github.sha }}"
+          
+          # Build from root to ensure COPY commands find the files
+          docker build -f data_pipeline/Dockerfile -t $IMAGE_PATH .
+          
+          # Push to Artifact Registry
+          docker push $IMAGE_PATH
+
+      
+      - name: Deploy to Cloud Run Job
+        run: |
+          IMAGE_PATH="${{ env.REGION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/${{ env.REPO_NAME }}/${{ env.IMAGE_NAME }}:${{ github.sha }}"
+          
+          gcloud run jobs deploy ${{ env.JOB_NAME }} \
+            --image $IMAGE_PATH \
+            --region ${{ env.REGION }} \
+            --service-account ${{ secrets.PIPELINE_SA_EMAIL }}

.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-name: CI/CD Pipeline
+name: CI - Code Quality & Tests
 
 on:
   pull_request:
@@ -19,21 +19,21 @@ jobs:
         with:
           python-version: "3.11"
 
-      - name: Install dependencies
+      - name: Install Dependencies
         run: |
           python -m pip install --upgrade pip
           pip install -r dev-requirements.txt
 
       - name: Set PYTHONPATH
         run: echo "PYTHONPATH=$PWD" >> $GITHUB_ENV
 
-      - name: Check formatting
+      - name: Check Code Formatting
         run: black --check .
 
-      - name: Lint with ruff
+      - name: Ruff Linting
         run: ruff check .
 
-      - name: Run tests with coverage enforcement
+      - name: Run Tests with Coverage
         run: |
           pytest \
             --cov=data_pipeline \

dev-requirements.txt

@@ -1,7 +1,17 @@
+# extract
+google-api-python-client
+google-auth-httplib2
+google-auth-oauthlib
+
+httplib2
+pyparsing<3.0.0
+tzdata
+
+# pipeline
 pandas==2.1.4
 pytest==9.0.2
 pyarrow==19.0.0
 black==24.3.0
 ruff==0.0.264
-google-cloud-storage==3.9.0
+google-cloud-storage
 pytest-cov