CD - Data Pipeline #2
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy Data Pipeline Job | |
| # Only run when merged changes happen in data_pipeline folder | |
| on: | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - 'data_pipeline/**' | |
| - '.github/workflows/deploy-pipeline.yml' | |
| # Manual run | |
| workflow_dispatch: | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| # Service configurations in GCP | |
| env: | |
| REGION: us-east1 | |
| REPO_NAME: operations-artifacts | |
| IMAGE_NAME: pipeline | |
| JOB_NAME: operations-pipeline | |
| jobs: | |
| build-and-deploy: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v4 | |
| # Handshake to GCP | |
| - name: Authenticate to Google Cloud | |
| id: auth | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| workload_identity_provider: ${{ secrets.WIF_PROVIDER }} | |
| service_account: ${{ secrets.DEPLOYER_SA_EMAIL }} | |
| - name: Set up Cloud SDK | |
| uses: google-github-actions/setup-gcloud@v2 | |
| # Handshake to Artifact registry | |
| - name: Configure Docker Auth | |
| run: gcloud auth configure-docker ${{ env.REGION }}-docker.pkg.dev --quiet | |
| - name: Build and Push Docker Image | |
| run: | | |
| # Full image path using the Git Commit Hash (github.sha) | |
| IMAGE_PATH="${{ env.REGION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/${{ env.REPO_NAME }}/${{ env.IMAGE_NAME }}:${{ github.sha }}" | |
| # Build from root to ensure COPY commands find the files | |
| docker build -f data_pipeline/Dockerfile -t $IMAGE_PATH . | |
| # Push to Artifact Registry | |
| docker push $IMAGE_PATH | |
| - name: Deploy to Cloud Run Job | |
| run: | | |
| IMAGE_PATH="${{ env.REGION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/${{ env.REPO_NAME }}/${{ env.IMAGE_NAME }}:${{ github.sha }}" | |
| gcloud run jobs deploy ${{ env.JOB_NAME }} \ | |
| --image $IMAGE_PATH \ | |
| --region ${{ env.REGION }} \ | |
| --service-account ${{ secrets.PIPELINE_SA_EMAIL }} | |
| --memory 4Gi \ | |
| --cpu 2 \ | |
| --task-timeout 30m \ | |
| --max-retries 0 |