refactor: implement polars engine and decouple CI/CD infrastructure #8
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy Data Extract Job | |
| # Only run when merged changes happen in data_extract folder | |
| on: | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - 'data_extract/**' | |
| - '.github/workflows/cd-extract.yml' | |
| # Manual run | |
| workflow_dispatch: | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| # Service configurations in GCP | |
| env: | |
| REGION: us-east1 | |
| REPO_NAME: operations-artifacts-dev | |
| IMAGE_NAME: extractor | |
| JOB_NAME: drive-extractor-dev | |
| jobs: | |
| # Call CI-Infra | |
| infrastructure-check: | |
| uses: ./.github/workflows/ci-infra.yml | |
| secrets: inherit | |
| build-and-deploy: | |
| needs: infrastructure-check | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v4 | |
| # Handshake to GCP | |
| - name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| workload_identity_provider: ${{ secrets.WIF_PROVIDER }} | |
| service_account: ${{ secrets.DEPLOYER_SA_EMAIL }} | |
| # Handshake to Artifact registry | |
| - name: Configure Docker Auth | |
| run: gcloud auth configure-docker ${{ env.REGION }}-docker.pkg.dev --quiet | |
| - name: Build and Push Docker Image | |
| run: | | |
| IMAGE_PATH="${{ env.REGION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/${{ env.REPO_NAME }}/${{ env.IMAGE_NAME }}:${{ github.sha }}" | |
| docker build -f data_extract/Dockerfile -t $IMAGE_PATH . | |
| docker push $IMAGE_PATH | |
| - name: Update Cloud Run Job | |
| run: | | |
| IMAGE_PATH="${{ env.REGION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/${{ env.REPO_NAME }}/${{ env.IMAGE_NAME }}:${{ github.sha }}" | |
| gcloud run jobs update ${{ env.JOB_NAME }} \ | |
| --image $IMAGE_PATH \ | |
| --region ${{ env.REGION }} |