datasets-batch-deployer.yml

name: GCP Batch Processing Deployer
on:
  workflow_call:
    inputs:
      STATE_BUCKET_NAME:
        description: Bucket name for the terraform state
        required: true
        type: string
      OBJECT_PREFIX:
        description: Object prefix for the terraform state
        required: true
        type: string
      PROJECT_ID:
        description: GCP Project ID
        required: true
        type: string
      REGION:
        description: GCP region
        required: true
        type: string
      DEPLOYER_SERVICE_ACCOUNT:
        description: Deployer service account
        required: true
        type: string
      JOB_SCHEDULE:
        description: Schedule for the GCP scheduler
        required: true
        type: string
      DATASETS_BUCKET_NAME:
        description: Bucket name containing the historical datasets
        required: true
        type: string
      ENVIRONMENT:
        description: Environment name
        required: true
        type: string
    secrets:
      GCP_MOBILITY_FEEDS_SA_KEY:
        description: Service account key
        required: true

env:
  python_version: '3.11'
  liquibase_version: '4.33.0'

jobs:
  terraform:
    name: 'Terraform'
    permissions: write-all
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Authenticate to Google Cloud
        id: gcloud_auth
        uses: google-github-actions/auth@v2
        with:
          credentials_json: ${{ secrets.GCP_MOBILITY_FEEDS_SA_KEY }}

      - name: Google Cloud Setup
        uses: google-github-actions/setup-gcloud@v2

      - name: Set Variables
        run: |
          echo "Setting variables"
          echo "BUCKET_NAME=${{ inputs.STATE_BUCKET_NAME }}" >> $GITHUB_ENV
          echo "OBJECT_PREFIX=${{ inputs.OBJECT_PREFIX }}" >> $GITHUB_ENV
          echo "PROJECT_ID=${{ inputs.PROJECT_ID }}" >> $GITHUB_ENV
          echo "REGION=${{ inputs.REGION }}" >> $GITHUB_ENV
          echo "ENVIRONMENT=${{ inputs.ENVIRONMENT }}" >> $GITHUB_ENV
          echo "DEPLOYER_SERVICE_ACCOUNT=${{ inputs.DEPLOYER_SERVICE_ACCOUNT }}" >> $GITHUB_ENV
          echo "JOB_SCHEDULE=${{ inputs.JOB_SCHEDULE }}" >> $GITHUB_ENV
          echo "DATASETS_BUCKET_NAME=${{ inputs.DATASETS_BUCKET_NAME }}" >> $GITHUB_ENV

      - name: Populate Variables
        run: |
          scripts/replace-variables.sh -in_file infra/backend.conf.rename_me -out_file infra/batch/backend.conf -variables BUCKET_NAME,OBJECT_PREFIX
          scripts/replace-variables.sh -in_file infra/batch/vars.tfvars.rename_me -out_file infra/batch/vars.tfvars -variables REGION,PROJECT_ID,DEPLOYER_SERVICE_ACCOUNT,JOB_SCHEDULE,ENVIRONMENT,DATASETS_BUCKET_NAME
          cat infra/batch/vars.tfvars

      - name: Docker Compose DB
        run: |
          docker compose --env-file ./config/.env.local up -d postgres
        working-directory: ${{ github.workspace }}

      - uses: actions/setup-python@v4
        with:
          python-version: ${{ env.python_version }}

      - name: Install Liquibase
        env:
          LIQUIBASE_VERSION: ${{ env.liquibase_version }}
        run: |
          curl -sSL https://github.com/liquibase/liquibase/releases/download/v${LIQUIBASE_VERSION}/liquibase-${LIQUIBASE_VERSION}.tar.gz -o liquibase.tar.gz
          rm -rf liquibase-dist
          mkdir liquibase-dist
          tar -xzf liquibase.tar.gz -C liquibase-dist
          sudo rm -rf /usr/local/liquibase
          sudo mv liquibase-dist /usr/local/liquibase
          sudo ln -sf /usr/local/liquibase/liquibase /usr/local/bin/liquibase
          liquibase --version
      
      - name: Run Liquibase on Python functions DB
        working-directory: ${{ github.workspace }}/liquibase
        run: |
          export LIQUIBASE_COMMAND_CHANGELOG_FILE="changelog.xml"
          export LIQUIBASE_COMMAND_URL=jdbc:postgresql://localhost:5432/MobilityDatabase
          export LIQUIBASE_COMMAND_USERNAME=postgres
          export LIQUIBASE_COMMAND_PASSWORD=postgres
          liquibase update
  
      - name: Generate DB code
        run: |
          scripts/db-gen.sh

      - name: Upload DB models
        uses: actions/upload-artifact@v4
        with:
          name: database_gen
          path: api/src/shared/database_gen/

      - name: Build python functions
        run: |
          scripts/function-python-build.sh --all

      - name: Install Terraform
        uses: hashicorp/setup-terraform@v3
        with:
          terraform_version: 1.5.3
          terraform_wrapper: false

      - name: Terraform Init
        run: |
          cd infra/batch
          terraform init -backend-config=backend.conf

      - name: Terraform Plan
        run: |
          cd infra/batch
          terraform plan -var-file=vars.tfvars -out=tf.plan
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Terraform Apply
        run: |
          cd infra/batch
          terraform apply -auto-approve tf.plan
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}