api-deployer.yml

# Deploys the Feeds API to an environment defined by the workflow inputs.
name: Terraform Deployment
on:
  workflow_call:
    secrets:
      GCP_MOBILITY_FEEDS_SA_KEY:
        description: Service account key
        required: true
      OAUTH2_CLIENT_ID:
        description: Oauth client id part of the the Identity Aware Proxy configuration
        required: true
      OAUTH2_CLIENT_SECRET:
        description: Oauth client secret part of the the Identity Aware Proxy configuration
        required: true
      OP_SERVICE_ACCOUNT_TOKEN:
        description: 1Password service account token
        required: true
    inputs:
      ENVIRONMENT:
        description: API environment. Possible values prod, staging and dev
        required: true
        type: string
      BUCKET_NAME:
        description: Bucket name where terraform state is persisted
        required: true
        type: string
      OBJECT_PREFIX:
        description: Storage object prefix where terraform state is persisted
        required: true
        type: string
      PROJECT_ID:
        description: GCP project ID
        required: true
        type: string
      REGION:
        description: GCP region
        required: true
        type: string
      DEPLOYER_SERVICE_ACCOUNT:
        description: Service account used to deploy resources
        required: true
        type: string
      FEED_API_IMAGE_VERSION:
        description: Docker image version to deploy
        required: true
        type: string
      TF_APPLY:
        description: Terraform apply changes
        required: true
        type: boolean
      GLOBAL_RATE_LIMIT_REQ_PER_MINUTE:
        description: Global Load Balancer rate limit per minute
        required: true
        type: string
      VALIDATOR_ENDPOINT:
        description: Validator endpoint
        required: true
        type: string
      OPERATIONS_OAUTH2_CLIENT_ID_1PASSWORD:
        description: Oauth client id part of the authorization for the operations API
        required: true
        type: string
      WEB_APP_REVALIDATE_URL:
        description: URL of the web app revalidation endpoint
        required: false
        type: string
      WEB_APP_REVALIDATE_SECRET_1PASSWORD:
        description: 1Password reference for the web app revalidation secret token
        required: false
        type: string
      SKIP_TESTS:
        description: The skip test parameter is useful for DEV environment deployments, not advised for QA and PROD.
        required: true
        type: boolean

env:
  python_version: '3.11'
  java_version: '11' # needed by setup-openapi-generator.sh
  local_postgres_db: postgres_db
  local_postgres_user: postgres_user
  local_postgres_pwd: postgres_password
  local_postgres_port: 5432
  ARTIFACT_REPO_NAME: feeds

jobs:
  api-build-test:
    uses: ./.github/workflows/build-test.yml
    name: Build & Test
    with:
      SKIP_TESTS: ${{ inputs.SKIP_TESTS }}

  create-artifact-repo:
    runs-on: ubuntu-latest
    permissions: write-all
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: ${{ env.python_version }}

      - name: Authenticate to Google Cloud
        id: gcloud_auth
        uses: google-github-actions/auth@v2
        with:
          credentials_json: ${{ secrets.GCP_MOBILITY_FEEDS_SA_KEY }}

      - name: GCloud Setup
        uses: google-github-actions/setup-gcloud@v2

      - name: Set Variables
        run: |
          echo "Setting variables"
          echo "BUCKET_NAME=${{ inputs.BUCKET_NAME }}" >> $GITHUB_ENV
          echo "OBJECT_PREFIX=${{ inputs.OBJECT_PREFIX }}-artifact" >> $GITHUB_ENV
          echo "PROJECT_ID=${{ inputs.PROJECT_ID }}" >> $GITHUB_ENV
          echo "REGION=${{ inputs.REGION }}" >> $GITHUB_ENV
          echo "ENVIRONMENT=${{ inputs.ENVIRONMENT }}" >> $GITHUB_ENV

      - name: Populate Variables
        run: |
          scripts/replace-variables.sh -in_file infra/backend.conf.rename_me -out_file infra/artifact-registry/backend.conf -variables BUCKET_NAME,OBJECT_PREFIX
          scripts/replace-variables.sh -in_file infra/artifact-registry/vars.tfvars.rename_me -out_file infra/artifact-registry/vars.tfvars -variables PROJECT_ID,REGION,ENVIRONMENT,ARTIFACT_REPO_NAME

      - uses: hashicorp/setup-terraform@v3
        with:
          terraform_version: 1.5.3
          terraform_wrapper: false

      - name: Terraform Init
        run: |
          cd infra/artifact-registry
          terraform init -backend-config=backend.conf

      - name: Terraform Plan
        id: plan
        run: |
          cd infra/artifact-registry
          terraform plan -var-file=vars.tfvars -out=tf.plan
          terraform show -no-color tf.plan > terraform-plan.txt
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Terraform Apply
        if: ${{ inputs.TF_APPLY }}
        run: |
          cd infra/artifact-registry
          terraform apply -auto-approve tf.plan
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Persist TF plan
        uses: actions/upload-artifact@v4
        with:
          name: terraform-artifact-plan.txt
          path: infra/artifact-registry/terraform-plan.txt
          overwrite: true

  docker-build-publish:
    # Add docker healthy test
    runs-on: ubuntu-latest
    permissions: write-all
    needs: [create-artifact-repo, api-build-test]
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Extract commit hash and version from git
        run: ./scripts/extract-hash-and-version.sh

      - name: Upload version_info to workflow artefacts
        uses: actions/upload-artifact@v4
        with:
          name: version_info
          path: api/src/version_info
          compression-level: 0

      - name: Authenticate to Google Cloud
        id: gcloud_auth
        uses: google-github-actions/auth@v2
        with:
          credentials_json: ${{ secrets.GCP_MOBILITY_FEEDS_SA_KEY }}

      - name: Login to Google Artifact Registry
        uses: docker/login-action@v3
        with:
          registry: ${{inputs.REGION}}-docker.pkg.dev
          username: _json_key_base64
          password: ${{ secrets.GCP_MOBILITY_FEEDS_SA_KEY }}

      - name: Set up JDK ${{ env.java_version }}
        uses: actions/setup-java@v4
        with:
          java-version: ${{ env.java_version }}
          distribution: 'temurin'

      - uses: actions/setup-python@v5
        with:
          python-version: ${{ env.python_version }}

      - name: Update .env file
        run: |
          echo "POSTGRES_USER=${{ env.local_postgres_user }}" > config/.env.local
          echo "PGUSER=${{ env.local_postgres_user }}" >> config/.env.local
          echo "POSTGRES_PASSWORD=${{ env.local_postgres_pwd }}" >> config/.env.local
          echo "POSTGRES_DB=${{ env.local_postgres_db }}" >> config/.env.local
          echo "POSTGRES_PORT=${{ env.local_postgres_port }}" >> config/.env.local
          echo "POSTGRES_HOST=localhost" >> config/.env.local
          echo "ENV=dev" >> config/.env.local

      # db models were generated and uploaded in api-build-test job above.
      - uses: actions/download-artifact@v4
        with:
          name: database_gen
          path: api/src/shared/database_gen/

      # api schema was generated and uploaded in api-build-test job above.
      - uses: actions/download-artifact@v4
        with:
          name: feeds_gen
          path: api/src/feeds_gen/

      - name: Set Variables
        id: set_variables
        run: |
          echo "Setting variables"
          echo "PROJECT_ID=${{ inputs.PROJECT_ID }}" >> $GITHUB_ENV
          echo "REGION=${{ inputs.REGION }}" >> $GITHUB_ENV
          echo "ENVIRONMENT=${{ inputs.ENVIRONMENT }}" >> $GITHUB_ENV
          echo "FEED_API_IMAGE_VERSION=${{ inputs.FEED_API_IMAGE_VERSION }}" >> $GITHUB_ENV

      - name: Build & Publish Docker Image
        run: |
          # We want to generate the image even if it's the same commit that has been tagged. So use the version
          # (coming from the tag) in the docker image tag (If the docker tag does not change it won't be uploaded)
          DOCKER_IMAGE_VERSION=$EXTRACTED_VERSION.$FEED_API_IMAGE_VERSION
          scripts/docker-build-push.sh -project_id $PROJECT_ID -repo_name feeds-$ENVIRONMENT -service feed-api -region $REGION -version $DOCKER_IMAGE_VERSION

  terraform-deploy:
    runs-on: ubuntu-latest
    permissions: write-all
    needs: docker-build-publish
    steps:

      - name: Checkout code
        uses: actions/checkout@v4

      - name: Extract commit hash and version from git
        run: ./scripts/extract-hash-and-version.sh

      - uses: actions/setup-python@v5
        with:
          python-version: ${{ env.python_version }}

      # db models were generated and uploaded in api-build-test job above.
      - uses: actions/download-artifact@v4
        with:
          name: database_gen
          path: api/src/shared/database_gen/

      # api schema was generated and uploaded in api-build-test job above.
      - uses: actions/download-artifact@v4
        with:
          name: feeds_gen
          path: api/src/feeds_gen/

      - uses: actions/download-artifact@v4
        with:
          name: feeds_operations_gen
          path: functions-python/operations_api/src/feeds_gen/

      - name: Build python functions
        run: |
          scripts/function-python-build.sh --all

      - name: Authenticate to Google Cloud
        id: gcloud_auth
        uses: google-github-actions/auth@v2
        with:
          credentials_json: ${{ secrets.GCP_MOBILITY_FEEDS_SA_KEY }}

      - name: GCloud Setup
        uses: google-github-actions/setup-gcloud@v2

      - name: Set Variables
        run: |
          echo "Setting variables"
          echo "BUCKET_NAME=${{ inputs.BUCKET_NAME }}" >> $GITHUB_ENV
          echo "OBJECT_PREFIX=${{ inputs.OBJECT_PREFIX }}" >> $GITHUB_ENV
          echo "PROJECT_ID=${{ inputs.PROJECT_ID }}" >> $GITHUB_ENV
          echo "REGION=${{ inputs.REGION }}" >> $GITHUB_ENV
          echo "ENVIRONMENT=${{ inputs.ENVIRONMENT }}" >> $GITHUB_ENV
          echo "DEPLOYER_SERVICE_ACCOUNT=${{ inputs.DEPLOYER_SERVICE_ACCOUNT }}" >> $GITHUB_ENV
          echo "FEED_API_IMAGE_VERSION=$EXTRACTED_VERSION.${{ inputs.FEED_API_IMAGE_VERSION }}" >> $GITHUB_ENV
          echo "OAUTH2_CLIENT_ID=${{ secrets.OAUTH2_CLIENT_ID }}" >> $GITHUB_ENV
          echo "OAUTH2_CLIENT_SECRET=${{ secrets.OAUTH2_CLIENT_SECRET }}" >> $GITHUB_ENV
          echo "GLOBAL_RATE_LIMIT_REQ_PER_MINUTE=${{ inputs.GLOBAL_RATE_LIMIT_REQ_PER_MINUTE }}" >> $GITHUB_ENV
          echo "VALIDATOR_ENDPOINT=${{ inputs.VALIDATOR_ENDPOINT }}" >> $GITHUB_ENV
          echo "WEB_APP_REVALIDATE_URL=${{ inputs.WEB_APP_REVALIDATE_URL }}" >> $GITHUB_ENV

      - name: Load secret from 1Password
        uses: 1password/load-secrets-action@v2
        with:
          export-env: true
        env:
          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
          TRANSITLAND_API_KEY: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/TansitLand API Key/credential"
          TDG_API_TOKEN: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/Transport.data.gouv.fr API Token/credential"
          OPERATIONS_OAUTH2_CLIENT_ID: ${{ inputs.OPERATIONS_OAUTH2_CLIENT_ID_1PASSWORD }}

      - name: Load web app revalidation secret from 1Password
        if: ${{ inputs.WEB_APP_REVALIDATE_SECRET_1PASSWORD != '' }}
        uses: 1password/load-secrets-action@v2
        with:
          export-env: true
        env:
          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
          WEB_APP_REVALIDATE_SECRET: ${{ inputs.WEB_APP_REVALIDATE_SECRET_1PASSWORD }}

      - name: Populate Variables
        run: |
          scripts/replace-variables.sh -in_file infra/backend.conf.rename_me -out_file infra/backend.conf -variables BUCKET_NAME,OBJECT_PREFIX
          scripts/replace-variables.sh -in_file infra/vars.tfvars.rename_me -out_file infra/vars.tfvars -variables PROJECT_ID,REGION,ENVIRONMENT,DEPLOYER_SERVICE_ACCOUNT,FEED_API_IMAGE_VERSION,OAUTH2_CLIENT_ID,OAUTH2_CLIENT_SECRET,GLOBAL_RATE_LIMIT_REQ_PER_MINUTE,ARTIFACT_REPO_NAME,VALIDATOR_ENDPOINT,TRANSITLAND_API_KEY,OPERATIONS_OAUTH2_CLIENT_ID,TDG_API_TOKEN,WEB_APP_REVALIDATE_URL,WEB_APP_REVALIDATE_SECRET

      - uses: hashicorp/setup-terraform@v3
        with:
          terraform_version: 1.5.3
          terraform_wrapper: false

      - name: Terraform Init
        run: |
          cd infra
          terraform init -backend-config=backend.conf

      - name: Terraform Plan
        id: plan
        run: |
          cd infra
          terraform plan -var-file=vars.tfvars -out=tf.plan
          terraform show -no-color tf.plan > terraform-plan.txt
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          PLAN_OUTPUT: ${{ steps.plan.outputs.stdout }}

      - name: Persist TF plan
        uses: actions/upload-artifact@v4
        with:
          name: terraform-plan.txt
          path: infra/terraform-plan.txt
          overwrite: true

      - name: Terraform Apply
        if: ${{ inputs.TF_APPLY }}
        run: |
          cd infra
          terraform apply -auto-approve tf.plan
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          PLAN_OUTPUT: ${{ steps.plan.outputs.stdout }}