integration-tests-pr.yml

# This workflow is triggered on pull requests to run the integration
# tests on the feature branch. It connects the API to the QA database through
# a tunnel before running the integration tests.
# Note: The test might fail if the QA database schema does not match the
# feature branch schema.
name: PR - Run Integration Tests

on:
  pull_request:
    branches:
      - main
    paths-ignore:
      - '**.md'
      - "web-app/**"
      - "functions/**"
      - ".github/workflows/web-*.yml"

env:
  python_version: '3.11'
  java_version: '11' # needed by setup-openapi-generator.sh
  API_URL: 'http://localhost:8080'
  liquibase_version: '4.33.0'  

jobs:
  integration-tests-pr:
    name: Integration Tests
    runs-on:
      labels: ubuntu-latest
    steps:
    - name: Checkout code
      uses: actions/checkout@v4

    - name: Authenticate to Google Cloud QA
      uses: google-github-actions/auth@v2
      with:
        credentials_json: ${{ secrets.QA_GCP_MOBILITY_FEEDS_SA_KEY }}

    - name: Extract commit hash and version from git
      run: ./scripts/extract-hash-and-version.sh

    - name: Set up Python ${{ env.python_version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ env.python_version }}

    - name: Download csv version of the database
      run: wget -O sources.csv https://bit.ly/catalogs-csv

    - name: Get full path of sources.csv
      id: getpath
      run: echo "FILE_PATH=$(realpath sources.csv)" >> $GITHUB_ENV

    - name: Set up JDK ${{ env.java_version }}
      uses: actions/setup-java@v4
      with:
        java-version: ${{ env.java_version }}
        distribution: 'temurin'

    - name: Docker Compose DB
      run: |
        docker compose --env-file ./config/.env.local up -d postgres
      working-directory: ${{ github.workspace }}

    - name: Install Liquibase
      env:
        LIQUIBASE_VERSION: ${{ env.liquibase_version }}
      run: |
        curl -sSL https://github.com/liquibase/liquibase/releases/download/v${LIQUIBASE_VERSION}/liquibase-${LIQUIBASE_VERSION}.tar.gz -o liquibase.tar.gz
        rm -rf liquibase-dist
        mkdir liquibase-dist
        tar -xzf liquibase.tar.gz -C liquibase-dist
        sudo rm -rf /usr/local/liquibase
        sudo mv liquibase-dist /usr/local/liquibase
        sudo ln -sf /usr/local/liquibase/liquibase /usr/local/bin/liquibase
        liquibase --version

    - name: Run Liquibase on API local DB
      working-directory: ${{ github.workspace }}/liquibase
      run: |
        export LIQUIBASE_COMMAND_CHANGELOG_FILE="changelog.xml"
        export LIQUIBASE_COMMAND_URL=jdbc:postgresql://localhost:5432/MobilityDatabase
        export LIQUIBASE_COMMAND_USERNAME=postgres
        export LIQUIBASE_COMMAND_PASSWORD=postgres
        export LIQUIBASE_LOG_LEVEL=FINE

        liquibase update

    - name: Generate code
      run: |
        scripts/db-gen.sh
        scripts/setup-openapi-generator.sh
        scripts/api-gen.sh

    - name: Load secrets from 1Password
      uses: 1password/load-secrets-action@v2.0.0
      with:
        export-env: true # Export loaded secrets as environment variables
      env:
        OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
        GCP_FEED_SSH_USER: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/GCP_FEED_SSH_USER/username"
        GCP_FEED_BASTION_NAME: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/GCP_FEED_BASTION_NAME/username"
        GCP_FEED_BASTION_SSH_KEY: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/GCP_FEED_BASTION_SSH_KEY/private key"

    - name: Tunnel
      run: |
        mkdir -p ~/.ssh
        echo "${{ env.GCP_FEED_BASTION_SSH_KEY }}" > ~/.ssh/id_rsa
        chmod 600 ~/.ssh/id_rsa             
        ./scripts/tunnel-create.sh -project_id ${{ vars.QA_MOBILITY_FEEDS_PROJECT_ID }} -zone ${{ vars.MOBILITY_FEEDS_REGION }}-a -instance ${{ env.GCP_FEED_BASTION_NAME }}-${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }} -target_account ${{ env.GCP_FEED_SSH_USER }} -db_instance ${{ secrets.DB_INSTANCE_NAME }} -port 5454
        sleep 10 # Wait for the tunnel to establish

    - name: Update .env.local
      run: |
        echo "FEEDS_DATABASE_URL=postgresql://${{ secrets.QA_POSTGRE_USER_NAME }}:${{ secrets.QA_POSTGRE_USER_PASSWORD }}@localhost:5454/${{ vars.QA_POSTGRE_SQL_DB_NAME }}" >> $GITHUB_ENV
        if grep -q "FEEDS_DATABASE_URL" config/.env.local; then
          sed -i 's|FEEDS_DATABASE_URL=.*|FEEDS_DATABASE_URL=${FEEDS_DATABASE_URL}|' config/.env.local
        else
          echo "FEEDS_DATABASE_URL=${FEEDS_DATABASE_URL}" >> config/.env.local
        fi

    - name: Start API
      run: |
        scripts/api-start.sh > api_logs.txt 2>&1 &  # Redirect stdout and stderr to api_logs.txt
        sleep 10 # Wait for the API to start

    - name: Health Check
      run: ./scripts/integration-tests.sh -u ${{ env.API_URL }} -f $FILE_PATH -c MetadataEndpointTests
      env:
        FILE_PATH: ${{ env.FILE_PATH }}
        DUMMY_TOKEN: DUMMY_TOKEN

    - name: Run Integration Tests
      run: ./scripts/integration-tests.sh -u ${{ env.API_URL }} -f $FILE_PATH
      env:
        FILE_PATH: ${{ env.FILE_PATH }}
        DUMMY_TOKEN: DUMMY_TOKEN
        DATASETS_LIMIT: 25 # Limit the number of datasets to test

    - name: Upload Test Logs
      if: ${{ always() }} # always upload the available logs even if the integration tests failed.
      uses: actions/upload-artifact@v4
      with:
        name: integration-tests-results
        path: |
          integration-tests/src/integration_tests_log.html
          integration-tests/src/datasets_validation.csv
          api_logs.txt