Feat/cal itp import #2970
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Test | |
| on: | |
| pull_request: | |
| branches: [ main ] | |
| paths-ignore: | |
| - '**.md' | |
| - "functions/**" | |
| workflow_call: | |
| inputs: | |
| SKIP_TESTS: | |
| description: The skip test parameter is useful for DEV environment deployments, not advised for QA and PROD. | |
| required: false | |
| type: boolean | |
| default: false | |
| env: | |
| python_version: '3.11' | |
| java_version: '11' # needed by setup-openapi-generator.sh | |
| liquibase_version: '4.33.0' | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-latest | |
| permissions: write-all | |
| name: Build & Setup | |
| outputs: | |
| should-run-tests: ${{ steps.set-should-run-tests.outputs.result }} | |
| test-matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set whether to run tests | |
| id: set-should-run-tests | |
| run: | | |
| if [ "$GITHUB_EVENT_NAME" != "workflow_call" ] && [ "$GITHUB_EVENT_NAME" != "workflow_dispatch" ]; then | |
| echo "result=true" >> "$GITHUB_OUTPUT" | |
| elif [[ "$INPUTS_SKIP_TESTS" == "false" ]]; then | |
| echo "result=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "result=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| env: | |
| INPUTS_SKIP_TESTS: ${{ inputs.SKIP_TESTS }} | |
| - name: Extract commit hash and version from git | |
| run: ./scripts/extract-hash-and-version.sh | |
| - name: Set up JDK ${{ env.java_version }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: ${{ env.java_version }} | |
| distribution: 'temurin' | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.python_version }} | |
| - name: Cache lint virtualenvs | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| api/venv | |
| functions-python/venv | |
| integration-tests/venv | |
| key: lint-venvs-${{ runner.os }}-${{ hashFiles('api/requirements_dev.txt', 'functions-python/requirements_dev.txt', 'integration-tests/requirements_dev.txt') }} | |
| - name: Docker Compose DB | |
| run: | | |
| docker compose --env-file ./config/.env.local up -d postgres postgres-test | |
| working-directory: ${{ github.workspace }} | |
| - name: Run lint checks | |
| if: ${{ steps.set-should-run-tests.outputs.result == 'true' }} | |
| shell: bash | |
| run: | | |
| scripts/lint-tests.sh | |
| - name: Cache Liquibase | |
| id: cache-liquibase | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.liquibase | |
| key: liquibase-${{ env.liquibase_version }} | |
| - name: Install Liquibase | |
| if: steps.cache-liquibase.outputs.cache-hit != 'true' | |
| env: | |
| LIQUIBASE_VERSION: ${{ env.liquibase_version }} | |
| run: | | |
| curl -sSL https://github.com/liquibase/liquibase/releases/download/v${LIQUIBASE_VERSION}/liquibase-${LIQUIBASE_VERSION}.tar.gz -o liquibase.tar.gz | |
| rm -rf liquibase-dist | |
| mkdir liquibase-dist | |
| tar -xzf liquibase.tar.gz -C liquibase-dist | |
| mv liquibase-dist ~/.liquibase | |
| - name: Link Liquibase | |
| run: | | |
| sudo ln -sf ~/.liquibase/liquibase /usr/local/bin/liquibase | |
| liquibase --version | |
| - name: Run Liquibase on Python functions test DB | |
| working-directory: ${{ github.workspace }}/liquibase | |
| run: | | |
| export LIQUIBASE_COMMAND_CHANGELOG_FILE="changelog.xml" | |
| export LIQUIBASE_COMMAND_URL=jdbc:postgresql://localhost:54320/MobilityDatabaseTest | |
| export LIQUIBASE_COMMAND_USERNAME=postgres | |
| export LIQUIBASE_COMMAND_PASSWORD=postgres | |
| liquibase update | |
| - name: Generate DB code | |
| run: | | |
| export USE_TEST_DB=true | |
| scripts/db-gen.sh | |
| - name: Generate API code | |
| run: | | |
| scripts/setup-openapi-generator.sh | |
| scripts/api-gen.sh | |
| - name: Generate Operations API code | |
| run: | | |
| scripts/api-operations-gen.sh | |
| - name: Set test matrix | |
| id: set-matrix | |
| run: | | |
| # Discover functions that are runnable: have a tests/ dir and a requirements.txt. | |
| # This automatically excludes test_utils (no tests/) and stubs like dataset_service (no requirements.txt). | |
| functions=$(find functions-python -maxdepth 1 -mindepth 1 -type d | while read dir; do | |
| if [ -d "$dir/tests" ] && [ -f "$dir/requirements.txt" ]; then | |
| basename "$dir" | |
| fi | |
| done | sort | jq -Rnc '[inputs | {folder: ("functions-python/" + .), group: .}]') | |
| api='[{"folder":"api","group":"api"}]' | |
| echo "matrix=$(printf '%s\n%s' "$api" "$functions" | jq -sc 'add | {include: .}')" >> "$GITHUB_OUTPUT" | |
| - name: Upload DB models | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: database_gen | |
| path: api/src/shared/database_gen/ | |
| overwrite: true | |
| - name: Upload API generated code | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: feeds_gen | |
| path: api/src/feeds_gen/ | |
| overwrite: true | |
| - name: Upload Operations API generated code | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: feeds_operations_gen | |
| path: functions-python/operations_api/src/feeds_gen/ | |
| overwrite: true | |
| test: | |
| needs: setup | |
| if: ${{ needs.setup.outputs.should-run-tests == 'true' }} | |
| runs-on: ubuntu-latest | |
| permissions: write-all | |
| name: Test (${{ matrix.group }}) | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.setup.outputs.test-matrix) }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.python_version }} | |
| - name: Download DB models | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: database_gen | |
| path: api/src/shared/database_gen/ | |
| - name: Download API generated code | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: feeds_gen | |
| path: api/src/feeds_gen/ | |
| - name: Download Operations API generated code | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: feeds_operations_gen | |
| path: functions-python/operations_api/src/feeds_gen/ | |
| - name: Docker Compose DB | |
| run: | | |
| docker compose --env-file ./config/.env.local up -d postgres-test --wait | |
| working-directory: ${{ github.workspace }} | |
| - name: Set up JDK ${{ env.java_version }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: ${{ env.java_version }} | |
| distribution: 'temurin' | |
| - name: Cache Liquibase | |
| id: cache-liquibase | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.liquibase | |
| key: liquibase-${{ env.liquibase_version }} | |
| - name: Install Liquibase | |
| if: steps.cache-liquibase.outputs.cache-hit != 'true' | |
| env: | |
| LIQUIBASE_VERSION: ${{ env.liquibase_version }} | |
| run: | | |
| curl -sSL https://github.com/liquibase/liquibase/releases/download/v${LIQUIBASE_VERSION}/liquibase-${LIQUIBASE_VERSION}.tar.gz -o liquibase.tar.gz | |
| rm -rf liquibase-dist | |
| mkdir liquibase-dist | |
| tar -xzf liquibase.tar.gz -C liquibase-dist | |
| mv liquibase-dist ~/.liquibase | |
| - name: Link Liquibase | |
| run: sudo ln -sf ~/.liquibase/liquibase /usr/local/bin/liquibase | |
| - name: Run Liquibase on test DB | |
| working-directory: ${{ github.workspace }}/liquibase | |
| run: | | |
| export LIQUIBASE_COMMAND_CHANGELOG_FILE="changelog.xml" | |
| export LIQUIBASE_COMMAND_URL=jdbc:postgresql://localhost:54320/MobilityDatabaseTest | |
| export LIQUIBASE_COMMAND_USERNAME=postgres | |
| export LIQUIBASE_COMMAND_PASSWORD=postgres | |
| liquibase update | |
| - name: Run tests | |
| shell: bash | |
| run: scripts/api-tests.sh --folder ${{ matrix.folder }} --html_report | |
| - name: Upload coverage report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage_report_${{ matrix.group }} | |
| path: scripts/coverage_reports/ | |
| overwrite: true |