coverage-check-parallel.yml

name: Parallel Code Coverage 

permissions:
  contents: read

on: [pull_request, workflow_dispatch]

jobs:
  discover-tests:
    runs-on: ubuntu-latest
    outputs:
      test-files: ${{ steps.discover.outputs.test-files }}
    steps:
      - name: Check out repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.ref || github.ref_name }}
          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}

      - name: Discover test files
        id: discover
        run: |
          # Find all test files in e2e directory and create JSON array
          TEST_FILES=$(find tests/e2e -name "test_*.py" -type f | sort | jq -R -s -c 'split("\n")[:-1]')
          echo "test-files=$TEST_FILES" >> $GITHUB_OUTPUT
          echo "Discovered test files: $TEST_FILES"

  e2e-tests:
    runs-on: ubuntu-latest
    environment: azure-prod
    needs: discover-tests
    strategy:
      matrix:
        test_file: ${{ fromJson(needs.discover-tests.outputs.test-files) }}
        mode: ["thrift", "sea"]
    env:
      DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }}
      DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }}
      DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
      DATABRICKS_CATALOG: peco
      DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }}
    steps:
      - name: Check out repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.ref || github.ref_name }}
          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}

      - name: Set up python
        id: setup-python
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true
          installer-parallel: true

      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@v4
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}

      - name: Install dependencies
        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
        run: poetry install --no-interaction --no-root

      - name: Install library
        run: poetry install --no-interaction --all-extras

      - name: Run ${{ matrix.mode }} tests for ${{ matrix.test_file }}
        run: |
          echo "Running ${{ matrix.mode }} tests for ${{ matrix.test_file }}"
          
          # Set test filter based on mode
          if [ "${{ matrix.mode }}" = "sea" ]; then
            TEST_FILTER="-k"
            TEST_EXPRESSION="extra_params1 or extra_params2"
          else
            TEST_FILTER="-k"
            TEST_EXPRESSION="extra_params0 or not extra_params"
          fi
          
          TEST_NAME=$(basename "${{ matrix.test_file }}" .py)
          COVERAGE_FILE="coverage-${TEST_NAME}-${{ matrix.mode }}.xml"
          
          poetry run pytest "${{ matrix.test_file }}" "$TEST_FILTER" "$TEST_EXPRESSION" \
            --cov=src --cov-report=xml:$COVERAGE_FILE --cov-report=term \
            -v
        continue-on-error: false

      - name: Upload coverage artifact
        uses: actions/upload-artifact@v4
        with:
          name: coverage-$(basename "${{ matrix.test_file }}" .py)-${{ matrix.mode }}
          path: |
            .coverage
            coverage-*-${{ matrix.mode }}.xml

  merge-coverage:
    runs-on: ubuntu-latest
    needs: [e2e-tests]
    steps:
      - name: Check out repository
        uses: actions/checkout@v4

      - name: Set up python
        id: setup-python
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true
          installer-parallel: true

      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@v4
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}

      - name: Install dependencies
        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
        run: poetry install --no-interaction --no-root

      - name: Install library
        run: poetry install --no-interaction --all-extras

      - name: Download all coverage artifacts
        uses: actions/download-artifact@v4
        with:
          path: coverage_files

      - name: Merge coverage
        run: |
          # Install xmllint if not available
          if ! command -v xmllint &> /dev/null; then
            sudo apt-get update && sudo apt-get install -y libxml2-utils
          fi
          
          # Copy all coverage files with unique names
          for artifact_dir in coverage_files/*/; do
            if [ -f "$artifact_dir/.coverage" ]; then
              artifact_name=$(basename "$artifact_dir")
              cp "$artifact_dir/.coverage" ".coverage.$artifact_name"
            fi
          done
          
          # Combine all coverage data
          poetry run coverage combine .coverage.*
          poetry run coverage xml
          poetry run coverage report

      - name: Report coverage percentage
        run: |
          COVERAGE_FILE="coverage.xml"
          if [ ! -f "$COVERAGE_FILE" ]; then
            echo "ERROR: Coverage file not found at $COVERAGE_FILE"
            exit 1
          fi

          COVERED=$(xmllint --xpath "string(//coverage/@lines-covered)" "$COVERAGE_FILE")
          TOTAL=$(xmllint --xpath "string(//coverage/@lines-valid)" "$COVERAGE_FILE")

          # Calculate percentage using Python for precision
          PERCENTAGE=$(python3 -c "covered=${COVERED}; total=${TOTAL}; print(round((covered/total)*100, 2))")

          echo "📊 Combined Coverage: ${PERCENTAGE}%"