coverage-check-parallel.yml

name: Parallel Code Coverage 

permissions:
  contents: read

on: [pull_request, workflow_dispatch]

jobs:
  discover-tests:
    runs-on: ubuntu-latest
    outputs:
      test-files: ${{ steps.discover.outputs.test-files }}
    steps:
      - name: Check out repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.ref || github.ref_name }}
          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}

      - name: Discover test files
        id: discover
        run: |
          # Find all test files in e2e directory
          TEST_FILES=$(find tests/e2e -name "test_*.py" -type f | sort | jq -R -s -c 'split("\n")[:-1]')
          echo "test-files=$TEST_FILES" >> $GITHUB_OUTPUT
          echo "Discovered test files: $TEST_FILES"

  e2e-tests:
    runs-on: ubuntu-latest
    environment: azure-prod
    needs: discover-tests
    strategy:
      matrix:
        test_file: ${{ fromJson(needs.discover-tests.outputs.test-files) }}
        mode: ["thrift", "sea"]
    env:
      DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }}
      DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }}
      DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
      DATABRICKS_CATALOG: peco
      DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }}
    steps:
      - name: Check out repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.ref || github.ref_name }}
          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}

      - name: Set up python
        id: setup-python
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true
          installer-parallel: true

      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@v4
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}

      - name: Install dependencies
        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
        run: poetry install --no-interaction --no-root

      - name: Install library
        run: poetry install --no-interaction --all-extras

      - name: Run ${{ matrix.mode }} tests for ${{ matrix.test_file }}
        run: |
          echo "Running ${{ matrix.mode }} tests for ${{ matrix.test_file }}"
          
          if [ "${{ matrix.mode }}" = "sea" ]; then
            TEST_FILTER="-k"
            TEST_EXPRESSION="extra_params1 or extra_params2"
          else
            TEST_FILTER="-k"
            TEST_EXPRESSION="extra_params0 or not extra_params"
          fi
          
          TEST_NAME=$(basename "${{ matrix.test_file }}" .py)
          COVERAGE_FILE="coverage-${TEST_NAME}-${{ matrix.mode }}.xml"
          COVERAGE_DATA="coverage-data-${TEST_NAME}-${{ matrix.mode }}"
          echo "TEST_NAME=$TEST_NAME" >> $GITHUB_ENV
          
          # Clean any previous coverage data to avoid conflicts
          rm -f .coverage*
          
          echo "Running pytest with coverage..."
          poetry run pytest "${{ matrix.test_file }}" "$TEST_FILTER" "$TEST_EXPRESSION" \
            --cov=src --cov-report=xml:$COVERAGE_FILE --cov-report=term -v || [ $? -eq 5 ]
          
          # Save coverage data with unique name
          if [ -f ".coverage" ]; then
            mv .coverage "$COVERAGE_DATA"
            echo "Coverage data saved as $COVERAGE_DATA"
          else
            # Create minimal coverage file for cases where no tests run
            poetry run coverage erase
            poetry run coverage run --source=src -m pytest --version > /dev/null 2>&1 || true
            if [ -f ".coverage" ]; then
              mv .coverage "$COVERAGE_DATA"
            else
              touch "$COVERAGE_DATA"
            fi
            echo "Created minimal coverage file as $COVERAGE_DATA"
          fi

      - name: Upload coverage artifact
        uses: actions/upload-artifact@v4
        if: always()  
        with:
          name: coverage-${{ env.TEST_NAME }}-${{ matrix.mode }}
          path: |
            coverage-data-*
            coverage-*-${{ matrix.mode }}.xml
          if-no-files-found: warn  

  merge-coverage:
    runs-on: ubuntu-latest
    needs: [e2e-tests]
    steps:
      - name: Check out repository
        uses: actions/checkout@v4

      - name: Set up python
        id: setup-python
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true
          installer-parallel: true

      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@v4
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}

      - name: Install dependencies
        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
        run: poetry install --no-interaction --no-root

      - name: Install library
        run: poetry install --no-interaction --all-extras

      - name: Download all coverage artifacts
        uses: actions/download-artifact@v4
        with:
          path: coverage_files

      - name: Merge coverage
        run: |
          # Install xmllint if not available
          if ! command -v xmllint &> /dev/null; then
            sudo apt-get update && sudo apt-get install -y libxml2-utils
          fi
          
          # Copy coverage data files from artifacts
          COVERAGE_FILES_FOUND=0
          for artifact_dir in coverage_files/*/; do
            if [ -d "$artifact_dir" ]; then
              for cov_file in "$artifact_dir"/coverage-data-*; do
                if [ -f "$cov_file" ]; then
                  cp "$cov_file" .
                  COVERAGE_FILES_FOUND=$((COVERAGE_FILES_FOUND + 1))
                fi
              done
            fi
          done

          echo "Found $COVERAGE_FILES_FOUND coverage data files"

          if [ $COVERAGE_FILES_FOUND -gt 0 ]; then
            echo "Combining coverage data..."
            
            for f in coverage-data-*; do
              if [ -f "$f" ]; then
                suffix=$(echo "$f" | sed 's/coverage-data-/.coverage./')
                mv "$f" "$suffix"
              fi
            done
            
            # Combine coverage files and generate reports
            poetry run coverage combine .coverage.* 2>/dev/null || true
            poetry run coverage xml 2>/dev/null || echo '<coverage lines-covered="0" lines-valid="1"></coverage>' > coverage.xml
            poetry run coverage report 2>/dev/null || true
          else
            echo "No coverage data files found, creating empty report"
            echo '<coverage lines-covered="0" lines-valid="1"></coverage>' > coverage.xml
          fi

      - name: Report coverage percentage
        run: |
          if [ ! -f "coverage.xml" ]; then
            echo "ERROR: Coverage file not found"
            exit 1
          fi

          COVERED=$(xmllint --xpath "string(//coverage/@lines-covered)" "coverage.xml")
          TOTAL=$(xmllint --xpath "string(//coverage/@lines-valid)" "coverage.xml")
          PERCENTAGE=$(python3 -c "covered=${COVERED}; total=${TOTAL}; print(round((covered/total)*100, 2))")

          echo "Combined Coverage: ${PERCENTAGE}%"