coverage-check-parallel.yml

name: Parallel Code Coverage 

permissions:
  contents: read

on: [pull_request, workflow_dispatch]

jobs:
  discover-tests:
    runs-on: ubuntu-latest
    outputs:
      test-files: ${{ steps.discover.outputs.test-files }}
    steps:
      - name: Check out repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.ref || github.ref_name }}
          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}

      - name: Discover test files
        id: discover
        run: |
          # Find all test files in e2e directory and create JSON array (excluding test_driver.py and test_parameterized_queries.py for now)
          TEST_FILES=$(find tests/e2e -name "test_*.py" -type f | grep -v -E "(test_driver\.py|test_parameterized_queries\.py)" | sort | jq -R -s -c 'split("\n")[:-1]')
          echo "test-files=$TEST_FILES" >> $GITHUB_OUTPUT
          echo "Discovered test files: $TEST_FILES"

  e2e-tests:
    runs-on: ubuntu-latest
    environment: azure-prod
    needs: discover-tests
    strategy:
      matrix:
        test_file: ${{ fromJson(needs.discover-tests.outputs.test-files) }}
        mode: ["thrift", "sea"]
    env:
      DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }}
      DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }}
      DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
      DATABRICKS_CATALOG: peco
      DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }}
    steps:
      - name: Check out repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.ref || github.ref_name }}
          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}

      - name: Set up python
        id: setup-python
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true
          installer-parallel: true

      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@v4
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}

      - name: Install dependencies
        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
        run: poetry install --no-interaction --no-root

      - name: Install library
        run: poetry install --no-interaction --all-extras

      - name: Run ${{ matrix.mode }} tests for ${{ matrix.test_file }}
        run: |
          echo "Running ${{ matrix.mode }} tests for ${{ matrix.test_file }}"
          
          if [ "${{ matrix.mode }}" = "sea" ]; then
            TEST_FILTER="-k"
            TEST_EXPRESSION="extra_params1 or extra_params2"
          else
            TEST_FILTER="-k"
            TEST_EXPRESSION="extra_params0 or not extra_params"
          fi
          
          TEST_NAME=$(basename "${{ matrix.test_file }}" .py)
          COVERAGE_FILE="coverage-${TEST_NAME}-${{ matrix.mode }}.xml"
          COVERAGE_DATA=".coverage-${TEST_NAME}-${{ matrix.mode }}"
          echo "TEST_NAME=$TEST_NAME" >> $GITHUB_ENV
          
          # Clean any previous coverage data to avoid conflicts
          rm -f .coverage*
          
          echo "🧪 Running pytest with coverage..."
          poetry run pytest "${{ matrix.test_file }}" "$TEST_FILTER" "$TEST_EXPRESSION" \
            --cov=src --cov-report=xml:$COVERAGE_FILE --cov-report=term -v || [ $? -eq 5 ]
          
          echo "🔍 DEBUG: Post-test file analysis..."
          echo "Current directory contents:"
          ls -la
          
          echo "Coverage-related files:"
          ls -la .coverage* coverage-* 2>/dev/null || echo "No coverage files found"
          
          if [ -f ".coverage" ]; then
            COVERAGE_SIZE=$(stat -c%s .coverage 2>/dev/null || stat -f%z .coverage)
            echo "✅ .coverage file found (${COVERAGE_SIZE} bytes)"
            mv .coverage "$COVERAGE_DATA"
            echo "✅ Saved coverage data as $COVERAGE_DATA"
          else
            echo "⚠️ No .coverage generated, creating minimal coverage file"
            # Create a minimal but valid coverage file using coverage.py
            poetry run coverage erase
            poetry run coverage run --source=src -m pytest --version > /dev/null 2>&1 || true
            if [ -f ".coverage" ]; then
              mv .coverage "$COVERAGE_DATA"
              echo "✅ Created minimal coverage file as $COVERAGE_DATA"
            else
              # Fallback: create empty file (will be handled gracefully in merge)
              touch "$COVERAGE_DATA"
              echo "⚠️ Created empty placeholder as $COVERAGE_DATA"
            fi
          fi
          
          if [ -f "$COVERAGE_FILE" ]; then
            XML_SIZE=$(stat -c%s "$COVERAGE_FILE" 2>/dev/null || stat -f%z "$COVERAGE_FILE")
            echo "✅ XML coverage file found: $COVERAGE_FILE (${XML_SIZE} bytes)"
            echo "XML preview:"
            head -3 "$COVERAGE_FILE" 2>/dev/null || echo "Cannot read XML file"
          else
            echo "❌ No XML coverage file generated: $COVERAGE_FILE"
          fi
          
          echo "📁 Final files available for upload:"
          ls -la .coverage* coverage-* 2>/dev/null || echo "No coverage files found"
          
          echo "🔍 Specifically checking for files to upload:"
          echo "  .coverage-* files:"
          ls -la .coverage-* 2>/dev/null || echo "    None found"
          echo "  coverage-*-${{ matrix.mode }}.xml files:"
          ls -la coverage-*-${{ matrix.mode }}.xml 2>/dev/null || echo "    None found"
          
          echo "🔍 CRITICAL DEBUG: Checking exact upload patterns:"
          echo "  Pattern '.coverage-*' matches:"
          find . -maxdepth 1 -name ".coverage-*" -type f 2>/dev/null || echo "    No matches"
          echo "  Pattern 'coverage-*-${{ matrix.mode }}.xml' matches:"
          find . -maxdepth 1 -name "coverage-*-${{ matrix.mode }}.xml" -type f 2>/dev/null || echo "    No matches"
          
          echo "🔍 Working directory: $(pwd)"
          echo "🔍 All files in current directory:"
          ls -la

      - name: Upload coverage artifact
        uses: actions/upload-artifact@v4
        if: always()  
        with:
          name: coverage-${{ env.TEST_NAME }}-${{ matrix.mode }}
          path: |
            .coverage-*
            coverage-*-${{ matrix.mode }}.xml
          if-no-files-found: warn  

  merge-coverage:
    runs-on: ubuntu-latest
    needs: [e2e-tests]
    steps:
      - name: Check out repository
        uses: actions/checkout@v4

      - name: Set up python
        id: setup-python
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true
          installer-parallel: true

      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@v4
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}

      - name: Install dependencies
        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
        run: poetry install --no-interaction --no-root

      - name: Install library
        run: poetry install --no-interaction --all-extras

      - name: Download all coverage artifacts
        uses: actions/download-artifact@v4
        with:
          path: coverage_files

      - name: Merge coverage
        run: |
          echo "🔧 Installing xmllint..."
          if ! command -v xmllint &> /dev/null; then
            sudo apt-get update && sudo apt-get install -y libxml2-utils
          fi
          
          echo "📁 DEBUG: Checking downloaded artifacts structure..."
          find coverage_files -type f -name "*" | head -20
          echo "Total files found: $(find coverage_files -type f | wc -l)"
          
          echo "📁 Checking for coverage files in artifacts..."
          COVERAGE_FILES_FOUND=0
          for artifact_dir in coverage_files/*/; do
            if [ -d "$artifact_dir" ]; then
              echo "🔍 Artifact: $(basename "$artifact_dir")"
              echo "  Contents:"
              ls -la "$artifact_dir" || echo "  (empty or inaccessible)"
              
              # Copy .coverage files
              for cov_file in "$artifact_dir"/.coverage-*; do
                if [ -f "$cov_file" ]; then
                  cp "$cov_file" .
                  COVERAGE_FILES_FOUND=$((COVERAGE_FILES_FOUND + 1))
                  echo "  ✅ Copied $(basename "$cov_file") ($(stat -c%s "$cov_file" 2>/dev/null || stat -f%z "$cov_file") bytes)"
                fi
              done
              
              # Copy XML files for debugging
              for xml_file in "$artifact_dir"/coverage-*.xml; do
                if [ -f "$xml_file" ]; then
                  cp "$xml_file" .
                  echo "  📄 Copied $(basename "$xml_file") ($(stat -c%s "$xml_file" 2>/dev/null || stat -f%z "$xml_file") bytes)"
                fi
              done
            fi
          done

          echo "📊 SUMMARY: Found $COVERAGE_FILES_FOUND .coverage files"
          echo "Available .coverage files for merging:"
          ls -la .coverage-* 2>/dev/null || echo "❌ No .coverage-* files found"
          
          echo "Available XML files:"
          ls -la coverage-*.xml 2>/dev/null || echo "❌ No XML files found"

          if [ $COVERAGE_FILES_FOUND -gt 0 ]; then
            echo "🔄 Combining coverage data..."
            poetry run coverage combine .coverage-* || {
              echo "❌ Coverage combine failed, checking individual files:"
              for f in .coverage-*; do
                if [ -f "$f" ]; then
                  echo "  File: $f ($(stat -c%s "$f" 2>/dev/null || stat -f%z "$f") bytes)"
                  file "$f" 2>/dev/null || echo "    Cannot determine file type"
                fi
              done
              echo "⚠️ Using fallback approach..."
              touch .coverage
            }
            
            echo "📊 Generating XML report..."
            poetry run coverage xml || {
              echo "❌ XML generation failed, using fallback"
              echo '<coverage lines-covered="0" lines-valid="1"></coverage>' > coverage.xml
            }
            
            echo "📋 Generating text report..."
            poetry run coverage report || echo "⚠️ Text report failed"
          else
            echo "⚠️ No coverage data files found, creating empty report"
            echo '<coverage lines-covered="0" lines-valid="1"></coverage>' > coverage.xml
          fi
          
          echo "📄 Final coverage.xml preview:"
          head -5 coverage.xml 2>/dev/null || echo "Cannot read coverage.xml"

      - name: Report coverage percentage
        run: |
          COVERAGE_FILE="coverage.xml"
          if [ ! -f "$COVERAGE_FILE" ]; then
            echo "ERROR: Coverage file not found at $COVERAGE_FILE"
            exit 1
          fi

          COVERED=$(xmllint --xpath "string(//coverage/@lines-covered)" "$COVERAGE_FILE")
          TOTAL=$(xmllint --xpath "string(//coverage/@lines-valid)" "$COVERAGE_FILE")

          PERCENTAGE=$(python3 -c "covered=${COVERED}; total=${TOTAL}; print(round((covered/total)*100, 2))")

          echo "📊 Combined Coverage: ${PERCENTAGE}%"