unittest.yml

on:
  pull_request:
    branches:
      - main
      - preview
  # Trigger workflow on GitHub merge queue events
  # See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#merge_group
  merge_group:
    types: [checks_requested]
name: unittest

permissions:
  contents: read

# Configurable global environment variables for batching
env:
  BATCH_SIZE: 10
  TEST_ALL_PACKAGES: "true" # Set to "false" to only run tests for packages with a git diff

jobs:
  # Dynamic package discovery job to calculate required matrix size automatically
  discover-packages:
    runs-on: ubuntu-latest
    outputs:
      batch-indices: ${{ steps.set-matrix.outputs.indices }}
    steps:
    - name: Checkout
      uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
      with:
        persist-credentials: false
    - name: Generate Batch Indices
      id: set-matrix
      run: |
        # Testing all monorepo packages sequentially on a single runner node is 
        # too slow and creates a severe CI bottleneck as the repository expands.
        #
        # To scale efficiently, we chunk the workload into parallel batch slices.
        # Instead of using a fixed, hardcoded matrix array (which risks silently 
        # skipping newly added packages if the repository outgrows the array size), 
        # this step dynamically audits the 'packages/' directory at runtime.
        #
        # It calculates exactly how many concurrent runners are required based on
        # the current repository size and the configured BATCH_SIZE variable, 
        # ensuring 100% test coverage with zero manual YAML maintenance.

        # 1. Count the number of total directories matching the 'packages/*' pattern.
        #    Redirect stderr to /dev/null so empty repos do not print unneeded errors.
        TOTAL_PACKAGES=$(ls -d packages/*/ 2>/dev/null | wc -l | tr -d ' ')
        
        # 2. Safety fallback: If no packages are detected, assign a single slice index [0]
        #    so subsequent matrix-dependent jobs do not break or fail validation on an empty matrix.
        if [ "$TOTAL_PACKAGES" -eq 0 ]; then
          echo "indices=[0]" >> "$GITHUB_OUTPUT"
          exit 0
        fi

        # 3. Calculate the number of batches required using ceiling division: ceil(TOTAL_PACKAGES / BATCH_SIZE).
        #    The formula ((A + B - 1) / B) ensures integer division rounds up if there's any remaining package leftover.
        #    Example: 251 packages with a batch size of 10 gives ((251 + 10 - 1) / 10) = 260 / 10 = 26 batches.
        NUM_BATCHES=$(( (TOTAL_PACKAGES + ${{ env.BATCH_SIZE }} - 1) / ${{ env.BATCH_SIZE }} ))
        
        # 4. Generate a zero-indexed sequence from 0 to (NUM_BATCHES - 1).
        #    Use jq to securely parse the raw numbers and compile them into a compacted JSON array string.
        #    Example output format: [0,1,2,3,...,25]
        INDICES=$(seq 0 $((NUM_BATCHES - 1)) | jq -R . | jq -s -c .)
        
        # 5. Output the finished JSON string to the GitHub environment outputs pipeline.
        #    This will safely feed directly into the execution matrix downstream.
        echo "indices=${INDICES}" >> "$GITHUB_OUTPUT"

  unit:
    name: "unit-run (${{ matrix.python }}, Batch ${{ matrix.batch-index }})"
    runs-on: ubuntu-22.04
    needs: discover-packages
    strategy:
      matrix:
        python: ['3.9', '3.10', "3.11", "3.12", "3.13", "3.14"]
        # Dynamically scales to fit every package perfectly without hardcoding array indices
        batch-index: ${{ fromJson(needs.discover-packages.outputs.batch-indices) }}
    steps:
    - name: Checkout
      uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
      # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base`
      # See https://github.com/googleapis/google-cloud-python/issues/12013
      # and https://github.com/actions/checkout#checkout-head.
      with:
        fetch-depth: 2
        persist-credentials: false
    - name: Setup Python
      uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
      with:
        python-version: ${{ matrix.python }}
    - name: Install nox
      run: |
        python -m pip install --upgrade setuptools pip wheel
        python -m pip install nox
    - name: Run unit tests
      env:
        COVERAGE_FILE: ${{ github.workspace }}/.coverage-${{ matrix.python }}
        # Dynamically set BUILD_TYPE to an empty string to skip the diff calculation if TEST_ALL_PACKAGES is true
        BUILD_TYPE: ${{ env.TEST_ALL_PACKAGES == 'true' && '' || 'presubmit' }}
        TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }}
        TEST_TYPE: unit
        PY_VERSION: ${{ matrix.python }}
      run: |
        # Gather all packages in alphabetical order
        ALL_PACKAGES=($(ls -d packages/*/ | sort))
        TOTAL_PACKAGES=${#ALL_PACKAGES[@]}

        # Determine this runner's slice window
        START_INDEX=$(( ${{ matrix.batch-index }} * ${{ env.BATCH_SIZE }} ))
        
        if [ $START_INDEX -ge $TOTAL_PACKAGES ]; then
          exit 0
        fi

        BATCH_PACKAGES=("${ALL_PACKAGES[@]:$START_INDEX:${{ env.BATCH_SIZE }}}")

        # Strip trailing slashes to pass down directly into ci/run_conditional_tests.sh
        subdirs=("${BATCH_PACKAGES[@]%/}")

        ci/run_conditional_tests.sh "${subdirs[@]}"
    - name: Upload coverage results
      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
      with:
        # Appended batch-index to separate parallel coverage uploads cleanly
        name: coverage-artifact-${{ matrix.python }}-${{ matrix.batch-index }}
        path: .coverage-${{ matrix.python }}
        include-hidden-files: true

  cover:
    runs-on: ubuntu-latest
    needs:
      - unit
    steps:
    - name: Checkout
      uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
      # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base`
      # See https://github.com/googleapis/google-cloud-python/issues/12013
      # and https://github.com/actions/checkout#checkout-head.
      with:
        fetch-depth: 2
        persist-credentials: false
    - name: Setup Python
      uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
      with:
        python-version: "3.10"
    - name: Set number of files changes in packages directory
      id: packages
      run: |
        git diff HEAD~1 -- packages > /dev/null
        num_files_changed=$(git diff HEAD~1 -- packages | wc -l | tr -d ' ')
        echo "num_files_changed=${num_files_changed}" >> "$GITHUB_OUTPUT"
    - name: Install coverage
      if: ${{ steps.packages.outputs.num_files_changed > 0 }}
      run: |
        python -m pip install --upgrade setuptools pip wheel
        python -m pip install coverage
    - name: Download coverage results
      if: ${{ steps.packages.outputs.num_files_changed > 0 }}
      uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
      with:
        path: .coverage-results/
    - name: Report coverage results
      if: ${{ steps.packages.outputs.num_files_changed > 0 }}
      env:
        # TODO: default to 100% coverage after next gapic-generator release
        # https://github.com/googleapis/google-cloud-python/issues/17459
        DEFAULT_FAIL_UNDER: 99
      run: |
        if [ -d .coverage-results ]; then
          # Unzip any zipped coverage results
          find .coverage-results -type f -name '*.zip' -exec unzip -o {} \;

          # Find all coverage files and combine them.
          # We find files starting with .coverage (excluding .coveragerc files and templates)
          coverage_files=$(find .coverage-results . -type f -name '.coverage*' ! -name '.coveragerc*')
          if [ -n "${coverage_files}" ]; then
            coverage combine ${coverage_files}
          else
            echo "Error: No coverage files found to combine."
            exit 1
          fi

          # Find all modified packages
          modified_packages=$(git diff --name-only HEAD~1 -- packages | cut -d/ -f1,2 | sort -u)

          failed_packages=()
          passed_packages=()

          for pkg in ${modified_packages}; do
            if [ -d "${pkg}" ]; then
              echo "============================================================"
              echo "Evaluating coverage for package: ${pkg}"
              echo "============================================================"

              set +e
              pushd "${pkg}" > /dev/null
              if [ -f ".coveragerc" ]; then
                echo "Using package-specific configuration: ${pkg}/.coveragerc"
                # If fail_under is specified in the package-specific .coveragerc, coverage report
                # will automatically enforce it. Otherwise, we enforce the default.
                if grep -q "fail_under" ".coveragerc"; then
                  COVERAGE_FILE=../../.coverage coverage report --include="$PWD/**"
                else
                  echo "No fail_under specified in ${pkg}/.coveragerc, enforcing default"
                  COVERAGE_FILE=../../.coverage coverage report --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}"
                fi
              else
                echo "No .coveragerc found for ${pkg}, enforcing default"
                COVERAGE_FILE=../../.coverage coverage report --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}"
              fi
              status=$?
              popd > /dev/null
              set -e

              if [ ${status} -ne 0 ]; then
                failed_packages+=("${pkg}")
              else
                passed_packages+=("${pkg}")
              fi
            fi
          done

          echo "============================================================"
          echo "Coverage Evaluation Summary"
          echo "============================================================"
          if [ ${#passed_packages[@]} -gt 0 ]; then
            echo "Passed packages:"
            for pkg in "${passed_packages[@]}"; do
              echo "  - ${pkg}"
            done
          fi
          if [ ${#failed_packages[@]} -gt 0 ]; then
            echo "Failed packages:"
            for pkg in "${failed_packages[@]}"; do
              echo "  - ${pkg}"
            done
            exit 1
          fi
        else
          echo "Error: No coverage results were downloaded from the unit test jobs."
          echo "This usually means the unit tests did not run or failed to upload their coverage files."
          exit 1
        fi

  unittest-runtime-result:
    name: "unit (${{ matrix.python }})"
    needs: unit
    if: always()
    strategy:
      matrix:
        python: ['3.9', '3.10', "3.11", "3.12", "3.13", "3.14"]
    runs-on: ubuntu-latest
    steps:
      - name: Check unit tests results
        run: |
          UNIT_STATUS="${{ needs.unit.result }}"

          if [[ "$UNIT_STATUS" == "success" ]]; then
            echo "Python ${{ matrix.python }} tests passed."
          else
            echo "Error: Python ${{ matrix.python }} status is '$UNIT_STATUS'."
            exit 1
          fi