integration-test.yml

name: Integration test

on:
  schedule:
    - cron: "0 0 * * *"  # Run daily at midnight UTC
  push:
    branches: [main]
  pull_request:
    branches: [main]

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  test:
    strategy:
      fail-fast: false
      matrix:
        python: ["3.12", "3.14"]
        is_pre: [true, false]
        package:
          # extras should be a comma seperated list of strings, like `extras: "lazy,accelerated"`
          # The 'test' extra is always installed
          - {name: "mudata", extras: ""}
          - {name: "spatialdata", extras: ""}
          - {name: "scirpy", extras: ""}
          - {name: "muon", extras: ""}
          - {name: "scanpy", extras: ""}
          - {name: "squidpy", extras: ""}
          - {name: "scvi-tools", extras: ""}
          - {name: "pertpy", extras: "de"}
          - {name: "decoupler", extras: ""}
          - {name: "SnapATAC2", extras: ""}
          - {name: "rapids-singlecell", extras: "rapids-cu12", gpu: true}
        exclude:
          - { python: 3.12, is_pre: true }

    defaults:
      run:
        shell: bash -el {0}

    runs-on: ${{ matrix.package.gpu && format('cirun-aws-gpu--{0}', github.run_id) || 'ubuntu-latest' }}

    steps:
      
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          filter: blob:none
          path: integration-testing

      - uses: actions/checkout@v4
        with:
          repository: scverse/${{ matrix.package.name }}
          fetch-depth: 0
          filter: blob:none
          path: ${{ matrix.package.name }}
  
      - name: Nvidia SMI sanity check
        if: matrix.package.gpu
        run: nvidia-smi

      # https://github.com/scverse/gpu-ci-config/issues/5
      - name: Add CUDA to PATH
        if: matrix.package.gpu
        run: |
          echo "/usr/local/cuda/bin" >> $GITHUB_PATH
          echo "LD_LIBRARY_PATH=/usr/local/cuda/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" >> $GITHUB_ENV

      - name: Install Python
        uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python }}

      - name: Install UV
        uses: hynek/setup-cached-uv@v2
      
      # For snapatac2
      - name: Install rust-cache
        uses: Swatinem/rust-cache@v2

      - name: Install toml parsers
        run: |
          uv pip install --system toml-cli

      - name: Install AnnData and scverse package
        env:
          UV_EXTRA_INDEX_URL: ${{ matrix.package.gpu && 'https://pypi.nvidia.com' || '' }}
        run: |
          EXTRAS=""
          GROUP=""
          if toml get 'dependency-groups.test' --toml-path pyproject.toml > /dev/null; then
            GROUP="--group test"
            if [ -n "${{ matrix.package.extras }}" ]; then
              EXTRAS="${{ matrix.package.extras }}"
            fi
          else
            if [ -n "${{ matrix.package.extras }}" ]; then
              EXTRAS="test,${{ matrix.package.extras }}"
            else
              EXTRAS="test"
            fi
          fi
          uv pip install ${{ matrix.is_pre && '--prerelease allow' ||  '' }} --compile --system ".[$EXTRAS]" git+https://github.com/scverse/anndata -c ../integration-testing/constraints.txt ${{ matrix.is_pre && '--override ../integration-testing/overrides.txt' || ''}} $GROUP -v
        working-directory: ${{ matrix.package.name }}
      
      - name: Set failure type for install
        if: failure()
        run: |
          echo "Installation failed for ${{ matrix.package.name }}"
          echo "failure_type=install" >> $GITHUB_ENV

      - name: Env list
        run: uv pip freeze

      - name: Run test
        env:
          DISPLAY: :42
          COLUMNS: 120
        run: |
          pytest
        working-directory: ${{ matrix.package.name }}

      - name: Set failure type for test
        if: failure() && env.failure_type != 'install'
        run: |
          echo "Test failed for ${{ matrix.package.name }}"
          echo "failure_type=test" >> $GITHUB_ENV

      - name: Generate tokens
        id: app-token
        if: always()
        uses: actions/create-github-app-token@v2
        with:
          app-id: ${{ vars.ISSUE_CREATOR_APP_ID }}
          private-key: ${{ secrets.ISSUE_CREATOR_PRIVATE_KEY }}
          owner: scverse

      # https://github.com/scverse/gpu-ci-config/issues/4
      - name: Install GitHub CLI
        if: matrix.package.gpu
        uses: sersoft-gmbh/setup-gh-cli-action@v2
        with:
          version: stable

      - name: Check for open failure issue
        if: failure()
        id: find_issue
        env:
          GH_TOKEN: ${{ steps.app-token.outputs.token }}
        run: |
          ISSUE_TITLE="Integration Testing CI ${failure_type^} Failure on python ${{ matrix.python }}${{matrix.is_pre && ' with prerelease dependencies' || ''}}"
          echo "issue_title=$ISSUE_TITLE" >> $GITHUB_ENV
          echo "Checking for existing issue: $ISSUE_TITLE"
          ISSUE=$(gh issue list --repo scverse/${{ matrix.package.name }} --state open --search "${ISSUE_TITLE}" --json number,createdAt,title | jq --arg title "$ISSUE_TITLE" '[.[] | select(.title == $title)] | .[0] // empty')
          if [[ -n "$ISSUE" ]]; then
            ISSUE_NUMBER=$(echo "$ISSUE" | jq -r '.number')
            CREATED_AT=$(echo "$ISSUE" | jq -r '.createdAt')
            CREATED_EPOCH=$(date -d "$CREATED_AT" +%s)
            NOW_EPOCH=$(date +%s)
            AGE_DAYS=$(( (NOW_EPOCH - CREATED_EPOCH) / 86400 ))
            echo "Found issue #$ISSUE_NUMBER, age: $AGE_DAYS days."
            echo "issue_exists=true" >> $GITHUB_OUTPUT
            echo "issue_number=$ISSUE_NUMBER" >> $GITHUB_OUTPUT
            echo "issue_age_days=$AGE_DAYS" >> $GITHUB_OUTPUT
          else
            echo "No open failure issue found."
            echo "issue_exists=false" >> $GITHUB_OUTPUT
          fi

      - name: Close stale failure issue
        if: failure() && steps.find_issue.outputs.issue_exists == 'true' && !matrix.is_pre && github.event_name == 'schedule'
        id: close_stale
        env:
          GH_TOKEN: ${{ steps.app-token.outputs.token }}
        run: |
          if [[ "${{ steps.find_issue.outputs.issue_age_days }}" -gt 7 ]]; then
            echo "Issue #${{ steps.find_issue.outputs.issue_number }} is ${{ steps.find_issue.outputs.issue_age_days }} days old, closing it to open a fresh one."
            # https://github.com/cli/cli/issues/8374 got graphql error about comment, could be a bug.
            # TODO: Add a comment for the closed issue.
            gh issue close "${{ steps.find_issue.outputs.issue_number }}" --repo scverse/${{ matrix.package.name }} --comment "Closing stale issue after ${{ steps.find_issue.outputs.issue_age_days }} days. A fresh issue will be opened."
            echo "issue_closed=true" >> $GITHUB_OUTPUT
          else
            echo "Issue #${{ steps.find_issue.outputs.issue_number }} is less than a week old, leaving it open."
            echo "issue_closed=false" >> $GITHUB_OUTPUT
          fi

      - name: Report failure issue
        if: failure() && (steps.find_issue.outputs.issue_exists == 'false' || steps.close_stale.outputs.issue_closed == 'true') && github.event_name == 'schedule'
        env:
          GH_TOKEN: ${{ steps.app-token.outputs.token }}
        run: |
          RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
          ISSUE_BODY="The daily CI failed on ${failure_type} for ${{ matrix.package.name }} failed.  Please go to [the logs of the integration testing repo](${RUN_URL}) to review. @scverse/anndata"
          gh issue create --repo scverse/${{ matrix.package.name }} --title "${{ env.issue_title }}" --body "${ISSUE_BODY}"

      - name: Close resolved failure issue
        if: success() && github.event_name == 'schedule'
        env:
          GH_TOKEN: ${{ steps.app-token.outputs.token }}
        run: |
          for FAILURE_TYPE in install test; do
            ISSUE_TITLE="Integration Testing CI ${FAILURE_TYPE^} Failure on python ${{ matrix.python }}${{matrix.is_pre && ' with prerelease dependencies' || ''}}"
            ISSUE=$(gh issue list --repo scverse/${{ matrix.package.name }} --state open --search "${ISSUE_TITLE}" --json number,title | jq --arg title "$ISSUE_TITLE" '[.[] | select(.title == $title)] | .[0] // empty')
            if [[ -n "$ISSUE" ]]; then
              ISSUE_NUMBER=$(echo "$ISSUE" | jq -r '.number')
              echo "Tests passing — closing issue #$ISSUE_NUMBER: $ISSUE_TITLE"
              # https://github.com/cli/cli/issues/8374 got graphql error about comment, could be a bug.
              # TODO: Add a comment for the closed issue.
              gh issue close "$ISSUE_NUMBER" --repo scverse/${{ matrix.package.name }} --comment "Closing issue as tests are now passing."
            fi
          done

  keepalive-job:
    name: Keepalive Workflow
    runs-on: ubuntu-latest
    permissions:
      actions: write
    steps:
      - name: Re-enable workflow
        env:
          GITHUB_TOKEN: ${{ inputs.GITHUB_TOKEN || github.token }}
        shell: sh
        run: |
          gh api --verbose -X PUT "repos/${GITHUB_REPOSITORY}/actions/workflows/integration-test.yml/enable"