Integration test #660
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration test | |
| on: | |
| schedule: | |
| - cron: "0 0 * * *" # Run daily at midnight UTC | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| test: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python: ["3.12", "3.13"] | |
| is_pre: [true, false] | |
| package: | |
| # extras should be a comma seperated list of strings, like `extras: "lazy,accelerated"` | |
| # The 'test' extra is always installed | |
| - {name: "mudata", extras: ""} | |
| - {name: "spatialdata", extras: ""} | |
| - {name: "scirpy", extras: ""} | |
| - {name: "muon", extras: ""} | |
| - {name: "scanpy", extras: ""} | |
| - {name: "squidpy", extras: ""} | |
| - {name: "scvi-tools", extras: ""} | |
| - {name: "pertpy", extras: "de"} | |
| - {name: "decoupler", extras: ""} | |
| - {name: "SnapATAC2", extras: ""} | |
| - {name: "rapids-singlecell", extras: "rapids-cu12", gpu: true} | |
| exclude: | |
| - { python: 3.12, is_pre: true } | |
| defaults: | |
| run: | |
| shell: bash -el {0} | |
| runs-on: ${{ matrix.package.gpu && format('cirun-aws-gpu--{0}', github.run_id) || 'ubuntu-latest' }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| filter: blob:none | |
| path: integration-testing | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: scverse/${{ matrix.package.name }} | |
| fetch-depth: 0 | |
| filter: blob:none | |
| path: ${{ matrix.package.name }} | |
| - name: Nvidia SMI sanity check | |
| if: matrix.package.gpu | |
| run: nvidia-smi | |
| # https://github.com/scverse/gpu-ci-config/issues/5 | |
| - name: Add CUDA to PATH | |
| if: matrix.package.gpu | |
| run: | | |
| echo "/usr/local/cuda/bin" >> $GITHUB_PATH | |
| echo "LD_LIBRARY_PATH=/usr/local/cuda/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" >> $GITHUB_ENV | |
| - name: Install Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - name: Install UV | |
| uses: hynek/setup-cached-uv@v2 | |
| # For snapatac2 | |
| - name: Install rust-cache | |
| uses: Swatinem/rust-cache@v2 | |
| - name: Install toml parsers | |
| run: | | |
| uv pip install --system toml-cli | |
| - name: Install AnnData and scverse package | |
| env: | |
| UV_EXTRA_INDEX_URL: ${{ matrix.package.gpu && 'https://pypi.nvidia.com' || '' }} | |
| run: | | |
| EXTRAS="" | |
| GROUP="" | |
| if toml get 'dependency-groups.test' --toml-path pyproject.toml > /dev/null; then | |
| GROUP="--group test" | |
| if [ -n "${{ matrix.package.extras }}" ]; then | |
| EXTRAS="${{ matrix.package.extras }}" | |
| fi | |
| else | |
| if [ -n "${{ matrix.package.extras }}" ]; then | |
| EXTRAS="test,${{ matrix.package.extras }}" | |
| else | |
| EXTRAS="test" | |
| fi | |
| fi | |
| uv pip install ${{ matrix.is_pre && '--prerelease allow' || '' }} --compile --system ".[$EXTRAS]" git+https://github.com/scverse/anndata -c ../integration-testing/constraints.txt ${{ matrix.is_pre && '--override ../integration-testing/overrides.txt' || ''}} $GROUP -v | |
| working-directory: ${{ matrix.package.name }} | |
| - name: Set failure type for install | |
| if: failure() | |
| run: | | |
| echo "Installation failed for ${{ matrix.package.name }}" | |
| echo "failure_type=install" >> $GITHUB_ENV | |
| - name: Env list | |
| run: uv pip freeze | |
| - name: Run test | |
| env: | |
| DISPLAY: :42 | |
| COLUMNS: 120 | |
| run: | | |
| pytest | |
| working-directory: ${{ matrix.package.name }} | |
| - name: Set failure type for test | |
| if: failure() && env.failure_type != 'install' | |
| run: | | |
| echo "Test failed for ${{ matrix.package.name }}" | |
| echo "failure_type=test" >> $GITHUB_ENV | |
| - name: Generate tokens | |
| id: app-token | |
| if: failure() | |
| uses: actions/create-github-app-token@v2 | |
| with: | |
| app-id: ${{ vars.ISSUE_CREATOR_APP_ID }} | |
| private-key: ${{ secrets.ISSUE_CREATOR_PRIVATE_KEY }} | |
| # https://github.com/scverse/gpu-ci-config/issues/4 | |
| - name: Install GitHub CLI | |
| if: failure() && matrix.package.gpu | |
| uses: sersoft-gmbh/setup-gh-cli-action@v2 | |
| with: | |
| version: stable | |
| - name: Check for open failure issue | |
| if: failure() | |
| id: find_issue | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| run: | | |
| ISSUE_TITLE="Integration Testing CI ${failure_type^} Failure on python ${{ matrix.python }}${{matrix.is_pre && ' with prerelease dependencies' || ''}}" | |
| echo "Checking for existing issue: $ISSUE_TITLE" | |
| ISSUE_COUNT=$(gh issue list --repo scverse/${{ matrix.package.name }} --state open --search "${ISSUE_TITLE}" --json number --jq 'length') | |
| if [[ "$ISSUE_COUNT" -gt 0 ]]; then | |
| echo "${failure_type^} failure issue already exists for today." | |
| echo "issue_exists=true" >> $GITHUB_ENV | |
| else | |
| echo "issue_exists=false" >> $GITHUB_ENV | |
| echo "issue_title=$ISSUE_TITLE" >> $GITHUB_ENV | |
| fi | |
| - name: Report failure issue | |
| if: failure() && env.issue_exists == 'false' && github.event_name == 'schedule' | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| run: | | |
| RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| ISSUE_BODY="The daily CI failed on ${failure_type} for ${{ matrix.package.name }} failed. Please go to [the logs of the integration testing repo](${RUN_URL}) to review. @scverse/anndata" | |
| gh issue create --repo scverse/${{ matrix.package.name }} --title "${{ env.issue_title }}" --body "${ISSUE_BODY}" | |
| keepalive-job: | |
| name: Keepalive Workflow | |
| runs-on: ubuntu-latest | |
| permissions: | |
| actions: write | |
| steps: | |
| - name: Re-enable workflow | |
| env: | |
| GITHUB_TOKEN: ${{ inputs.GITHUB_TOKEN || github.token }} | |
| shell: sh | |
| run: | | |
| gh api --verbose -X PUT "repos/${GITHUB_REPOSITORY}/actions/workflows/integration-test.yml/enable" |