unstructured.yml

# This workflow comes from https://github.com/ofek/hatch-mypyc
# https://github.com/ofek/hatch-mypyc/blob/5a198c0ba8660494d02716cfc9d79ce4adfb1442/.github/workflows/test.yml
name: Test / unstructured

on:
  schedule:
    - cron: "0 0 * * *"
  pull_request:
    paths:
      - "integrations/unstructured/**"
      - ".github/workflows/unstructured.yml"
      - "!integrations/unstructured/*.md"
  push:
    branches:
      - main
    paths:
      - "integrations/unstructured/**"
      - ".github/workflows/unstructured.yml"
      - "!integrations/unstructured/*.md"

defaults:
  run:
    working-directory: integrations/unstructured

concurrency:
  group: unstructured-${{ github.head_ref || github.sha }}
  cancel-in-progress: true

env:
  PYTHONUNBUFFERED: "1"
  FORCE_COLOR: "1"
  TEST_MATRIX_OS: '["ubuntu-latest"]'
  TEST_MATRIX_PYTHON: '["3.10", "3.14"]'

jobs:
  compute-test-matrix:
    runs-on: ubuntu-slim
    defaults:
      run:
        working-directory: .
    outputs:
      os: ${{ steps.set.outputs.os }}
      python-version: ${{ steps.set.outputs.python-version }}
    steps:
      - id: set
        run: |
          echo 'os=${{ github.event_name == 'push' && '["ubuntu-latest"]' || env.TEST_MATRIX_OS }}' >> "$GITHUB_OUTPUT"
          echo 'python-version=${{ github.event_name == 'push' && '["3.10"]' || env.TEST_MATRIX_PYTHON }}' >> "$GITHUB_OUTPUT"

  run:
    name: Python ${{ matrix.python-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }}
    needs: compute-test-matrix
    permissions:
      contents: write
      pull-requests: write
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: ${{ fromJSON(needs.compute-test-matrix.outputs.os) }}
        python-version: ${{ fromJSON(needs.compute-test-matrix.outputs.python-version) }}

    steps:
      - name: Free up disk space
        working-directory: .
        run: |
          sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/share/boost /usr/local/lib/android /opt/hostedtoolcache/CodeQL

      - name: Run Unstructured API (docker)
        working-directory: .
        run: |
          docker run -d \
            --name unstructured-api \
            -p 8000:8000 \
            --health-cmd "curl --fail http://localhost:8000/healthcheck || exit 1" \
            --health-interval 10s \
            --health-timeout 1s \
            --health-retries 10 \
            quay.io/unstructured-io/unstructured-api:latest

      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install Hatch
        run: pip install hatch

      - name: Lint
        if: matrix.python-version == '3.10' && runner.os == 'Linux'
        run: hatch run fmt-check && hatch run test:types

      - name: Run unit tests
        run: hatch run test:unit-cov-retry

      # On PR: posts coverage comment (directly on same-repo PRs; via artifact for fork PRs). On push to main: stores coverage baseline on data branch.
      - name: Store unit tests coverage
        id: coverage_comment
        if: matrix.python-version == '3.10' && runner.os == 'Linux' && github.event_name != 'schedule'
        uses: py-cov-action/python-coverage-comment-action@63f52f4fbbffada6e8dee8ec432de7e01df9ba79 # v3.41
        with:
          GITHUB_TOKEN: ${{ github.token }}
          COVERAGE_PATH: integrations/unstructured
          SUBPROJECT_ID: unstructured
          MINIMUM_GREEN: 90
          MINIMUM_ORANGE: 60

      - name: Upload coverage comment to be posted
        if: matrix.python-version == '3.10' && runner.os == 'Linux' && github.event_name == 'pull_request' && steps.coverage_comment.outputs.COMMENT_FILE_WRITTEN == 'true'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: coverage-comment-unstructured
          path: python-coverage-comment-action-unstructured.txt

      - name: Run integration tests
        run: hatch run test:integration-cov-append-retry

      - name: Store combined coverage
        if: github.event_name == 'push'
        uses: py-cov-action/python-coverage-comment-action@63f52f4fbbffada6e8dee8ec432de7e01df9ba79 # v3.41
        with:
          GITHUB_TOKEN: ${{ github.token }}
          COVERAGE_PATH: integrations/unstructured
          SUBPROJECT_ID: unstructured-combined
          MINIMUM_GREEN: 90
          MINIMUM_ORANGE: 60

      - name: Run unit tests with lowest direct dependencies
        if: github.event_name != 'push'
        run: |
          hatch run uv pip compile pyproject.toml --resolution lowest-direct --output-file requirements_lowest_direct.txt
          hatch -e test env run -- uv pip install -r requirements_lowest_direct.txt
          hatch run test:unit

      - name: Nightly - run unit tests with Haystack main branch
        if: github.event_name == 'schedule'
        run: |
          hatch env prune
          hatch -e test env run -- uv pip install git+https://github.com/deepset-ai/haystack.git@main
          hatch run test:unit

  notify-slack-on-failure:
    needs: run
    if: failure() && github.event_name == 'schedule'
    runs-on: ubuntu-slim
    steps:
      - uses: deepset-ai/notify-slack-action@3cda73b77a148f16f703274198e7771340cf862b # v1
        with:
          slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL_NOTIFICATIONS }}