slow.yml

# If you change this name also do it in ci_metrics.yml
name: Slow Integration Tests

# The workflow will always run, but the actual tests will only execute when:
# - The workflow is triggered manually
# - The workflow is scheduled
# - The PR has the "run-slow-tests" label
# - The push is to a release branch
# - There are changes to relevant files.
# Note: If no conditions are met, the workflow will complete successfully without running tests
# to satisfy Branch Protection rules.

env:
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  HF_API_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }}
  PYTHON_VERSION: "3.10"
  HATCH_VERSION: "1.16.5"
  HAYSTACK_MPS_ENABLED: false
  HAYSTACK_XPU_ENABLED: false

on:
  workflow_dispatch: # Activate this workflow manually
  schedule:
    - cron: "0 0 * * *"
  push:
    branches:
      # release branches have the form v1.9.x
      - "v[0-9].*[0-9].x"
  pull_request:
    types:
      - opened
      - reopened
      - synchronize
      - labeled
      - unlabeled

jobs:
  check-if-changed:
    # This job checks if the relevant files have been changed.
    # We check for changes in the check-if-changed job instead of using paths/paths-ignore at workflow level.
    # This ensures the "Slow Integration Tests completed" job always runs, which is required by Branch Protection rules.
    name: Check if changed
    runs-on: ubuntu-slim
    permissions:
      pull-requests: read
    # Specifying outputs is not needed to make the job work, but only to comply with actionlint
    outputs:
      changes: ${{ steps.changes.outputs.changes }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - name: Check for changed code
        id: changes
        uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
        with:
          # List of Python files that trigger slow integration tests when modified
          filters: |
            changes:
              - "haystack/components/audio/whisper_local.py"
              - "haystack/components/classifiers/zero_shot_document_classifier.py"
              - "haystack/components/converters/tika.py"
              - "haystack/components/embedders/hugging_face_api_document_embedder.py"
              - "haystack/components/embedders/hugging_face_api_text_embedder.py"
              - "haystack/components/embedders/backends/sentence_transformers_backend.py"
              - "haystack/components/embedders/backends/sentence_transformers_sparse_backend.py"
              - "haystack/components/embedders/image/sentence_transformers_doc_image_embedder.py"
              - "haystack/components/embedders/sentence_transformers_text_embedder.py"
              - "haystack/components/embedders/sentence_transformers_sparse_document_embedder.py"
              - "haystack/components/embedders/sentence_transformers_sparse_text_embedder.py"
              - "haystack/components/evaluators/sas_evaluator.py"
              - "haystack/components/generators/chat/hugging_face_api.py"
              - "haystack/components/generators/chat/hugging_face_local.py"
              - "haystack/components/generators/hugging_face_api.py"
              - "haystack/components/generators/hugging_face_local_generator.py"
              - "haystack/components/generators/openai_dalle.py"
              - "haystack/components/preprocessors/embedding_based_document_splitter.py"
              - "haystack/components/rankers/sentence_transformers_diversity.py"
              - "haystack/components/rankers/sentence_transformers_similarity.py"
              - "haystack/components/rankers/transformers_similarity.py"
              - "haystack/components/readers/extractive.py"
              - "haystack/components/retrievers/multi_query_embedding_retriever.py"
              - "haystack/components/routers/transformers_text_router.py"
              - "haystack/components/routers/zero_shot_text_router.py"

              - "test/components/audio/test_whisper_local.py"
              - "test/components/classifiers/test_zero_shot_document_classifier.py"
              - "test/components/converters/test_tika_doc_converter.py"
              - "test/components/embedders/test_hugging_face_api_document_embedder.py"
              - "test/components/embedders/test_hugging_face_api_text_embedder.py"
              - "test/components/embedders/image/test_sentence_transformers_doc_image_embedder.py"
              - "test/components/embedders/test_sentence_transformers_text_embedder.py"
              - "test/components/embedders/test_sentence_transformers_sparse_document_embedder.py"
              - "test/components/embedders/test_sentence_transformers_sparse_text_embedder.py"
              - "test/components/evaluators/test_sas_evaluator.py"
              - "test/components/generators/chat/test_hugging_face_api.py"
              - "test/components/generators/chat/test_hugging_face_local.py"
              - "test/components/generators/test_hugging_face_api.py"
              - "test/components/generators/test_hugging_face_local_generator.py"
              - "test/components/generators/test_openai_dalle.py"
              - "test/components/preprocessors/test_embedding_based_document_splitter.py"
              - "test/components/rankers/test_sentence_transformers_diversity.py"
              - "test/components/rankers/test_sentence_transformers_similarity.py"
              - "test/components/rankers/test_transformers_similarity.py"
              - "test/components/readers/test_extractive.py"
              - "test/components/retrievers/test_multi_query_embedding_retriever.py"
              - "test/components/routers/test_transformers_text_router.py"
              - "test/components/routers/test_zero_shot_text_router.py"

  slow-integration-tests:
    name: Slow Tests / ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    needs: check-if-changed
    timeout-minutes: 30
    # Run tests if: manual trigger, scheduled, PR has label, release branch, or relevant files changed
    if: |
      github.event_name == 'workflow_dispatch' ||
      github.event_name == 'schedule' ||
      (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-slow-tests')) ||
      (github.event_name == 'push' && github.ref == 'refs/heads/v[0-9].*[0-9].x') ||
      (needs.check-if-changed.outputs.changes == 'true')

    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        include:
          - os: ubuntu-latest
            install_cmd: "sudo apt update && sudo apt install ffmpeg"
          - os: macos-latest
            install_cmd: "brew install ffmpeg"
          - os: windows-latest
            install_cmd: "echo 'No additional dependencies needed'"

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "${{ env.PYTHON_VERSION }}"

      - name: Install Hatch
        id: hatch
        shell: bash
        run: |
          python -m pip install --upgrade pip
          pip install hatch==${{ env.HATCH_VERSION }} --uploaded-prior-to=P1D

      - name: Run Tika
        if: matrix.os == 'ubuntu-latest'
        run: |
          docker run -d -p 9998:9998 apache/tika:2.9.0.0

      - name: Install Whisper dependencies
        shell: bash
        run: ${{ matrix.install_cmd }}

      - name: Run tests
        run: hatch run test:integration-only-slow

      - name: Notify Slack on nightly failure
        if: failure() && github.event_name == 'schedule'
        uses: deepset-ai/notify-slack-action@a65def0c8bf91d6520286ab34280151c76a5a008 # v1.1.0
        with:
          slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL_NOTIFICATIONS }}

  slow-integration-tests-completed:
    # This job always runs and succeeds if all tests succeed or are skipped. It is required by Branch Protection rules.
    name: Slow Integration Tests completed
    runs-on: ubuntu-slim
    if: ${{ always() && !cancelled() }}
    needs: slow-integration-tests

    steps:
      - name: Mark tests as completed
        run: |
          if [ "${{ needs.slow-integration-tests.result }}" = "failure" ]; then
            echo "Slow Integration Tests failed!"
            exit 1
          else
            echo "Slow Integration Tests completed!"
          fi