test: track test coverage for all integrations

.github/workflows/CI_coverage_comment.yml

@@ -3,7 +3,57 @@ name: Add comment about test coverage to PRs
 on:
   workflow_run:
     workflows:
+      - "Test / aimlapi"
+      - "Test / amazon-bedrock"
+      - "Test / amazon-sagemaker"
+      - "Test / anthropic"
+      - "Test / arcadedb"
+      - "Test / astra"
+      - "Test / azure_ai_search"
+      - "Test / azure_doc_intelligence"
+      - "Test / chroma"
+      - "Test / cohere"
+      - "Test / cometapi"
+      - "Test / deepeval"
+      - "Test / elasticsearch"
+      - "Test / faiss"
+      - "Test / fastembed"
+      - "Test / firecrawl"
+      - "Test / github"
+      - "Test / google-genai"
+      - "Test / hanlp"
+      - "Test / jina"
       - "Test / kreuzberg"
+      - "Test / langfuse"
+      - "Test / lara"
+      - "Test / libreoffice"
+      - "Test / llama_cpp"
+      - "Test / llama_stack"
+      - "Test / markitdown"
+      - "Test / mcp"
+      - "Test / meta_llama"
+      - "Test / mistral"
+      - "Test / mongodb_atlas"
+      - "Test / nvidia"
+      - "Test / ollama"
+      - "Test / openrouter"
+      - "Test / opensearch"
+      - "Test / optimum"
+      - "Test / paddleocr"
+      - "Test / pgvector"
+      - "Test / pinecone"
+      - "Test / pyversity"
+      - "Test / qdrant"
+      - "Test / ragas"
+      - "Test / snowflake"
+      - "Test / stackit"
+      - "Test / tavily"
+      - "Test / togetherai"
+      - "Test / unstructured"
+      - "Test / valkey"
+      - "Test / watsonx"
+      - "Test / weave"
+      - "Test / weaviate"
     types: [completed]
 
 jobs:
@@ -16,8 +66,8 @@ jobs:
       - name: Get integration name from workflow name
         id: integration
         run: |
-          # Workflow names follow "Test / <name>" convention
-          echo "name=$(echo '${{ github.event.workflow_run.name }}' | sed 's/Test \/ //')" >> $GITHUB_OUTPUT
+          # Workflow names follow "Test / <name>" convention; normalize hyphens to underscores to match directory names
+          echo "name=$(echo '${{ github.event.workflow_run.name }}' | sed 's/Test \/ //' | tr '-' '_')" >> $GITHUB_OUTPUT
 
       - uses: py-cov-action/python-coverage-comment-action@7188638f871f721a365d644f505d1ff3df20d683 # v3.40
         with:

.github/workflows/aimlapi.yml

@@ -10,29 +10,56 @@ on:
       - "integrations/aimlapi/**"
       - "!integrations/aimlapi/*.md"
       - ".github/workflows/aimlapi.yml"
+  push:
+    branches:
+      - main
+    paths:
+      - "integrations/aimlapi/**"
+      - "!integrations/aimlapi/*.md"
+      - ".github/workflows/aimlapi.yml"
 
 defaults:
   run:
     working-directory: integrations/aimlapi
 
 concurrency:
-  group: aimlapi-${{ github.head_ref }}
+  group: aimlapi-${{ github.head_ref || github.sha }}
   cancel-in-progress: true
 
 env:
   PYTHONUNBUFFERED: "1"
   FORCE_COLOR: "1"
   AIMLAPI_API_KEY: ${{ secrets.AIMLAPI_API_KEY }}
+  TEST_MATRIX_OS: '["ubuntu-latest", "windows-latest", "macos-latest"]'
+  TEST_MATRIX_PYTHON: '["3.10", "3.14"]'
 
 jobs:
+  compute-test-matrix:
+    runs-on: ubuntu-slim
+    defaults:
+      run:
+        working-directory: .
+    outputs:
+      os: ${{ steps.set.outputs.os }}
+      python-version: ${{ steps.set.outputs.python-version }}
+    steps:
+      - id: set
+        run: |
+          echo 'os=${{ github.event_name == 'push' && '["ubuntu-latest"]' || env.TEST_MATRIX_OS }}' >> $GITHUB_OUTPUT
+          echo 'python-version=${{ github.event_name == 'push' && '["3.10"]' || env.TEST_MATRIX_PYTHON }}' >> $GITHUB_OUTPUT
+
   run:
     name: Python ${{ matrix.python-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }}
+    needs: compute-test-matrix
+    permissions:
+      contents: write
+      pull-requests: write
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ["3.10", "3.14"]
+        os: ${{ fromJSON(needs.compute-test-matrix.outputs.os) }}
+        python-version: ${{ fromJSON(needs.compute-test-matrix.outputs.python-version) }}
 
     steps:
       - name: Support longpaths
@@ -54,10 +81,33 @@ jobs:
         if: matrix.python-version == '3.10' && runner.os == 'Linux'
         run: hatch run fmt-check && hatch run test:types
 
-      - name: Run tests
-        run: hatch run test:cov-retry
+      - name: Run unit tests
+        run: hatch run test:unit-cov-retry
+
+      # On PR: generates coverage comment artifact. On push to main: stores coverage baseline on data branch.
+      - name: Store unit tests coverage
+        if: matrix.python-version == '3.10' && runner.os == 'Linux' && github.event_name != 'schedule'
+        uses: py-cov-action/python-coverage-comment-action@7188638f871f721a365d644f505d1ff3df20d683 # v3.40
+        with:
+          GITHUB_TOKEN: ${{ github.token }}
+          COVERAGE_PATH: integrations/aimlapi
+          SUBPROJECT_ID: aimlapi
+          COMMENT_ARTIFACT_NAME: coverage-comment-aimlapi
+
+      - name: Run integration tests
+        run: hatch run test:integration-cov-append-retry
+
+      - name: Store combined coverage
+        if: github.event_name == 'push'
+        uses: py-cov-action/python-coverage-comment-action@7188638f871f721a365d644f505d1ff3df20d683 # v3.40
+        with:
+          GITHUB_TOKEN: ${{ github.token }}
+          COVERAGE_PATH: integrations/aimlapi
+          SUBPROJECT_ID: aimlapi-combined
+          COMMENT_ARTIFACT_NAME: coverage-comment-aimlapi-combined
 
       - name: Run unit tests with lowest direct dependencies
+        if: github.event_name != 'push'
         run: |
           hatch run uv pip compile pyproject.toml --resolution lowest-direct --output-file requirements_lowest_direct.txt
           hatch -e test env run -- uv pip install -r requirements_lowest_direct.txt
@@ -69,7 +119,8 @@ jobs:
         run: |
           hatch env prune
           hatch -e test env run -- uv pip install git+https://github.com/deepset-ai/haystack.git@main
-          hatch run test:cov-retry
+          hatch run test:unit-cov-retry
+          hatch run test:integration-cov-append-retry
 
   notify-slack-on-failure:
     needs: run

.github/workflows/amazon_bedrock.yml

@@ -10,13 +10,20 @@ on:
       - "integrations/amazon_bedrock/**"
       - "!integrations/amazon_bedrock/*.md"
       - ".github/workflows/amazon_bedrock.yml"
+  push:
+    branches:
+      - main
+    paths:
+      - "integrations/amazon_bedrock/**"
+      - "!integrations/amazon_bedrock/*.md"
+      - ".github/workflows/amazon_bedrock.yml"
 
 defaults:
   run:
     working-directory: integrations/amazon_bedrock
 
 concurrency:
-  group: amazon-bedrock-${{ github.head_ref }}
+  group: amazon-bedrock-${{ github.head_ref || github.sha }}
   cancel-in-progress: true
 
 permissions:
@@ -26,6 +33,8 @@ permissions:
 env:
   PYTHONUNBUFFERED: "1"
   FORCE_COLOR: "1"
+  TEST_MATRIX_OS: '["ubuntu-latest", "windows-latest", "macos-latest"]'
+  TEST_MATRIX_PYTHON: '["3.10", "3.14"]'
 
   AWS_REGION: "us-east-1"
   AWS_BEDROCK_GUARDRAIL_ID: ${{ secrets.AWS_BEDROCK_GUARDRAIL_ID }}
@@ -34,14 +43,33 @@ env:
 
 
 jobs:
+  compute-test-matrix:
+    runs-on: ubuntu-slim
+    defaults:
+      run:
+        working-directory: .
+    outputs:
+      os: ${{ steps.set.outputs.os }}
+      python-version: ${{ steps.set.outputs.python-version }}
+    steps:
+      - id: set
+        run: |
+          echo 'os=${{ github.event_name == 'push' && '["ubuntu-latest"]' || env.TEST_MATRIX_OS }}' >> $GITHUB_OUTPUT
+          echo 'python-version=${{ github.event_name == 'push' && '["3.10"]' || env.TEST_MATRIX_PYTHON }}' >> $GITHUB_OUTPUT
+
   run:
     name: Python ${{ matrix.python-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }}
+    needs: compute-test-matrix
+    permissions:
+      id-token: write
+      contents: write
+      pull-requests: write
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ["3.10", "3.14"]
+        os: ${{ fromJSON(needs.compute-test-matrix.outputs.os) }}
+        python-version: ${{ fromJSON(needs.compute-test-matrix.outputs.python-version) }}
 
     steps:
       - name: Support longpaths
@@ -64,7 +92,17 @@ jobs:
         run: hatch run fmt-check && hatch run test:types
 
       - name: Run unit tests
-        run: hatch run test:unit
+        run: hatch run test:unit-cov-retry
+
+      # On PR: generates coverage comment artifact. On push to main: stores coverage baseline on data branch.
+      - name: Store unit tests coverage
+        if: matrix.python-version == '3.10' && runner.os == 'Linux' && github.event_name != 'schedule'
+        uses: py-cov-action/python-coverage-comment-action@7188638f871f721a365d644f505d1ff3df20d683 # v3.40
+        with:
+          GITHUB_TOKEN: ${{ github.token }}
+          COVERAGE_PATH: integrations/amazon_bedrock
+          SUBPROJECT_ID: amazon_bedrock
+          COMMENT_ARTIFACT_NAME: coverage-comment-amazon_bedrock
 
       # Do not authenticate on PRs from forks and on PRs created by dependabot
       - name: AWS authentication
@@ -77,9 +115,19 @@ jobs:
 
       - name: Run integration tests
         if: success() && steps.aws-auth.outcome == 'success'
-        run: hatch run test:cov-retry -m "integration"
+        run: hatch run test:integration-cov-append-retry
+
+      - name: Store combined coverage
+        if: github.event_name == 'push'
+        uses: py-cov-action/python-coverage-comment-action@7188638f871f721a365d644f505d1ff3df20d683 # v3.40
+        with:
+          GITHUB_TOKEN: ${{ github.token }}
+          COVERAGE_PATH: integrations/amazon_bedrock
+          SUBPROJECT_ID: amazon_bedrock-combined
+          COMMENT_ARTIFACT_NAME: coverage-comment-amazon_bedrock-combined
 
       - name: Run unit tests with lowest direct dependencies
+        if: github.event_name != 'push'
         run: |
           hatch run uv pip compile pyproject.toml --resolution lowest-direct --output-file requirements_lowest_direct.txt
           hatch -e test env run -- uv pip install -r requirements_lowest_direct.txt

.github/workflows/amazon_sagemaker.yml

@@ -10,28 +10,55 @@ on:
       - "integrations/amazon_sagemaker/**"
       - "!integrations/amazon_sagemaker/*.md"
       - ".github/workflows/amazon_sagemaker.yml"
+  push:
+    branches:
+      - main
+    paths:
+      - "integrations/amazon_sagemaker/**"
+      - "!integrations/amazon_sagemaker/*.md"
+      - ".github/workflows/amazon_sagemaker.yml"
 
 defaults:
   run:
     working-directory: integrations/amazon_sagemaker
 
 concurrency:
-  group: amazon-sagemaker-${{ github.head_ref }}
+  group: amazon-sagemaker-${{ github.head_ref || github.sha }}
   cancel-in-progress: true
 
 env:
   PYTHONUNBUFFERED: "1"
   FORCE_COLOR: "1"
+  TEST_MATRIX_OS: '["ubuntu-latest", "windows-latest", "macos-latest"]'
+  TEST_MATRIX_PYTHON: '["3.10", "3.14"]'
 
 jobs:
+  compute-test-matrix:
+    runs-on: ubuntu-slim
+    defaults:
+      run:
+        working-directory: .
+    outputs:
+      os: ${{ steps.set.outputs.os }}
+      python-version: ${{ steps.set.outputs.python-version }}
+    steps:
+      - id: set
+        run: |
+          echo 'os=${{ github.event_name == 'push' && '["ubuntu-latest"]' || env.TEST_MATRIX_OS }}' >> $GITHUB_OUTPUT
+          echo 'python-version=${{ github.event_name == 'push' && '["3.10"]' || env.TEST_MATRIX_PYTHON }}' >> $GITHUB_OUTPUT
+
   run:
     name: Python ${{ matrix.python-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }}
+    needs: compute-test-matrix
+    permissions:
+      contents: write
+      pull-requests: write
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ["3.10", "3.14"]
+        os: ${{ fromJSON(needs.compute-test-matrix.outputs.os) }}
+        python-version: ${{ fromJSON(needs.compute-test-matrix.outputs.python-version) }}
 
     steps:
       - name: Support longpaths
@@ -53,10 +80,33 @@ jobs:
         if: matrix.python-version == '3.10' && runner.os == 'Linux'
         run: hatch run fmt-check && hatch run test:types
 
-      - name: Run tests
-        run: hatch run test:cov-retry
+      - name: Run unit tests
+        run: hatch run test:unit-cov-retry
+
+      # On PR: generates coverage comment artifact. On push to main: stores coverage baseline on data branch.
+      - name: Store unit tests coverage
+        if: matrix.python-version == '3.10' && runner.os == 'Linux' && github.event_name != 'schedule'
+        uses: py-cov-action/python-coverage-comment-action@7188638f871f721a365d644f505d1ff3df20d683 # v3.40
+        with:
+          GITHUB_TOKEN: ${{ github.token }}
+          COVERAGE_PATH: integrations/amazon_sagemaker
+          SUBPROJECT_ID: amazon_sagemaker
+          COMMENT_ARTIFACT_NAME: coverage-comment-amazon_sagemaker
+
+      - name: Run integration tests
+        run: hatch run test:integration-cov-append-retry
+
+      - name: Store combined coverage
+        if: github.event_name == 'push'
+        uses: py-cov-action/python-coverage-comment-action@7188638f871f721a365d644f505d1ff3df20d683 # v3.40
+        with:
+          GITHUB_TOKEN: ${{ github.token }}
+          COVERAGE_PATH: integrations/amazon_sagemaker
+          SUBPROJECT_ID: amazon_sagemaker-combined
+          COMMENT_ARTIFACT_NAME: coverage-comment-amazon_sagemaker-combined
 
       - name: Run unit tests with lowest direct dependencies
+        if: github.event_name != 'push'
         run: |
           hatch run uv pip compile pyproject.toml --resolution lowest-direct --output-file requirements_lowest_direct.txt
           hatch -e test env run -- uv pip install -r requirements_lowest_direct.txt