test: start tracking test coverage (#3061)

* test: start tracking test coverage

* improvements

* more

Author: anakin87

.github/workflows/CI_coverage_comment.yml

@@ -0,0 +1,27 @@
+name: Add comment about test coverage to PRs
+
+on:
+  workflow_run:
+    workflows:
+      - "Test / kreuzberg"
+    types: [completed]
+
+jobs:
+  comment:
+    if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request'
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Get integration name from workflow name
+        id: integration
+        run: |
+          # Workflow names follow "Test / <name>" convention
+          echo "name=$(echo '${{ github.event.workflow_run.name }}' | sed 's/Test \/ //')" >> $GITHUB_OUTPUT
+
+      - uses: py-cov-action/python-coverage-comment-action@7188638f871f721a365d644f505d1ff3df20d683 # v3.40
+        with:
+          GITHUB_TOKEN: ${{ github.token }}
+          GITHUB_PR_RUN_ID: ${{ github.event.workflow_run.id }}
+          SUBPROJECT_ID: ${{ steps.integration.outputs.name }}
+          COMMENT_ARTIFACT_NAME: coverage-comment-${{ steps.integration.outputs.name }}

.github/workflows/kreuzberg.yml

@@ -10,28 +10,55 @@ on:
       - "integrations/kreuzberg/**"
       - "!integrations/kreuzberg/*.md"
       - ".github/workflows/kreuzberg.yml"
+  push:
+    branches:
+      - main
+    paths:
+      - "integrations/kreuzberg/**"
+      - "!integrations/kreuzberg/*.md"
+      - ".github/workflows/kreuzberg.yml"
 
 defaults:
   run:
     working-directory: integrations/kreuzberg
 
 concurrency:
-  group: kreuzberg-${{ github.head_ref }}
+  group: kreuzberg-${{ github.head_ref || github.sha }}
   cancel-in-progress: true
 
 env:
   PYTHONUNBUFFERED: "1"
   FORCE_COLOR: "1"
+  TEST_MATRIX_OS: '["ubuntu-latest", "windows-latest", "macos-latest"]'
+  TEST_MATRIX_PYTHON: '["3.10", "3.14"]'
 
 jobs:
+  compute-test-matrix:
+    runs-on: ubuntu-slim
+    defaults:
+      run:
+        working-directory: .
+    outputs:
+      os: ${{ steps.set.outputs.os }}
+      python-version: ${{ steps.set.outputs.python-version }}
+    steps:
+      - id: set
+        run: |
+          echo "os=${{ github.event_name == 'push' && '["ubuntu-latest"]' || env.TEST_MATRIX_OS }}" >> $GITHUB_OUTPUT
+          echo "python-version=${{ github.event_name == 'push' && '["3.10"]' || env.TEST_MATRIX_PYTHON }}" >> $GITHUB_OUTPUT
+
   run:
     name: Python ${{ matrix.python-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }}
+    needs: compute-test-matrix
     runs-on: ${{ matrix.os }}
+    permissions:
+      contents: write
+      pull-requests: write
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ["3.10", "3.14"]
+        os: ${{ fromJSON(needs.compute-test-matrix.outputs.os) }}
+        python-version: ${{ fromJSON(needs.compute-test-matrix.outputs.python-version) }}
 
     steps:
       - name: Support longpaths
@@ -50,13 +77,36 @@ jobs:
         run: pip install hatch
 
       - name: Lint
-        if: matrix.python-version == '3.10' && runner.os == 'Linux'
+        if: github.event_name != 'push' && matrix.python-version == '3.10' && runner.os == 'Linux'
         run: hatch run fmt-check && hatch run test:types
 
-      - name: Run tests
-        run: hatch run test:cov-retry
+      - name: Run unit tests
+        run: hatch run test:unit-cov-retry
+
+      # On PR: generates coverage comment artifact. On push to main: stores coverage baseline on data branch.
+      - name: Store unit tests coverage
+        if: matrix.python-version == '3.10' && runner.os == 'Linux' && github.event_name != 'schedule'
+        uses: py-cov-action/python-coverage-comment-action@7188638f871f721a365d644f505d1ff3df20d683 # v3.40
+        with:
+          GITHUB_TOKEN: ${{ github.token }}
+          COVERAGE_PATH: integrations/kreuzberg
+          SUBPROJECT_ID: kreuzberg
+          COMMENT_ARTIFACT_NAME: coverage-comment-kreuzberg
+
+      - name: Run integration tests
+        run: hatch run test:integration-cov-append-retry
+
+      - name: Store combined coverage
+        if: github.event_name == 'push'
+        uses: py-cov-action/python-coverage-comment-action@7188638f871f721a365d644f505d1ff3df20d683 # v3.40
+        with:
+          GITHUB_TOKEN: ${{ github.token }}
+          COVERAGE_PATH: integrations/kreuzberg
+          SUBPROJECT_ID: kreuzberg-combined
+          COMMENT_ARTIFACT_NAME: coverage-comment-kreuzberg-combined
 
       - name: Run unit tests with lowest direct dependencies
+        if: github.event_name != 'push'
         run: |
           hatch run uv pip compile pyproject.toml --resolution lowest-direct --output-file requirements_lowest_direct.txt
           hatch -e test env run -- uv pip install -r requirements_lowest_direct.txt
@@ -69,7 +119,6 @@ jobs:
           hatch -e test env run -- uv pip install git+https://github.com/deepset-ai/haystack.git@main
           hatch run test:unit
 
-
   notify-slack-on-failure:
     needs: run
     if: failure() && github.event_name == 'schedule'

integrations/kreuzberg/pyproject.toml

@@ -66,7 +66,8 @@ dependencies = [
 unit = 'pytest -m "not integration" {args:tests}'
 integration = 'pytest -m "integration" {args:tests}'
 all = 'pytest {args:tests}'
-cov-retry = 'pytest --cov=haystack_integrations --reruns 3 --reruns-delay 30 -x {args:tests}'
+unit-cov-retry = 'pytest --cov=haystack_integrations --reruns 3 --reruns-delay 30 -x -m "not integration" {args:tests}'
+integration-cov-append-retry = 'pytest --cov=haystack_integrations --cov-append --reruns 3 --reruns-delay 30 -x -m "integration" {args:tests}'
 types = "mypy -p haystack_integrations.components.converters.kreuzberg {args}"
 
 [tool.ruff]
@@ -161,6 +162,7 @@ asyncio_default_fixture_loop_scope = "function"
 source = ["haystack_integrations"]
 branch = true
 parallel = false
+relative_files = true
 
 [tool.coverage.report]
 omit = ["*/tests/*", "*/__init__.py"]