Skip to content

Commit fa2f757

Browse files
anakin87anakin87
authored
test: track test coverage for all integrations (#3065)
* ci: fix github action expression * test: track test coverage for all integrations * fix inconsistencies * tests must not fail if all are skipped * do not run integration tests if missing * langfuse: ignore coverage errors * unstructured: do not docker prune
1 parent 78d1aa2 commit fa2f757

File tree

103 files changed

+2987
-338
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

103 files changed

+2987
-338
lines changed

‎.github/workflows/CI_coverage_comment.yml‎

Lines changed: 52 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,57 @@ name: Add comment about test coverage to PRs
33
on:
44
workflow_run:
55
workflows:
6+
- "Test / aimlapi"
7+
- "Test / amazon-bedrock"
8+
- "Test / amazon-sagemaker"
9+
- "Test / anthropic"
10+
- "Test / arcadedb"
11+
- "Test / astra"
12+
- "Test / azure_ai_search"
13+
- "Test / azure_doc_intelligence"
14+
- "Test / chroma"
15+
- "Test / cohere"
16+
- "Test / cometapi"
17+
- "Test / deepeval"
18+
- "Test / elasticsearch"
19+
- "Test / faiss"
20+
- "Test / fastembed"
21+
- "Test / firecrawl"
22+
- "Test / github"
23+
- "Test / google-genai"
24+
- "Test / hanlp"
25+
- "Test / jina"
626
- "Test / kreuzberg"
27+
- "Test / langfuse"
28+
- "Test / lara"
29+
- "Test / libreoffice"
30+
- "Test / llama_cpp"
31+
- "Test / llama_stack"
32+
- "Test / markitdown"
33+
- "Test / mcp"
34+
- "Test / meta_llama"
35+
- "Test / mistral"
36+
- "Test / mongodb_atlas"
37+
- "Test / nvidia"
38+
- "Test / ollama"
39+
- "Test / openrouter"
40+
- "Test / opensearch"
41+
- "Test / optimum"
42+
- "Test / paddleocr"
43+
- "Test / pgvector"
44+
- "Test / pinecone"
45+
- "Test / pyversity"
46+
- "Test / qdrant"
47+
- "Test / ragas"
48+
- "Test / snowflake"
49+
- "Test / stackit"
50+
- "Test / tavily"
51+
- "Test / togetherai"
52+
- "Test / unstructured"
53+
- "Test / valkey"
54+
- "Test / watsonx"
55+
- "Test / weave"
56+
- "Test / weaviate"
757
types: [completed]
858

959
jobs:
@@ -16,8 +66,8 @@ jobs:
1666
- name: Get integration name from workflow name
1767
id: integration
1868
run: |
19-
# Workflow names follow "Test / <name>" convention
20-
echo "name=$(echo '${{ github.event.workflow_run.name }}' | sed 's/Test \/ //')" >> $GITHUB_OUTPUT
69+
# Workflow names follow "Test / <name>" convention; normalize hyphens to underscores to match directory names
70+
echo "name=$(echo '${{ github.event.workflow_run.name }}' | sed 's/Test \/ //' | tr '-' '_')" >> $GITHUB_OUTPUT
2171
2272
- uses: py-cov-action/python-coverage-comment-action@7188638f871f721a365d644f505d1ff3df20d683 # v3.40
2373
with:

‎.github/workflows/aimlapi.yml‎

Lines changed: 57 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,29 +10,56 @@ on:
1010
- "integrations/aimlapi/**"
1111
- "!integrations/aimlapi/*.md"
1212
- ".github/workflows/aimlapi.yml"
13+
push:
14+
branches:
15+
- main
16+
paths:
17+
- "integrations/aimlapi/**"
18+
- "!integrations/aimlapi/*.md"
19+
- ".github/workflows/aimlapi.yml"
1320

1421
defaults:
1522
run:
1623
working-directory: integrations/aimlapi
1724

1825
concurrency:
19-
group: aimlapi-${{ github.head_ref }}
26+
group: aimlapi-${{ github.head_ref || github.sha }}
2027
cancel-in-progress: true
2128

2229
env:
2330
PYTHONUNBUFFERED: "1"
2431
FORCE_COLOR: "1"
2532
AIMLAPI_API_KEY: ${{ secrets.AIMLAPI_API_KEY }}
33+
TEST_MATRIX_OS: '["ubuntu-latest", "windows-latest", "macos-latest"]'
34+
TEST_MATRIX_PYTHON: '["3.10", "3.14"]'
2635

2736
jobs:
37+
compute-test-matrix:
38+
runs-on: ubuntu-slim
39+
defaults:
40+
run:
41+
working-directory: .
42+
outputs:
43+
os: ${{ steps.set.outputs.os }}
44+
python-version: ${{ steps.set.outputs.python-version }}
45+
steps:
46+
- id: set
47+
run: |
48+
echo 'os=${{ github.event_name == 'push' && '["ubuntu-latest"]' || env.TEST_MATRIX_OS }}' >> $GITHUB_OUTPUT
49+
echo 'python-version=${{ github.event_name == 'push' && '["3.10"]' || env.TEST_MATRIX_PYTHON }}' >> $GITHUB_OUTPUT
50+
2851
run:
2952
name: Python ${{ matrix.python-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }}
53+
needs: compute-test-matrix
54+
permissions:
55+
contents: write
56+
pull-requests: write
3057
runs-on: ${{ matrix.os }}
3158
strategy:
3259
fail-fast: false
3360
matrix:
34-
os: [ubuntu-latest, windows-latest, macos-latest]
35-
python-version: ["3.10", "3.14"]
61+
os: ${{ fromJSON(needs.compute-test-matrix.outputs.os) }}
62+
python-version: ${{ fromJSON(needs.compute-test-matrix.outputs.python-version) }}
3663

3764
steps:
3865
- name: Support longpaths
@@ -54,10 +81,33 @@ jobs:
5481
if: matrix.python-version == '3.10' && runner.os == 'Linux'
5582
run: hatch run fmt-check && hatch run test:types
5683

57-
- name: Run tests
58-
run: hatch run test:cov-retry
84+
- name: Run unit tests
85+
run: hatch run test:unit-cov-retry
86+
87+
# On PR: generates coverage comment artifact. On push to main: stores coverage baseline on data branch.
88+
- name: Store unit tests coverage
89+
if: matrix.python-version == '3.10' && runner.os == 'Linux' && github.event_name != 'schedule'
90+
uses: py-cov-action/python-coverage-comment-action@7188638f871f721a365d644f505d1ff3df20d683 # v3.40
91+
with:
92+
GITHUB_TOKEN: ${{ github.token }}
93+
COVERAGE_PATH: integrations/aimlapi
94+
SUBPROJECT_ID: aimlapi
95+
COMMENT_ARTIFACT_NAME: coverage-comment-aimlapi
96+
97+
- name: Run integration tests
98+
run: hatch run test:integration-cov-append-retry
99+
100+
- name: Store combined coverage
101+
if: github.event_name == 'push'
102+
uses: py-cov-action/python-coverage-comment-action@7188638f871f721a365d644f505d1ff3df20d683 # v3.40
103+
with:
104+
GITHUB_TOKEN: ${{ github.token }}
105+
COVERAGE_PATH: integrations/aimlapi
106+
SUBPROJECT_ID: aimlapi-combined
107+
COMMENT_ARTIFACT_NAME: coverage-comment-aimlapi-combined
59108

60109
- name: Run unit tests with lowest direct dependencies
110+
if: github.event_name != 'push'
61111
run: |
62112
hatch run uv pip compile pyproject.toml --resolution lowest-direct --output-file requirements_lowest_direct.txt
63113
hatch -e test env run -- uv pip install -r requirements_lowest_direct.txt
@@ -69,7 +119,8 @@ jobs:
69119
run: |
70120
hatch env prune
71121
hatch -e test env run -- uv pip install git+https://github.com/deepset-ai/haystack.git@main
72-
hatch run test:cov-retry
122+
hatch run test:unit-cov-retry
123+
hatch run test:integration-cov-append-retry
73124
74125
notify-slack-on-failure:
75126
needs: run

‎.github/workflows/amazon_bedrock.yml‎

Lines changed: 53 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,20 @@ on:
1010
- "integrations/amazon_bedrock/**"
1111
- "!integrations/amazon_bedrock/*.md"
1212
- ".github/workflows/amazon_bedrock.yml"
13+
push:
14+
branches:
15+
- main
16+
paths:
17+
- "integrations/amazon_bedrock/**"
18+
- "!integrations/amazon_bedrock/*.md"
19+
- ".github/workflows/amazon_bedrock.yml"
1320

1421
defaults:
1522
run:
1623
working-directory: integrations/amazon_bedrock
1724

1825
concurrency:
19-
group: amazon-bedrock-${{ github.head_ref }}
26+
group: amazon-bedrock-${{ github.head_ref || github.sha }}
2027
cancel-in-progress: true
2128

2229
permissions:
@@ -26,6 +33,8 @@ permissions:
2633
env:
2734
PYTHONUNBUFFERED: "1"
2835
FORCE_COLOR: "1"
36+
TEST_MATRIX_OS: '["ubuntu-latest", "windows-latest", "macos-latest"]'
37+
TEST_MATRIX_PYTHON: '["3.10", "3.14"]'
2938

3039
AWS_REGION: "us-east-1"
3140
AWS_BEDROCK_GUARDRAIL_ID: ${{ secrets.AWS_BEDROCK_GUARDRAIL_ID }}
@@ -34,14 +43,33 @@ env:
3443

3544

3645
jobs:
46+
compute-test-matrix:
47+
runs-on: ubuntu-slim
48+
defaults:
49+
run:
50+
working-directory: .
51+
outputs:
52+
os: ${{ steps.set.outputs.os }}
53+
python-version: ${{ steps.set.outputs.python-version }}
54+
steps:
55+
- id: set
56+
run: |
57+
echo 'os=${{ github.event_name == 'push' && '["ubuntu-latest"]' || env.TEST_MATRIX_OS }}' >> $GITHUB_OUTPUT
58+
echo 'python-version=${{ github.event_name == 'push' && '["3.10"]' || env.TEST_MATRIX_PYTHON }}' >> $GITHUB_OUTPUT
59+
3760
run:
3861
name: Python ${{ matrix.python-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }}
62+
needs: compute-test-matrix
63+
permissions:
64+
id-token: write
65+
contents: write
66+
pull-requests: write
3967
runs-on: ${{ matrix.os }}
4068
strategy:
4169
fail-fast: false
4270
matrix:
43-
os: [ubuntu-latest, windows-latest, macos-latest]
44-
python-version: ["3.10", "3.14"]
71+
os: ${{ fromJSON(needs.compute-test-matrix.outputs.os) }}
72+
python-version: ${{ fromJSON(needs.compute-test-matrix.outputs.python-version) }}
4573

4674
steps:
4775
- name: Support longpaths
@@ -64,7 +92,17 @@ jobs:
6492
run: hatch run fmt-check && hatch run test:types
6593

6694
- name: Run unit tests
67-
run: hatch run test:unit
95+
run: hatch run test:unit-cov-retry
96+
97+
# On PR: generates coverage comment artifact. On push to main: stores coverage baseline on data branch.
98+
- name: Store unit tests coverage
99+
if: matrix.python-version == '3.10' && runner.os == 'Linux' && github.event_name != 'schedule'
100+
uses: py-cov-action/python-coverage-comment-action@7188638f871f721a365d644f505d1ff3df20d683 # v3.40
101+
with:
102+
GITHUB_TOKEN: ${{ github.token }}
103+
COVERAGE_PATH: integrations/amazon_bedrock
104+
SUBPROJECT_ID: amazon_bedrock
105+
COMMENT_ARTIFACT_NAME: coverage-comment-amazon_bedrock
68106

69107
# Do not authenticate on PRs from forks and on PRs created by dependabot
70108
- name: AWS authentication
@@ -77,9 +115,19 @@ jobs:
77115

78116
- name: Run integration tests
79117
if: success() && steps.aws-auth.outcome == 'success'
80-
run: hatch run test:cov-retry -m "integration"
118+
run: hatch run test:integration-cov-append-retry
119+
120+
- name: Store combined coverage
121+
if: github.event_name == 'push'
122+
uses: py-cov-action/python-coverage-comment-action@7188638f871f721a365d644f505d1ff3df20d683 # v3.40
123+
with:
124+
GITHUB_TOKEN: ${{ github.token }}
125+
COVERAGE_PATH: integrations/amazon_bedrock
126+
SUBPROJECT_ID: amazon_bedrock-combined
127+
COMMENT_ARTIFACT_NAME: coverage-comment-amazon_bedrock-combined
81128

82129
- name: Run unit tests with lowest direct dependencies
130+
if: github.event_name != 'push'
83131
run: |
84132
hatch run uv pip compile pyproject.toml --resolution lowest-direct --output-file requirements_lowest_direct.txt
85133
hatch -e test env run -- uv pip install -r requirements_lowest_direct.txt

‎.github/workflows/amazon_sagemaker.yml‎

Lines changed: 55 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,28 +10,55 @@ on:
1010
- "integrations/amazon_sagemaker/**"
1111
- "!integrations/amazon_sagemaker/*.md"
1212
- ".github/workflows/amazon_sagemaker.yml"
13+
push:
14+
branches:
15+
- main
16+
paths:
17+
- "integrations/amazon_sagemaker/**"
18+
- "!integrations/amazon_sagemaker/*.md"
19+
- ".github/workflows/amazon_sagemaker.yml"
1320

1421
defaults:
1522
run:
1623
working-directory: integrations/amazon_sagemaker
1724

1825
concurrency:
19-
group: amazon-sagemaker-${{ github.head_ref }}
26+
group: amazon-sagemaker-${{ github.head_ref || github.sha }}
2027
cancel-in-progress: true
2128

2229
env:
2330
PYTHONUNBUFFERED: "1"
2431
FORCE_COLOR: "1"
32+
TEST_MATRIX_OS: '["ubuntu-latest", "windows-latest", "macos-latest"]'
33+
TEST_MATRIX_PYTHON: '["3.10", "3.14"]'
2534

2635
jobs:
36+
compute-test-matrix:
37+
runs-on: ubuntu-slim
38+
defaults:
39+
run:
40+
working-directory: .
41+
outputs:
42+
os: ${{ steps.set.outputs.os }}
43+
python-version: ${{ steps.set.outputs.python-version }}
44+
steps:
45+
- id: set
46+
run: |
47+
echo 'os=${{ github.event_name == 'push' && '["ubuntu-latest"]' || env.TEST_MATRIX_OS }}' >> $GITHUB_OUTPUT
48+
echo 'python-version=${{ github.event_name == 'push' && '["3.10"]' || env.TEST_MATRIX_PYTHON }}' >> $GITHUB_OUTPUT
49+
2750
run:
2851
name: Python ${{ matrix.python-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }}
52+
needs: compute-test-matrix
53+
permissions:
54+
contents: write
55+
pull-requests: write
2956
runs-on: ${{ matrix.os }}
3057
strategy:
3158
fail-fast: false
3259
matrix:
33-
os: [ubuntu-latest, windows-latest, macos-latest]
34-
python-version: ["3.10", "3.14"]
60+
os: ${{ fromJSON(needs.compute-test-matrix.outputs.os) }}
61+
python-version: ${{ fromJSON(needs.compute-test-matrix.outputs.python-version) }}
3562

3663
steps:
3764
- name: Support longpaths
@@ -53,10 +80,33 @@ jobs:
5380
if: matrix.python-version == '3.10' && runner.os == 'Linux'
5481
run: hatch run fmt-check && hatch run test:types
5582

56-
- name: Run tests
57-
run: hatch run test:cov-retry
83+
- name: Run unit tests
84+
run: hatch run test:unit-cov-retry
85+
86+
# On PR: generates coverage comment artifact. On push to main: stores coverage baseline on data branch.
87+
- name: Store unit tests coverage
88+
if: matrix.python-version == '3.10' && runner.os == 'Linux' && github.event_name != 'schedule'
89+
uses: py-cov-action/python-coverage-comment-action@7188638f871f721a365d644f505d1ff3df20d683 # v3.40
90+
with:
91+
GITHUB_TOKEN: ${{ github.token }}
92+
COVERAGE_PATH: integrations/amazon_sagemaker
93+
SUBPROJECT_ID: amazon_sagemaker
94+
COMMENT_ARTIFACT_NAME: coverage-comment-amazon_sagemaker
95+
96+
- name: Run integration tests
97+
run: hatch run test:integration-cov-append-retry
98+
99+
- name: Store combined coverage
100+
if: github.event_name == 'push'
101+
uses: py-cov-action/python-coverage-comment-action@7188638f871f721a365d644f505d1ff3df20d683 # v3.40
102+
with:
103+
GITHUB_TOKEN: ${{ github.token }}
104+
COVERAGE_PATH: integrations/amazon_sagemaker
105+
SUBPROJECT_ID: amazon_sagemaker-combined
106+
COMMENT_ARTIFACT_NAME: coverage-comment-amazon_sagemaker-combined
58107

59108
- name: Run unit tests with lowest direct dependencies
109+
if: github.event_name != 'push'
60110
run: |
61111
hatch run uv pip compile pyproject.toml --resolution lowest-direct --output-file requirements_lowest_direct.txt
62112
hatch -e test env run -- uv pip install -r requirements_lowest_direct.txt

0 commit comments

Comments
 (0)