ci: Add reruns when running nightly tests if tests fail (#2584)

* Add reruns when running nightly tests if tests fail

* Switch to using cov-retry for running nightly tests for integrations that inherit from OpenAIChatGenerator

* Switch back to all just running tests, no retries, etc.

* Switch some integrations so all just runs tests with no retries

Author: sjrl

.github/workflows/llama_stack.yml

@@ -133,7 +133,7 @@ jobs:
         run: |
           hatch env prune
           hatch -e test env run -- uv pip install git+https://github.com/deepset-ai/haystack.git@main
-          hatch run test:all
+          hatch run test:cov-retry
 
       - name: Send event to Datadog for nightly failures
         if: failure() && github.event_name == 'schedule'

.github/workflows/meta_llama.yml

@@ -72,7 +72,7 @@ jobs:
         run: |
           hatch env prune
           hatch -e test env run -- uv pip install git+https://github.com/deepset-ai/haystack.git@main
-          hatch run test:all
+          hatch run test:cov-retry
 
       - name: Send event to Datadog for nightly failures
         if: failure() && github.event_name == 'schedule'

.github/workflows/mistral.yml

@@ -73,7 +73,7 @@ jobs:
         run: |
           hatch env prune
           hatch -e test env run -- uv pip install git+https://github.com/deepset-ai/haystack.git@main
-          hatch run test:all
+          hatch run test:cov-retry
 
       - name: Send event to Datadog for nightly failures
         if: failure() && github.event_name == 'schedule'

.github/workflows/nvidia.yml

@@ -74,7 +74,7 @@ jobs:
         run: |
           hatch env prune
           hatch -e test env run -- uv pip install git+https://github.com/deepset-ai/haystack.git@main
-          hatch run test:all
+          hatch run test:cov-retry
 
       - name: Send event to Datadog for nightly failures
         if: failure() && github.event_name == 'schedule'

.github/workflows/openrouter.yml

@@ -72,7 +72,7 @@ jobs:
         run: |
           hatch env prune
           hatch -e test env run -- uv pip install git+https://github.com/deepset-ai/haystack.git@main
-          hatch run test:all
+          hatch run test:cov-retry
 
       - name: Send event to Datadog for nightly failures
         if: failure() && github.event_name == 'schedule'

.github/workflows/stackit.yml

@@ -72,7 +72,7 @@ jobs:
         run: |
           hatch env prune
           hatch -e test env run -- uv pip install git+https://github.com/deepset-ai/haystack.git@main
-          hatch run test:all
+          hatch run test:cov-retry
 
       - name: Send event to Datadog for nightly failures
         if: failure() && github.event_name == 'schedule'

.github/workflows/togetherai.yml

@@ -72,7 +72,7 @@ jobs:
         run: |
           hatch env prune
           hatch -e test env run -- uv pip install git+https://github.com/deepset-ai/haystack.git@main
-          hatch run test:all
+          hatch run test:cov-retry
 
       - name: Send event to Datadog for nightly failures
         if: failure() && github.event_name == 'schedule'

integrations/llama_stack/pyproject.toml

@@ -60,7 +60,7 @@ dependencies = [
 [tool.hatch.envs.test.scripts]
 unit = 'pytest -m "not integration" {args:tests}'
 integration = 'pytest -m "integration" {args:tests}'
-all = 'pytest --reruns 3 --reruns-delay 30 -x {args:tests}'
+all = 'pytest {args:tests}'
 cov-retry = 'pytest --cov=haystack_integrations --reruns 3 --reruns-delay 30 -x {args:tests}'
 
 types = "mypy -p haystack_integrations.components.generators.llama_stack {args}"

integrations/meta_llama/pyproject.toml

@@ -63,7 +63,7 @@ dependencies = [
 [tool.hatch.envs.test.scripts]
 unit = 'pytest -m "not integration" {args:tests}'
 integration = 'pytest -m "integration" {args:tests}'
-all = 'pytest --reruns 3 --reruns-delay 30 -x {args:tests}'
+all = 'pytest {args:tests}'
 cov-retry = 'pytest --cov="haystack_integrations" --reruns 3 --reruns-delay 30 -x {args:tests}'
 
 types = "mypy -p haystack_integrations.components.generators.meta_llama {args}"

integrations/togetherai/pyproject.toml

@@ -65,7 +65,7 @@ dependencies = [
 [tool.hatch.envs.test.scripts]
 unit = 'pytest -m "not integration" {args:tests}'
 integration = 'pytest -m "integration" {args:tests}'
-all = 'pytest --reruns 3 --reruns-delay 30 -x {args:tests}'
+all = 'pytest {args:tests}'
 cov-retry = 'pytest --cov=haystack_integrations --reruns 3 --reruns-delay 30 -x {args:tests}'
 
 types = """mypy -p haystack_integrations.components.generators.togetherai {args}"""