test: improve del_hf_env_vars fixture (#3428)

Author: anakin87

integrations/huggingface_api/tests/conftest.py

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+import os
 from pathlib import Path
 
 import pytest
@@ -13,15 +14,16 @@ def test_files_path():
 
 
 @pytest.fixture()
-def del_hf_env_vars(monkeypatch):
+def del_hf_env_vars_if_empty(monkeypatch):
     """
-    Delete Hugging Face environment variables for tests.
+    Delete Hugging Face environment variables for tests if empty.
 
     Prevents passing empty tokens to Hugging Face, which would cause API calls to fail.
     This is particularly relevant for PRs opened from forks, where secrets are not available
     and empty environment variables might be set instead of being removed.
 
     See https://github.com/deepset-ai/haystack/issues/8811 for more details.
     """
-    monkeypatch.delenv("HF_API_TOKEN", raising=False)
-    monkeypatch.delenv("HF_TOKEN", raising=False)
+    for var in ("HF_API_TOKEN", "HF_TOKEN"):
+        if not os.environ.get(var, "").strip():
+            monkeypatch.delenv(var, raising=False)

integrations/huggingface_api/tests/test_ranker.py

@@ -13,14 +13,14 @@
 
 
 class TestHuggingFaceTEIRanker:
-    def test_init(self, del_hf_env_vars):
+    def test_init(self, del_hf_env_vars_if_empty):
         """Test initialization with default and custom parameters"""
         # Default parameters
         ranker = HuggingFaceTEIRanker(url="https://api.my-tei-service.com")
         assert ranker.url == "https://api.my-tei-service.com"
         assert ranker.top_k == 10
         assert ranker.timeout == 30
-        assert not ranker.token.resolve_value()
+        assert ranker.token == Secret.from_env_var(["HF_API_TOKEN", "HF_TOKEN"], strict=False)
         assert ranker.max_retries == 3
         assert ranker.retry_status_codes is None
 
@@ -41,7 +41,7 @@ def test_init(self, del_hf_env_vars):
         assert ranker.max_retries == 5
         assert ranker.retry_status_codes == [500, 502, 503]
 
-    def test_to_dict(self, del_hf_env_vars):
+    def test_to_dict(self, del_hf_env_vars_if_empty):
         """Test serialization to dict with Secret token"""
         component = HuggingFaceTEIRanker(
             url="https://api.my-tei-service.com", top_k=5, timeout=30, max_retries=4, retry_status_codes=[500, 502]
@@ -60,7 +60,7 @@ def test_to_dict(self, del_hf_env_vars):
         assert data["init_parameters"]["max_retries"] == 4
         assert data["init_parameters"]["retry_status_codes"] == [500, 502]
 
-    def test_from_dict(self, del_hf_env_vars):
+    def test_from_dict(self, del_hf_env_vars_if_empty):
         """Test deserialization from dict with environment variable token"""
         data = {
             "type": "haystack_integrations.components.rankers.huggingface_api.ranker.HuggingFaceTEIRanker",
@@ -82,14 +82,14 @@ def test_from_dict(self, del_hf_env_vars):
         assert component.max_retries == 4
         assert component.retry_status_codes == [500, 502]
 
-    def test_empty_documents(self, del_hf_env_vars):
+    def test_empty_documents(self, del_hf_env_vars_if_empty):
         """Test that empty documents list returns empty result"""
         ranker = HuggingFaceTEIRanker(url="https://api.my-tei-service.com")
         result = ranker.run(query="test query", documents=[])
         assert result == {"documents": []}
 
     @patch("haystack_integrations.components.rankers.huggingface_api.ranker.request_with_retry")
-    def test_run_with_mock(self, mock_request, del_hf_env_vars):
+    def test_run_with_mock(self, mock_request, del_hf_env_vars_if_empty):
         """Test run method with mocked API response"""
         # Setup mock response
         mock_response = MagicMock(spec=httpx.Response)
@@ -137,7 +137,7 @@ def test_run_with_mock(self, mock_request, del_hf_env_vars):
         assert result["documents"][2].score == 0.75
 
     @patch("haystack_integrations.components.rankers.huggingface_api.ranker.request_with_retry")
-    def test_run_with_truncation_direction(self, mock_request, del_hf_env_vars):
+    def test_run_with_truncation_direction(self, mock_request, del_hf_env_vars_if_empty):
         """Test run method with truncation direction parameter"""
         # Setup mock response
         mock_response = MagicMock(spec=httpx.Response)
@@ -170,7 +170,7 @@ def test_run_with_truncation_direction(self, mock_request, del_hf_env_vars):
         )
 
     @patch("haystack_integrations.components.rankers.huggingface_api.ranker.request_with_retry")
-    def test_run_with_custom_top_k(self, mock_request, del_hf_env_vars):
+    def test_run_with_custom_top_k(self, mock_request, del_hf_env_vars_if_empty):
         """Test run method with custom top_k parameter"""
         # Setup mock response with 5 documents
         mock_response = MagicMock(spec=httpx.Response)
@@ -207,7 +207,7 @@ def test_run_with_custom_top_k(self, mock_request, del_hf_env_vars):
         assert result["documents"][1].content == "Document 3"
 
     @patch("haystack_integrations.components.rankers.huggingface_api.ranker.request_with_retry")
-    def test_run_deduplicates_documents(self, mock_request, del_hf_env_vars):
+    def test_run_deduplicates_documents(self, mock_request, del_hf_env_vars_if_empty):
         """Test that duplicate documents are removed before sending to the API."""
         mock_response = MagicMock(spec=httpx.Response)
         mock_response.json.return_value = [{"index": 1, "score": 0.9}, {"index": 0, "score": 0.2}]
@@ -228,7 +228,7 @@ def test_run_deduplicates_documents(self, mock_request, del_hf_env_vars):
             url="https://api.my-tei-service.com/rerank",
             json={"query": "test query", "texts": ["keep me", "unique"], "raw_scores": False},
             timeout=30,
-            headers={},
+            headers={"Authorization": f"Bearer {ranker.token.resolve_value()}"} if ranker.token.resolve_value() else {},
             attempts=3,
             status_codes_to_retry=None,
         )
@@ -237,7 +237,7 @@ def test_run_deduplicates_documents(self, mock_request, del_hf_env_vars):
         assert result["documents"][1].content == "keep me"
 
     @patch("haystack_integrations.components.rankers.huggingface_api.ranker.request_with_retry")
-    def test_error_handling(self, mock_request, del_hf_env_vars):
+    def test_error_handling(self, mock_request, del_hf_env_vars_if_empty):
         """Test error handling in the ranker"""
         # Setup mock response with error
         mock_response = MagicMock(spec=httpx.Response)
@@ -261,7 +261,7 @@ def test_error_handling(self, mock_request, del_hf_env_vars):
 
     @pytest.mark.asyncio
     @patch("haystack_integrations.components.rankers.huggingface_api.ranker.async_request_with_retry")
-    async def test_run_async_with_mock(self, mock_request, del_hf_env_vars):
+    async def test_run_async_with_mock(self, mock_request, del_hf_env_vars_if_empty):
         """Test run_async method with mocked API response"""
         # Setup mock response
         mock_response = MagicMock(spec=httpx.Response)
@@ -310,7 +310,7 @@ async def test_run_async_with_mock(self, mock_request, del_hf_env_vars):
 
     @pytest.mark.asyncio
     @patch("haystack_integrations.components.rankers.huggingface_api.ranker.async_request_with_retry")
-    async def test_run_async_deduplicates_documents(self, mock_request, del_hf_env_vars):
+    async def test_run_async_deduplicates_documents(self, mock_request, del_hf_env_vars_if_empty):
         """Test that duplicate documents are removed before sending to the API."""
         mock_response = MagicMock(spec=httpx.Response)
         mock_response.json.return_value = [{"index": 1, "score": 0.9}, {"index": 0, "score": 0.2}]
@@ -331,7 +331,7 @@ async def test_run_async_deduplicates_documents(self, mock_request, del_hf_env_v
             url="https://api.my-tei-service.com/rerank",
             json={"query": "test query", "texts": ["keep me", "unique"], "raw_scores": False},
             timeout=30,
-            headers={},
+            headers={"Authorization": f"Bearer {ranker.token.resolve_value()}"} if ranker.token.resolve_value() else {},
             attempts=3,
             status_codes_to_retry=None,
         )
@@ -341,7 +341,7 @@ async def test_run_async_deduplicates_documents(self, mock_request, del_hf_env_v
 
     @pytest.mark.asyncio
     @patch("haystack_integrations.components.rankers.huggingface_api.ranker.async_request_with_retry")
-    async def test_run_async_empty_documents(self, mock_request, del_hf_env_vars):
+    async def test_run_async_empty_documents(self, mock_request, del_hf_env_vars_if_empty):
         """Test run_async with empty documents list"""
         ranker = HuggingFaceTEIRanker(url="https://api.my-tei-service.com")
         result = await ranker.run_async(query="test query", documents=[])

integrations/transformers/tests/conftest.py

@@ -2,23 +2,26 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+import os
+
 import pytest
 from haystack.document_stores.in_memory import InMemoryDocumentStore
 
 
 @pytest.fixture()
-def del_hf_env_vars(monkeypatch):
+def del_hf_env_vars_if_empty(monkeypatch):
     """
-    Delete Hugging Face environment variables for tests.
+    Delete Hugging Face environment variables for tests if empty.
 
     Prevents passing empty tokens to Hugging Face, which would cause API calls to fail.
     This is particularly relevant for PRs opened from forks, where secrets are not available
     and empty environment variables might be set instead of being removed.
 
     See https://github.com/deepset-ai/haystack/issues/8811 for more details.
     """
-    monkeypatch.delenv("HF_API_TOKEN", raising=False)
-    monkeypatch.delenv("HF_TOKEN", raising=False)
+    for var in ("HF_API_TOKEN", "HF_TOKEN"):
+        if not os.environ.get(var, "").strip():
+            monkeypatch.delenv(var, raising=False)
 
 
 @pytest.fixture()

integrations/transformers/tests/test_chat_generator.py

@@ -256,7 +256,7 @@ def test_from_dict(self, model_info_mock, tools):
         }
 
     @patch("haystack_integrations.components.generators.transformers.chat.chat_generator.pipeline")
-    def test_warm_up(self, pipeline_mock, del_hf_env_vars):
+    def test_warm_up(self, pipeline_mock, del_hf_env_vars_if_empty):
         generator = TransformersChatGenerator(
             model="mistralai/Mistral-7B-Instruct-v0.2", task="text-generation", device=ComponentDevice.from_str("cpu")
         )
@@ -266,11 +266,14 @@ def test_warm_up(self, pipeline_mock, del_hf_env_vars):
         generator.warm_up()
 
         pipeline_mock.assert_called_once_with(
-            model="mistralai/Mistral-7B-Instruct-v0.2", task="text-generation", token=None, device="cpu"
+            model="mistralai/Mistral-7B-Instruct-v0.2",
+            task="text-generation",
+            token=generator.token.resolve_value(),
+            device="cpu",
         )
 
     @patch("haystack_integrations.components.generators.transformers.chat.chat_generator.pipeline")
-    def test_warm_up_with_tools(self, pipeline_mock, del_hf_env_vars):
+    def test_warm_up_with_tools(self, pipeline_mock, del_hf_env_vars_if_empty):
         """Test that warm_up() calls warm_up on tools and is idempotent."""
 
         # Create a mock tool that tracks if warm_up() was called
@@ -324,7 +327,7 @@ def warm_up(self):
         pipeline_mock.assert_called_once()
 
     @patch("haystack_integrations.components.generators.transformers.chat.chat_generator.pipeline")
-    def test_warm_up_with_no_tools(self, pipeline_mock, del_hf_env_vars):
+    def test_warm_up_with_no_tools(self, pipeline_mock, del_hf_env_vars_if_empty):
         """Test that warm_up() works when no tools are provided."""
 
         generator = TransformersChatGenerator(
@@ -349,7 +352,7 @@ def test_warm_up_with_no_tools(self, pipeline_mock, del_hf_env_vars):
         pipeline_mock.assert_called_once()
 
     @patch("haystack_integrations.components.generators.transformers.chat.chat_generator.pipeline")
-    def test_warm_up_with_multiple_tools(self, pipeline_mock, del_hf_env_vars):
+    def test_warm_up_with_multiple_tools(self, pipeline_mock, del_hf_env_vars_if_empty):
         """Test that warm_up() works with multiple tools."""
 
         # Track warm_up calls
@@ -507,7 +510,7 @@ def test_messages_conversion_is_called(self, mock_convert, model_info_mock):
 
     @pytest.mark.integration
     @pytest.mark.flaky(reruns=3, reruns_delay=10)
-    def test_live_run(self, del_hf_env_vars):
+    def test_live_run(self, del_hf_env_vars_if_empty):
         """Test live run with default behavior (no thinking)."""
         messages = [ChatMessage.from_user("Please create a summary about the following topic: Climate change")]
 
@@ -521,7 +524,7 @@ def test_live_run(self, del_hf_env_vars):
 
     @pytest.mark.integration
     @pytest.mark.flaky(reruns=3, reruns_delay=10)
-    def test_live_run_thinking(self, del_hf_env_vars):
+    def test_live_run_thinking(self, del_hf_env_vars_if_empty):
         """Test live run with enable_thinking=True."""
         messages = [ChatMessage.from_user("What is 2+2?")]
 
@@ -865,7 +868,7 @@ def mock_pipeline_call(*args, **kwargs):
     @pytest.mark.integration
     @pytest.mark.flaky(reruns=3, reruns_delay=10)
     @pytest.mark.asyncio
-    async def test_live_run_async_with_streaming(self, del_hf_env_vars):
+    async def test_live_run_async_with_streaming(self, del_hf_env_vars_if_empty):
         """Test async streaming with a live model."""
         streaming_chunks = []
 

integrations/transformers/tests/test_extractive_reader.py

@@ -640,7 +640,7 @@ def __init__(self):
     "haystack_integrations.components.readers.transformers.extractive_reader."
     "AutoModelForQuestionAnswering.from_pretrained"
 )
-def test_device_map_auto(mocked_automodel, _mocked_autotokenizer, del_hf_env_vars):
+def test_device_map_auto(mocked_automodel, _mocked_autotokenizer, del_hf_env_vars_if_empty):
     reader = TransformersExtractiveReader("deepset/roberta-base-squad2", model_kwargs={"device_map": "auto"})
     auto_device = ComponentDevice.resolve_device(None)
 
@@ -651,7 +651,9 @@ def __init__(self):
     mocked_automodel.return_value = MockedModel()
     reader.warm_up()
 
-    mocked_automodel.assert_called_once_with("deepset/roberta-base-squad2", token=None, device_map="auto")
+    mocked_automodel.assert_called_once_with(
+        "deepset/roberta-base-squad2", token=reader.token.resolve_value(), device_map="auto"
+    )
     assert reader.device == ComponentDevice.from_multiple(DeviceMap.from_hf({"": auto_device.to_hf()}))
 
 
@@ -660,7 +662,7 @@ def __init__(self):
     "haystack_integrations.components.readers.transformers.extractive_reader."
     "AutoModelForQuestionAnswering.from_pretrained"
 )
-def test_device_map_str(mocked_automodel, _mocked_autotokenizer, del_hf_env_vars):
+def test_device_map_str(mocked_automodel, _mocked_autotokenizer, del_hf_env_vars_if_empty):
     reader = TransformersExtractiveReader("deepset/roberta-base-squad2", model_kwargs={"device_map": "cpu:0"})
 
     class MockedModel:
@@ -670,7 +672,9 @@ def __init__(self):
     mocked_automodel.return_value = MockedModel()
     reader.warm_up()
 
-    mocked_automodel.assert_called_once_with("deepset/roberta-base-squad2", token=None, device_map="cpu:0")
+    mocked_automodel.assert_called_once_with(
+        "deepset/roberta-base-squad2", token=reader.token.resolve_value(), device_map="cpu:0"
+    )
     assert reader.device == ComponentDevice.from_multiple(DeviceMap.from_hf({"": "cpu:0"}))
 
 
@@ -679,7 +683,7 @@ def __init__(self):
     "haystack_integrations.components.readers.transformers.extractive_reader."
     "AutoModelForQuestionAnswering.from_pretrained"
 )
-def test_device_map_dict(mocked_automodel, _mocked_autotokenizer, del_hf_env_vars):
+def test_device_map_dict(mocked_automodel, _mocked_autotokenizer, del_hf_env_vars_if_empty):
     reader = TransformersExtractiveReader(
         "deepset/roberta-base-squad2", model_kwargs={"device_map": {"layer_1": 1, "classifier": "cpu"}}
     )
@@ -692,7 +696,9 @@ def __init__(self):
     reader.warm_up()
 
     mocked_automodel.assert_called_once_with(
-        "deepset/roberta-base-squad2", token=None, device_map={"layer_1": 1, "classifier": "cpu"}
+        "deepset/roberta-base-squad2",
+        token=reader.token.resolve_value(),
+        device_map={"layer_1": 1, "classifier": "cpu"},
     )
     assert reader.device == ComponentDevice.from_multiple(DeviceMap.from_hf({"layer_1": 1, "classifier": "cpu"}))
 
@@ -907,7 +913,7 @@ def test_deduplicate_by_overlap(
 
 
 @pytest.mark.integration
-def test_t5(del_hf_env_vars):
+def test_t5(del_hf_env_vars_if_empty):
     reader = TransformersExtractiveReader("sjrhuschlee/flan-t5-base-squad2")
     answers = reader.run(example_queries[0], example_documents[0], top_k=2)[
         "answers"
@@ -930,7 +936,7 @@ def test_t5(del_hf_env_vars):
 
 
 @pytest.mark.integration
-def test_roberta(del_hf_env_vars):
+def test_roberta(del_hf_env_vars_if_empty):
     reader = TransformersExtractiveReader("deepset/tinyroberta-squad2")
     answers = reader.run(example_queries[0], example_documents[0], top_k=2)[
         "answers"

integrations/transformers/tests/test_named_entity_extractor.py

@@ -101,7 +101,7 @@ def test_named_entity_extractor_serde():
         _ = TransformersNamedEntityExtractor.from_dict(serde_data)
 
 
-def test_to_dict_default(del_hf_env_vars):
+def test_to_dict_default(del_hf_env_vars_if_empty):
     component = TransformersNamedEntityExtractor(
         model="dslim/bert-base-NER",
         device=ComponentDevice.from_str("mps"),
@@ -144,7 +144,7 @@ def test_to_dict_with_parameters():
     }
 
 
-def test_named_entity_extractor_from_dict_no_default_parameters(del_hf_env_vars):
+def test_named_entity_extractor_from_dict_no_default_parameters(del_hf_env_vars_if_empty):
     data = {
         "type": COMPONENT_TYPE,
         "init_parameters": {"model": "dslim/bert-base-NER"},
@@ -226,15 +226,15 @@ def test_named_entity_extractor_run_fails_with_wrong_number_of_annotations():
 
 
 @pytest.mark.integration
-def test_ner_extractor_init(del_hf_env_vars):
+def test_ner_extractor_init(del_hf_env_vars_if_empty):
     extractor = TransformersNamedEntityExtractor(model="dslim/bert-base-NER")
     extractor.warm_up()
     assert extractor.initialized
 
 
 @pytest.mark.integration
 @pytest.mark.parametrize("batch_size", [1, 3])
-def test_ner_extractor(raw_texts, hf_annotations, batch_size, del_hf_env_vars):
+def test_ner_extractor(raw_texts, hf_annotations, batch_size, del_hf_env_vars_if_empty):
     extractor = TransformersNamedEntityExtractor(model="dslim/bert-base-NER")
     extractor.warm_up()
 
@@ -256,7 +256,7 @@ def test_ner_extractor_private_models(raw_texts, hf_annotations, batch_size):
 
 @pytest.mark.integration
 @pytest.mark.parametrize("batch_size", [1, 3])
-def test_ner_extractor_in_pipeline(raw_texts, hf_annotations, batch_size, del_hf_env_vars):
+def test_ner_extractor_in_pipeline(raw_texts, hf_annotations, batch_size, del_hf_env_vars_if_empty):
     pipeline = Pipeline()
     pipeline.add_component(
         name="ner_extractor",