Skip to content

Commit 006485c

Browse files
authored
test(backends): fix vision Ollama tests failing in CI with 400 model does not support vision (#1185) (#1188)
* test(backends): fix vision Ollama tests failing in CI with 400 model does not support vision Structural payload tests (test_image_block_in_instruction, test_image_block_in_chat) were failing in CI because the m_session fixture called start_session() with no model_id, resolving to IBM_GRANITE_4_1_3B (granite4.1:3b) — a text-only model. Attaching images caused Ollama to reject the request with 400, preventing post_processing from running and the structural assertions from ever executing. Fixes are: 1. Add test/backends/conftest.py with a shared mock_ollama_backend fixture that constructs an OllamaModelBackend entirely offline (patches _check_ollama_server, _pull_ollama_model, ollama.Client, ollama.AsyncClient). No live server required. 2. Rewrite test_vision_ollama.py into three tiers: - Tier 1 (construction): pure ImageBlock unit tests, no model or server. - Tier 2 (structural payload): mocked offline tests that verify images are embedded correctly in the Ollama conversation payload. The _async_client property is mocked via PropertyMock at the class level so the mock is returned regardless of which event loop _run_async_in_thread creates in its background thread. Runs in CI unconditionally. - Tier 3 (dormant e2e): skipped until granite-vision-4.1 lands on Ollama; tracked in #1187. 3. Refactor test_ollama_unit.py to use the shared mock_ollama_backend fixture, removing the duplicated _make_backend() helper. Closes #1185. Assisted-by: Claude Code Signed-off-by: Nigel Jones <jonesn@uk.ibm.com> * test(backends): address code review findings for vision Ollama tests Remove dead code after unconditional pytest.skip() in vision_session — the availability check is now the sole gate, which auto-activates once granite-vision-4.1 lands on Ollama. Update conftest.py docstring to show the correct PropertyMock class-level patching pattern (instance assignment does not override the event-loop-keyed _async_client property). Compute ImageBlock.from_pil_image() once in test_image_block_in_chat instead of three times. Assisted-by: Claude Code Signed-off-by: Nigel Jones <jonesn@uk.ibm.com> --------- Signed-off-by: Nigel Jones <jonesn@uk.ibm.com>
1 parent 6b2132b commit 006485c

3 files changed

Lines changed: 219 additions & 106 deletions

File tree

test/backends/conftest.py

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
"""Shared fixtures for backend tests."""
2+
3+
from unittest.mock import MagicMock, patch
4+
5+
import pytest
6+
7+
from mellea.backends.ollama import OllamaModelBackend
8+
9+
10+
@pytest.fixture
11+
def mock_ollama_backend():
12+
"""Factory fixture: returns an OllamaModelBackend with all network calls patched out.
13+
14+
No live Ollama server or pulled model is required. The returned backend has
15+
real client objects replaced with MagicMocks, so subsequent tests can set
16+
attributes such as ``backend._async_client.chat`` to control behaviour.
17+
18+
Usage::
19+
20+
def test_something(mock_ollama_backend):
21+
backend = mock_ollama_backend(model_options={ModelOption.MAX_NEW_TOKENS: 5})
22+
# _async_client is an event-loop-keyed property; instance assignment won't
23+
# override it for tests that call through _run_async_in_thread. Patch at
24+
# the class level instead:
25+
mock_async = MagicMock()
26+
mock_async.chat = AsyncMock(return_value=canned_response)
27+
with patch.object(
28+
type(backend), "_async_client", new_callable=PropertyMock, return_value=mock_async
29+
):
30+
yield MelleaSession(backend)
31+
"""
32+
33+
def _make(
34+
model_id: str = "granite4.1:3b",
35+
model_options: dict | None = None,
36+
timeout: float | None = None,
37+
) -> OllamaModelBackend:
38+
with (
39+
patch.object(OllamaModelBackend, "_check_ollama_server", return_value=True),
40+
patch.object(OllamaModelBackend, "_pull_ollama_model", return_value=True),
41+
patch("mellea.backends.ollama.ollama.Client", return_value=MagicMock()),
42+
patch(
43+
"mellea.backends.ollama.ollama.AsyncClient", return_value=MagicMock()
44+
),
45+
):
46+
return OllamaModelBackend(
47+
model_id=model_id, model_options=model_options, timeout=timeout
48+
)
49+
50+
return _make

test/backends/test_ollama_unit.py

Lines changed: 8 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -15,25 +15,10 @@
1515
from mellea.stdlib.context import SimpleContext
1616

1717

18-
def _make_backend(
19-
model_options: dict | None = None, timeout: float | None = None
20-
) -> OllamaModelBackend:
21-
"""Return an OllamaModelBackend with all network calls patched."""
22-
with (
23-
patch.object(OllamaModelBackend, "_check_ollama_server", return_value=True),
24-
patch.object(OllamaModelBackend, "_pull_ollama_model", return_value=True),
25-
patch("mellea.backends.ollama.ollama.Client", return_value=MagicMock()),
26-
patch("mellea.backends.ollama.ollama.AsyncClient", return_value=MagicMock()),
27-
):
28-
return OllamaModelBackend(
29-
model_id="granite3.3:8b", model_options=model_options, timeout=timeout
30-
)
31-
32-
3318
@pytest.fixture
34-
def backend():
19+
def backend(mock_ollama_backend):
3520
"""Return an OllamaModelBackend with no pre-set model options."""
36-
return _make_backend()
21+
return mock_ollama_backend(model_id="granite3.3:8b")
3722

3823

3924
# --- Map consistency ---
@@ -73,9 +58,11 @@ def test_simplify_and_merge_remaps_num_predict(backend):
7358
assert result[ModelOption.MAX_NEW_TOKENS] == 128
7459

7560

76-
def test_simplify_and_merge_per_call_overrides_backend():
61+
def test_simplify_and_merge_per_call_overrides_backend(mock_ollama_backend):
7762
# Backend sets num_predict=128; per-call value of 256 must win.
78-
b = _make_backend(model_options={"num_predict": 128})
63+
b = mock_ollama_backend(
64+
model_id="granite3.3:8b", model_options={"num_predict": 128}
65+
)
7966
result = b._simplify_and_merge({"num_predict": 256})
8067
assert result[ModelOption.MAX_NEW_TOKENS] == 256
8168

@@ -189,9 +176,9 @@ def test_timeout_forwarded_to_sync_and_async_clients():
189176
assert async_kwargs.get("timeout") == 12.5
190177

191178

192-
def test_timeout_forwarded_to_new_async_clients_per_event_loop():
179+
def test_timeout_forwarded_to_new_async_clients_per_event_loop(mock_ollama_backend):
193180
"""Newly created AsyncClients (one per event loop) must inherit the timeout."""
194-
backend = _make_backend(timeout=7.0)
181+
backend = mock_ollama_backend(model_id="granite3.3:8b", timeout=7.0)
195182
with patch(
196183
"mellea.backends.ollama.ollama.AsyncClient", return_value=MagicMock()
197184
) as mock_async_client:

0 commit comments

Comments
 (0)