Skip to content

Commit 6481724

Browse files
phernandezclaude
phernandez
and
claude
committed
fix(core): self-heal corrupt FastEmbed model cache
An interrupted FastEmbed model download leaves the HuggingFace snapshot dir present but missing model_optimized.onnx. The ONNX runtime then raises NO_SUCHFILE on every load, and the failure is self-perpetuating until the cache is cleared by hand. Search surfaced only the generic 'Search Failed' message with no hint. FastEmbedEmbeddingProvider now detects a missing/corrupt-artifact load failure, deletes only this model's own models--<org>--<repo> cache subtree (resolved from FastEmbed's model description), and retries the load exactly once to force a fresh download. A second failure fails fast with the original error. The search error formatter gains an ONNX/model-load branch that names the resolved fastembed cache dir to delete and offers search_type="text" as an immediate workaround. Closes #895 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Signed-off-by: phernandez <paul@basicmachines.co>
1 parent 1667cdc commit 6481724

4 files changed

Lines changed: 364 additions & 25 deletions

File tree

‎src/basic_memory/mcp/tools/search.py‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,44 @@ def _format_search_error_response(
8383
`search_notes("{project}", "{query}", search_type="{search_type}")`
8484
""").strip()
8585

86+
# Corrupt/missing FastEmbed model cache (interrupted download leaves a partial
87+
# snapshot missing model_optimized.onnx; the ONNX runtime then raises NO_SUCHFILE).
88+
# Basic Memory self-heals by re-downloading on the next load, but if the user still
89+
# hits this, point them at the cache dir to clear manually and offer a text fallback.
90+
error_lower = error_message.lower()
91+
if (
92+
"onnxruntime" in error_lower
93+
or "no_suchfile" in error_lower
94+
or "model_optimized.onnx" in error_lower
95+
or "load model" in error_lower
96+
):
97+
# Deferred import: keeps the repository layer out of the tool's import graph
98+
# (matches the SearchClient deferral below) and is only needed on this error path.
99+
from basic_memory.repository.embedding_provider_factory import _resolve_cache_dir
100+
101+
try:
102+
cache_dir = _resolve_cache_dir(get_container().config)
103+
except RuntimeError:
104+
cache_dir = _resolve_cache_dir(ConfigManager().config)
105+
return dedent(f"""
106+
# Search Failed - Embedding Model Missing or Corrupt
107+
108+
The local FastEmbed model could not be loaded for query '{query}': {error_message}
109+
110+
This usually means an earlier model download was interrupted and left an
111+
incomplete file in the model cache.
112+
113+
## How to fix
114+
1. Delete the FastEmbed model cache so it re-downloads on the next search:
115+
`{cache_dir}`
116+
2. Run your search again (the model downloads automatically on first use):
117+
`search_notes("{project}", "{query}", search_type="{search_type}")`
118+
119+
## Workaround right now
120+
- Use full-text search, which needs no embedding model:
121+
`search_notes("{project}", "{query}", search_type="text")`
122+
""").strip()
123+
86124
# FTS5 syntax errors
87125
if "syntax error" in error_message.lower() or "fts5" in error_message.lower():
88126
clean_query = (

‎src/basic_memory/repository/fastembed_provider.py‎

Lines changed: 116 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44

55
import asyncio
66
import math
7+
import shutil
8+
from pathlib import Path
79
from typing import TYPE_CHECKING
810

911
from loguru import logger
@@ -15,6 +17,19 @@
1517
from fastembed import TextEmbedding # pragma: no cover
1618

1719

20+
# Substrings that identify a missing/corrupt on-disk model artifact (as opposed to a
21+
# config error or a genuinely offline machine). An interrupted FastEmbed download leaves
22+
# the HuggingFace snapshot dir present but missing ``model_optimized.onnx``; the ONNX
23+
# runtime then raises ``NO_SUCHFILE`` and every subsequent load repeats it until the
24+
# cache is cleared. Matched case-insensitively against the exception text.
25+
_CORRUPT_MODEL_ERROR_MARKERS = (
26+
"no_suchfile",
27+
"model_optimized.onnx",
28+
"file doesn't exist",
29+
"no such file",
30+
)
31+
32+
1833
class FastEmbedEmbeddingProvider(EmbeddingProvider):
1934
"""Local ONNX embedding provider backed by FastEmbed."""
2035

@@ -53,6 +68,84 @@ def __init__(
5368
self._model: TextEmbedding | None = None
5469
self._model_lock = asyncio.Lock()
5570

71+
def _create_model(self) -> "TextEmbedding":
72+
try:
73+
from fastembed import TextEmbedding
74+
except ImportError as exc: # pragma: no cover - exercised via tests with monkeypatch
75+
raise SemanticDependenciesMissingError(
76+
"fastembed package is missing. "
77+
"Install/update basic-memory to include semantic dependencies: "
78+
"pip install -U basic-memory"
79+
) from exc
80+
resolved_model_name = self._MODEL_ALIASES.get(self.model_name, self.model_name)
81+
if self.cache_dir is not None and self.threads is not None:
82+
return TextEmbedding(
83+
model_name=resolved_model_name,
84+
cache_dir=self.cache_dir,
85+
threads=self.threads,
86+
)
87+
if self.cache_dir is not None:
88+
return TextEmbedding(model_name=resolved_model_name, cache_dir=self.cache_dir)
89+
if self.threads is not None:
90+
return TextEmbedding(model_name=resolved_model_name, threads=self.threads)
91+
return TextEmbedding(model_name=resolved_model_name)
92+
93+
def _model_cache_subdirs(self) -> list[Path]:
94+
"""Resolve the HuggingFace cache subdir(s) for this model under ``cache_dir``.
95+
96+
FastEmbed stores each model under ``<cache_dir>/models--<org>--<repo>`` where the
97+
repo is the model's HuggingFace source (e.g. ``BAAI/bge-small-en-v1.5`` resolves to
98+
``models--qdrant--bge-small-en-v1.5-onnx-q``). We resolve the source from FastEmbed's
99+
own model description so the deletion is scoped to exactly this model's tree — never
100+
the whole cache or unrelated models.
101+
"""
102+
if self.cache_dir is None:
103+
return []
104+
105+
resolved_model_name = self._MODEL_ALIASES.get(self.model_name, self.model_name)
106+
hf_sources: set[str] = set()
107+
try:
108+
from fastembed import TextEmbedding
109+
110+
for description in TextEmbedding._list_supported_models():
111+
if description.model == resolved_model_name:
112+
hf_source = description.sources.hf
113+
if hf_source:
114+
hf_sources.add(hf_source)
115+
except Exception as exc: # pragma: no cover - defensive: never block self-heal on lookup
116+
logger.warning(
117+
"Could not resolve FastEmbed model source for cache cleanup: "
118+
"model_name={model_name} error={error}",
119+
model_name=resolved_model_name,
120+
error=exc,
121+
)
122+
123+
cache_root = Path(self.cache_dir)
124+
# HuggingFace hub names cache dirs ``models--<repo with '/' -> '--'>``.
125+
return [cache_root / f"models--{source.replace('/', '--')}" for source in hf_sources]
126+
127+
def _purge_corrupt_model_cache(self) -> bool:
128+
"""Delete this model's on-disk cache subtree so the next load re-downloads it.
129+
130+
Returns True when at least one model cache subdir existed and was removed.
131+
"""
132+
removed = False
133+
for subdir in self._model_cache_subdirs():
134+
if subdir.exists():
135+
logger.warning(
136+
"Removing corrupt FastEmbed model cache to force re-download: {path}",
137+
path=str(subdir),
138+
)
139+
shutil.rmtree(subdir, ignore_errors=True)
140+
removed = True
141+
return removed
142+
143+
@staticmethod
144+
def _is_corrupt_model_error(exc: Exception) -> bool:
145+
"""Return True when the load failure looks like a missing/corrupt model artifact."""
146+
message = str(exc).lower()
147+
return any(marker in message for marker in _CORRUPT_MODEL_ERROR_MARKERS)
148+
56149
async def _load_model(self) -> "TextEmbedding":
57150
if self._model is not None:
58151
return self._model
@@ -61,31 +154,29 @@ async def _load_model(self) -> "TextEmbedding":
61154
if self._model is not None:
62155
return self._model
63156

64-
def _create_model() -> "TextEmbedding":
65-
try:
66-
from fastembed import TextEmbedding
67-
except (
68-
ImportError
69-
) as exc: # pragma: no cover - exercised via tests with monkeypatch
70-
raise SemanticDependenciesMissingError(
71-
"fastembed package is missing. "
72-
"Install/update basic-memory to include semantic dependencies: "
73-
"pip install -U basic-memory"
74-
) from exc
75-
resolved_model_name = self._MODEL_ALIASES.get(self.model_name, self.model_name)
76-
if self.cache_dir is not None and self.threads is not None:
77-
return TextEmbedding(
78-
model_name=resolved_model_name,
79-
cache_dir=self.cache_dir,
80-
threads=self.threads,
81-
)
82-
if self.cache_dir is not None:
83-
return TextEmbedding(model_name=resolved_model_name, cache_dir=self.cache_dir)
84-
if self.threads is not None:
85-
return TextEmbedding(model_name=resolved_model_name, threads=self.threads)
86-
return TextEmbedding(model_name=resolved_model_name)
87-
88-
self._model = await asyncio.to_thread(_create_model)
157+
try:
158+
self._model = await asyncio.to_thread(self._create_model)
159+
except Exception as exc:
160+
# Trigger: model construction failed with a missing/corrupt-artifact error
161+
# (an interrupted download left a partial snapshot in the cache).
162+
# Why: the raw ONNXRuntimeError is self-perpetuating — every retry hits the
163+
# same broken snapshot until the cache is cleared. Scope the deletion to
164+
# this model's own ``models--...`` subdir and retry exactly once so a
165+
# fresh download can land. A single retry avoids an infinite re-download
166+
# loop if the failure is not actually a cache problem.
167+
# Outcome: on success the user transparently recovers; on a second failure we
168+
# fail fast with the original error so the message stays actionable.
169+
if not self._is_corrupt_model_error(exc):
170+
raise
171+
if not self._purge_corrupt_model_cache():
172+
raise
173+
logger.info(
174+
"Retrying FastEmbed model load after clearing corrupt cache: "
175+
"model_name={model_name}",
176+
model_name=self._MODEL_ALIASES.get(self.model_name, self.model_name),
177+
)
178+
self._model = await asyncio.to_thread(self._create_model)
179+
89180
logger.info(
90181
"FastEmbed model loaded: model_name={model_name} batch_size={batch_size} "
91182
"threads={threads} configured_parallel={configured_parallel} "

‎tests/mcp/test_tool_search.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -479,6 +479,27 @@ def test_format_search_error_semantic_dependencies_missing(self):
479479
assert "# Search Failed - Semantic Dependencies Missing" in result
480480
assert "pip install -U basic-memory" in result
481481

482+
def test_format_search_error_corrupt_embedding_model(self):
483+
"""Test formatting for a corrupt/missing FastEmbed model (ONNX NO_SUCHFILE)."""
484+
from basic_memory.config import ConfigManager
485+
from basic_memory.repository.embedding_provider_factory import _resolve_cache_dir
486+
487+
result = _format_search_error_response(
488+
"test-project",
489+
"[ONNXRuntimeError] : 3 : NO_SUCHFILE : Load model from "
490+
"/home/u/.basic-memory/fastembed_cache/models--qdrant--bge-small-en-v1.5-onnx-q/"
491+
"snapshots/abc/model_optimized.onnx failed. File doesn't exist",
492+
"semantic query",
493+
"hybrid",
494+
)
495+
496+
expected_cache_dir = _resolve_cache_dir(ConfigManager().config)
497+
assert "# Search Failed - Embedding Model Missing or Corrupt" in result
498+
# Names the actual resolved cache dir so the user knows what to delete.
499+
assert expected_cache_dir in result
500+
# Offers full-text search as an immediate workaround.
501+
assert 'search_type="text"' in result
502+
482503
def test_format_search_error_generic(self):
483504
"""Test formatting for generic errors."""
484505
result = _format_search_error_response("test-project", "unknown error", "test query")

0 commit comments

Comments
 (0)