Merge pull request #82 from GoodbyePlanet/fix/parser-error-purges-index-68

fix: preserve index entries when a parser fails instead of purging them

Author: GoodbyePlanet

server/indexer/pipeline.py

@@ -15,7 +15,7 @@
 from server.embeddings.bm25 import BM25SparseProvider, get_sparse_embedding_provider
 from server.embeddings import get_embedding_provider
 from server.indexer.github_source import fetch_blob_content, list_github_files
-from server.parser.base import CodeSymbol
+from server.parser.base import CodeSymbol, ParseError
 from server.parser.registry import parse_file
 from server.store.qdrant import QdrantStore
 
@@ -209,7 +209,16 @@ async def index_service(
                     logger.error("Failed to fetch %s: %s", stored_path, exc)
                     continue
 
-                symbols = parse_file(content, stored_path)
+                try:
+                    symbols = parse_file(content, stored_path)
+                except ParseError:
+                    logger.error(
+                        "Skipping index update for %s: parser failed, "
+                        "existing entries preserved",
+                        stored_path,
+                    )
+                    continue
+
                 if not symbols:
                     # File has no indexable symbols; clean up any stale entries.
                     await self._store.delete_by_file(svc.name, stored_path)

server/parser/base.py

@@ -10,6 +10,10 @@ def _node_text(node: Node, source: bytes) -> str:
     return source[node.start_byte : node.end_byte].decode("utf-8", errors="replace")
 
 
+class ParseError(Exception):
+    """Raised when a language parser fails on a file's content."""
+
+
 @dataclass
 class CodeSymbol:
     name: str

server/parser/registry.py

@@ -2,7 +2,7 @@
 
 import logging
 
-from server.parser.base import CodeSymbol, LanguageParser
+from server.parser.base import CodeSymbol, LanguageParser, ParseError
 from server.parser.bash import BashParser
 from server.parser.c import CParser
 from server.parser.compose import ComposeParser
@@ -109,6 +109,6 @@ def parse_file(source: bytes, file_path: str) -> list[CodeSymbol]:
         return []
     try:
         return parser.parse_file(source, file_path)
-    except Exception:
+    except Exception as exc:
         logger.exception("Parser %s failed for %s", type(parser).__name__, file_path)
-        return []
+        raise ParseError(file_path) from exc

tests/parser/test_registry.py

@@ -3,6 +3,7 @@
 import pytest
 
 from server.parser import registry
+from server.parser.base import ParseError
 from server.parser.bash import BashParser
 from server.parser.c import CParser
 from server.parser.compose import ComposeParser
@@ -99,7 +100,7 @@ def test_parse_file_returns_empty_for_unknown_extension() -> None:
     assert registry.parse_file(b"whatever", "svc/data.bin") == []
 
 
-def test_parse_file_swallows_parser_exceptions(monkeypatch) -> None:
+def test_parse_file_raises_parse_error_for_parser_exceptions(monkeypatch) -> None:
     class BoomParser:
         def parse_file(self, source, file_path):
             raise RuntimeError("boom")
@@ -110,7 +111,8 @@ def supported_extensions(self):
     monkeypatch.setattr(registry, "_PARSERS", {".boom": BoomParser()})
     monkeypatch.setattr(registry, "_FILENAME_PARSERS", {})
 
-    assert registry.parse_file(b"x", "svc/file.boom") == []
+    with pytest.raises(ParseError):
+        registry.parse_file(b"x", "svc/file.boom")
 
 
 def test_parse_file_dispatches_to_correct_parser(read_fixture) -> None:

tests/test_pipeline.py

@@ -1,13 +1,36 @@
 from __future__ import annotations
 
 import logging
-
-from server.indexer.pipeline import _build_bm25_text, _build_embedding_text
-from server.parser.base import CodeSymbol
+from unittest.mock import AsyncMock, patch
+
+import server.indexer.pipeline as pipeline_module
+from server.config import ServiceConfig
+from server.indexer.github_source import GitHubFile
+from server.indexer.pipeline import (
+    IndexPipeline,
+    _build_bm25_text,
+    _build_embedding_text,
+)
+from server.parser.base import CodeSymbol, ParseError
 
 _TRUNCATION_MARKER = "// ... (truncated)"
 
 
+class _StubEmbedder:
+    dimensions = 1
+
+    async def embed_batch(self, texts: list[str]) -> list[list[float]]:
+        return [[0.0]] * len(texts)
+
+
+def _make_pipeline(store) -> IndexPipeline:
+    pipeline = IndexPipeline.__new__(IndexPipeline)
+    pipeline._store = store
+    pipeline._embedder = _StubEmbedder()
+    pipeline._sparse_embedder = _StubEmbedder()
+    return pipeline
+
+
 def _sym_with_source(source: str, docstring: str = "") -> CodeSymbol:
     return CodeSymbol(
         name="fn",
@@ -127,3 +150,33 @@ def test_max_chars_is_configurable() -> None:
     sym = _sym_with_source("z" * 5_000)
     assert _TRUNCATION_MARKER not in _build_embedding_text(sym, "svc", max_chars=6000)
     assert _TRUNCATION_MARKER in _build_embedding_text(sym, "svc", max_chars=1000)
+
+
+async def test_parser_failure_preserves_existing_index_entries() -> None:
+    svc = ServiceConfig(name="svc", github_repo="org/repo", exclude=[])
+    github_file = GitHubFile(rel_path="broken.py", blob_sha="sha1")
+
+    store = AsyncMock()
+    store.get_indexed_file_hashes.return_value = {"svc/broken.py": "old-sha"}
+
+    pipeline = _make_pipeline(store)
+
+    with (
+        patch.object(
+            type(pipeline_module.settings), "load_services", return_value=[svc]
+        ),
+        patch.object(
+            pipeline_module, "list_github_files", AsyncMock(return_value=[github_file])
+        ),
+        patch.object(
+            pipeline_module, "fetch_blob_content", AsyncMock(return_value=b"garbage")
+        ),
+        patch.object(
+            pipeline_module, "parse_file", side_effect=ParseError("svc/broken.py")
+        ),
+    ):
+        result = await pipeline.index_service("svc", force=True)
+
+    store.delete_by_file.assert_not_called()
+    store.upsert_chunks.assert_not_called()
+    assert result["files"] == 0