fix(core): keep response markdown while indexing body content

Signed-off-by: phernandez <paul@basicmachines.co>

Author: phernandez

src/basic_memory/api/v2/routers/knowledge_router.py

@@ -307,10 +307,12 @@ async def create_entity(
             if fast:
                 entity = await entity_service.fast_write_entity(data)
                 written_content = None
+                search_content = None
             else:
                 write_result = await entity_service.create_entity_with_content(data)
                 entity = write_result.entity
                 written_content = write_result.content
+                search_content = write_result.search_content
 
         if fast:
             with telemetry.scope(
@@ -332,7 +334,7 @@ async def create_entity(
                 action="create_entity",
                 phase="search_index",
             ):
-                await search_service.index_entity(entity, content=written_content)
+                await search_service.index_entity(entity, content=search_content)
             with telemetry.scope(
                 "api.knowledge.create_entity.vector_sync",
                 domain="knowledge",
@@ -360,6 +362,9 @@ async def create_entity(
             if fast:
                 content = await file_service.read_file_content(entity.file_path)
             else:
+                # Non-fast writes already captured the markdown in memory. Reuse it here
+                # instead of re-reading the file; format_on_save is the one config that can
+                # still make the persisted file diverge because write_file only returns a checksum.
                 content = written_content
         result = result.model_copy(update={"content": content})
 
@@ -429,17 +434,20 @@ async def update_entity_by_id(
             if fast:
                 entity = await entity_service.fast_write_entity(data, external_id=entity_id)
                 written_content = None
+                search_content = None
                 response.status_code = 200 if existing else 201
             else:
                 if existing:
                     write_result = await entity_service.update_entity_with_content(existing, data)
                     entity = write_result.entity
                     written_content = write_result.content
+                    search_content = write_result.search_content
                     response.status_code = 200
                 else:
                     write_result = await entity_service.create_entity_with_content(data)
                     entity = write_result.entity
                     written_content = write_result.content
+                    search_content = write_result.search_content
                     if entity.external_id != entity_id:
                         entity = await entity_repository.update(
                             entity.id,
@@ -475,7 +483,7 @@ async def update_entity_by_id(
                 action="update_entity",
                 phase="search_index",
             ):
-                await search_service.index_entity(entity, content=written_content)
+                await search_service.index_entity(entity, content=search_content)
             with telemetry.scope(
                 "api.knowledge.update_entity.vector_sync",
                 domain="knowledge",
@@ -503,6 +511,9 @@ async def update_entity_by_id(
             if fast:
                 content = await file_service.read_file_content(entity.file_path)
             else:
+                # Non-fast writes already captured the markdown in memory. Reuse it here
+                # instead of re-reading the file; format_on_save is the one config that can
+                # still make the persisted file diverge because write_file only returns a checksum.
                 content = written_content
         result = result.model_copy(update={"content": content})
 
@@ -582,6 +593,7 @@ async def edit_entity_by_id(
                         expected_replacements=data.expected_replacements,
                     )
                     written_content = None
+                    search_content = None
                 else:
                     identifier = entity.permalink or entity.file_path
                     write_result = await entity_service.edit_entity_with_content(
@@ -594,6 +606,7 @@ async def edit_entity_by_id(
                     )
                     updated_entity = write_result.entity
                     written_content = write_result.content
+                    search_content = write_result.search_content
 
             if fast:
                 with telemetry.scope(
@@ -615,7 +628,7 @@ async def edit_entity_by_id(
                     action="edit_entity",
                     phase="search_index",
                 ):
-                    await search_service.index_entity(updated_entity, content=written_content)
+                    await search_service.index_entity(updated_entity, content=search_content)
                 with telemetry.scope(
                     "api.knowledge.edit_entity.vector_sync",
                     domain="knowledge",
@@ -643,6 +656,9 @@ async def edit_entity_by_id(
                 if fast:
                     content = await file_service.read_file_content(updated_entity.file_path)
                 else:
+                    # Non-fast writes already captured the markdown in memory. Reuse it here
+                    # instead of re-reading the file; format_on_save is the one config that can
+                    # still make the persisted file diverge because write_file only returns a checksum.
                     content = written_content
             result = result.model_copy(update={"content": content})
 

src/basic_memory/services/entity_service.py

@@ -53,10 +53,11 @@
 
 @dataclass(frozen=True)
 class EntityWriteResult:
-    """Persisted entity plus the markdown written during this call."""
+    """Persisted entity plus the response/search content produced during this call."""
 
     entity: EntityModel
     content: str
+    search_content: str
 
 
 class EntityService(BaseService[EntityModel]):
@@ -334,7 +335,11 @@ async def create_entity_with_content(self, schema: EntitySchema) -> EntityWriteR
             updated = await self.repository.update(entity.id, {"checksum": checksum})
         if not updated:  # pragma: no cover
             raise ValueError(f"Failed to update entity checksum after create: {entity.id}")
-        return EntityWriteResult(entity=updated, content=final_content)
+        return EntityWriteResult(
+            entity=updated,
+            content=final_content,
+            search_content=remove_frontmatter(final_content),
+        )
 
     async def update_entity(self, entity: EntityModel, schema: EntitySchema) -> EntityModel:
         """Update an entity's content and metadata."""
@@ -459,7 +464,11 @@ async def update_entity_with_content(
         if not entity:  # pragma: no cover
             raise ValueError(f"Failed to update entity checksum after update: {file_path}")
 
-        return EntityWriteResult(entity=entity, content=final_content)
+        return EntityWriteResult(
+            entity=entity,
+            content=final_content,
+            search_content=remove_frontmatter(final_content),
+        )
 
     async def fast_write_entity(
         self,
@@ -1093,7 +1102,11 @@ async def edit_entity_with_content(
         if not entity:  # pragma: no cover
             raise ValueError(f"Failed to update entity checksum after edit: {file_path}")
 
-        return EntityWriteResult(entity=entity, content=new_content)
+        return EntityWriteResult(
+            entity=entity,
+            content=new_content,
+            search_content=remove_frontmatter(new_content),
+        )
 
     def apply_edit_operation(
         self,

tests/api/v2/test_knowledge_router_telemetry.py

@@ -59,10 +59,15 @@ async def test_create_entity_emits_root_and_nested_spans(monkeypatch) -> None:
     monkeypatch.setattr(knowledge_router_module.telemetry, "span", fake_span)
 
     entity = _fake_entity()
+    response_content = "---\ntitle: Telemetry Entity\ntype: note\npermalink: notes/test\n---\n\ntelemetry content"
 
     class FakeEntityService:
         async def create_entity_with_content(self, data):
-            return SimpleNamespace(entity=entity, content="telemetry content")
+            return SimpleNamespace(
+                entity=entity,
+                content=response_content,
+                search_content="telemetry content",
+            )
 
     class FakeSearchService:
         async def index_entity(self, entity, content=None):
@@ -95,7 +100,7 @@ async def read_file_content(self, path):
         fast=False,
     )
 
-    assert result.content == "telemetry content"
+    assert result.content == response_content
     _assert_names_in_order(
         [name for name, _ in spans],
         [
@@ -114,10 +119,17 @@ async def test_update_entity_emits_root_and_nested_spans(monkeypatch) -> None:
     monkeypatch.setattr(knowledge_router_module.telemetry, "span", fake_span)
 
     entity = _fake_entity()
+    response_content = (
+        "---\ntitle: Telemetry Entity\ntype: note\npermalink: notes/test\n---\n\nupdated telemetry content"
+    )
 
     class FakeEntityService:
         async def update_entity_with_content(self, existing, data):
-            return SimpleNamespace(entity=entity, content="updated telemetry content")
+            return SimpleNamespace(
+                entity=entity,
+                content=response_content,
+                search_content="updated telemetry content",
+            )
 
     class FakeSearchService:
         async def index_entity(self, entity, content=None):
@@ -158,7 +170,7 @@ async def read_file_content(self, path):
         fast=False,
     )
 
-    assert result.content == "updated telemetry content"
+    assert result.content == response_content
     _assert_names_in_order(
         [name for name, _ in spans],
         [
@@ -178,10 +190,17 @@ async def test_edit_entity_emits_root_and_nested_spans(monkeypatch) -> None:
     monkeypatch.setattr(knowledge_router_module.telemetry, "span", fake_span)
 
     entity = _fake_entity()
+    response_content = (
+        "---\ntitle: Telemetry Entity\ntype: note\npermalink: notes/test\n---\n\nedited telemetry content"
+    )
 
     class FakeEntityService:
         async def edit_entity_with_content(self, **kwargs):
-            return SimpleNamespace(entity=entity, content="edited telemetry content")
+            return SimpleNamespace(
+                entity=entity,
+                content=response_content,
+                search_content="edited telemetry content",
+            )
 
     class FakeSearchService:
         async def index_entity(self, entity, content=None):
@@ -214,7 +233,7 @@ async def read_file_content(self, path):
         fast=False,
     )
 
-    assert result.content == "edited telemetry content"
+    assert result.content == response_content
     _assert_names_in_order(
         [name for name, _ in spans],
         [

tests/services/test_entity_service_write_result.py

@@ -0,0 +1,86 @@
+"""Tests for EntityWriteResult content variants."""
+
+import pytest
+
+from basic_memory.file_utils import remove_frontmatter
+from basic_memory.schemas import Entity as EntitySchema
+
+
+@pytest.mark.asyncio
+async def test_create_entity_with_content_returns_full_and_search_content(
+    entity_service, file_service
+) -> None:
+    result = await entity_service.create_entity_with_content(
+        EntitySchema(
+            title="Create Write Result",
+            directory="notes",
+            note_type="note",
+            content="Create body content",
+        )
+    )
+
+    file_path = file_service.get_entity_path(result.entity)
+    file_content, _ = await file_service.read_file(file_path)
+
+    assert result.content == file_content
+    assert result.search_content == remove_frontmatter(file_content)
+    assert result.search_content == "Create body content"
+
+
+@pytest.mark.asyncio
+async def test_update_entity_with_content_returns_full_and_search_content(
+    entity_service, file_service
+) -> None:
+    created = await entity_service.create_entity(
+        EntitySchema(
+            title="Update Write Result",
+            directory="notes",
+            note_type="note",
+            content="Original body content",
+        )
+    )
+
+    result = await entity_service.update_entity_with_content(
+        created,
+        EntitySchema(
+            title="Update Write Result",
+            directory="notes",
+            note_type="note",
+            content="Updated body content",
+        ),
+    )
+
+    file_path = file_service.get_entity_path(result.entity)
+    file_content, _ = await file_service.read_file(file_path)
+
+    assert result.content == file_content
+    assert result.search_content == remove_frontmatter(file_content)
+    assert result.search_content == "Updated body content"
+
+
+@pytest.mark.asyncio
+async def test_edit_entity_with_content_returns_full_and_search_content(
+    entity_service, file_service
+) -> None:
+    created = await entity_service.create_entity(
+        EntitySchema(
+            title="Edit Write Result",
+            directory="notes",
+            note_type="note",
+            content="Original body content",
+        )
+    )
+
+    result = await entity_service.edit_entity_with_content(
+        identifier=created.permalink,
+        operation="find_replace",
+        content="Edited body content",
+        find_text="Original body content",
+    )
+
+    file_path = file_service.get_entity_path(result.entity)
+    file_content, _ = await file_service.read_file(file_path)
+
+    assert result.content == file_content
+    assert result.search_content == remove_frontmatter(file_content)
+    assert result.search_content == "Edited body content"