fix: remove-preview detects JSON-quoted sources; _write_entity preserves sources on malformed FM

- remove --dry-run preview parsed the sources list with a hand-rolled comma
  split that kept JSON quotes (["summaries/x.md"]), so the marker never
  matched and the preview always reported 0 affected concept/entity pages
  (executor was correct). Extract _scan_affected_pages using the real
  _parse_yaml_list_value; dedups the two copied scan loops too.
- _write_entity's malformed-frontmatter rebuild seeded sources with only the
  new doc, dropping prior sources for multi-source entities. Recover existing
  sources from the broken block and merge.
Both bugs were masked by tests using unquoted / single-source fixtures.

Author: KylinMountain

openkb/agent/compiler.py

@@ -893,11 +893,18 @@ def _build_frontmatter(sources: list[str]) -> str:
         else:
             # Malformed/absent frontmatter (opening ``---`` with no closing
             # delimiter, or no frontmatter at all): rebuild valid frontmatter
-            # rather than writing a body-only page and dropping sources/type/
-            # brief. ``_prepend_source_to_frontmatter`` already ensured the
-            # new source is present in the (still-malformed) block, so seed
-            # with it here.
-            existing = _build_frontmatter([source_file]) + clean
+            # rather than writing a body-only page. Recover any sources already
+            # listed in the broken block first — otherwise a multi-source
+            # entity would be truncated to just this document.
+            recovered: list[str] = []
+            for ln in existing.split("\n"):
+                if ln.lstrip().startswith("sources:"):
+                    parsed = _parse_yaml_list_value(ln)
+                    if parsed:
+                        recovered = parsed
+                    break
+            merged = [source_file] + [s for s in recovered if s != source_file]
+            existing = _build_frontmatter(merged) + clean
         path.write_text(existing, encoding="utf-8")
         return
 

openkb/cli.py

@@ -739,6 +739,37 @@ def _cleanup_pageindex(
     return True, f"deleted PageIndex doc ({doc_id[:12]}…)"
 
 
+def _scan_affected_pages(pages_dir: Path, source_file_marker: str) -> list[tuple[str, int]]:
+    """Return ``(slug, remaining_sources)`` for pages whose frontmatter
+    ``sources:`` list contains ``source_file_marker``.
+
+    Uses the same ``_parse_yaml_list_value`` parser the executor uses, so
+    JSON-quoted values (``sources: ["summaries/x.md"]`` — exactly how the
+    compiler writes them) are matched correctly. A hand-rolled comma-split
+    here previously kept the surrounding quotes, so the marker never matched
+    and the remove preview silently reported 0 affected pages.
+    """
+    from openkb.agent.compiler import _parse_yaml_list_value
+
+    affected: list[tuple[str, int]] = []
+    if not pages_dir.is_dir():
+        return affected
+    for path in sorted(pages_dir.glob("*.md")):
+        text = path.read_text(encoding="utf-8")
+        if not text.startswith("---"):
+            continue
+        fm_end = text.find("---", 3)
+        if fm_end == -1:
+            continue
+        for line in text[:fm_end].split("\n"):
+            if line.lstrip().startswith("sources:"):
+                items = _parse_yaml_list_value(line)
+                if items is not None and source_file_marker in items:
+                    affected.append((path.stem, max(len(items) - 1, 0)))
+                break
+    return affected
+
+
 def _resolve_doc_identifier(registry, identifier: str) -> list[tuple[str, dict]]:
     """Find registry entries matching ``identifier``.
 
@@ -861,31 +892,7 @@ def remove(ctx, identifier, keep_raw, keep_empty_concepts, dry_run, yes):
     # affect the delete/edit classification, so the plan reflects what
     # the executor will actually do.
     source_file_marker = f"summaries/{doc_name}.md"
-    affected_concepts: list[tuple[str, int]] = []  # (slug, remaining_sources)
-    concepts_dir = wiki_dir / "concepts"
-    if concepts_dir.is_dir():
-        for path in sorted(concepts_dir.glob("*.md")):
-            text = path.read_text(encoding="utf-8")
-            if not text.startswith("---"):
-                continue
-            fm_end = text.find("---", 3)
-            if fm_end == -1:
-                continue
-            sources_count = 0
-            source_in_frontmatter = False
-            for line in text[:fm_end].split("\n"):
-                if line.lstrip().startswith("sources:"):
-                    lb = line.find("[")
-                    rb = line.rfind("]")
-                    if lb != -1 and rb != -1 and rb > lb:
-                        items = [s.strip() for s in line[lb + 1:rb].split(",") if s.strip()]
-                        sources_count = len(items)
-                        source_in_frontmatter = source_file_marker in items
-                    break
-            if not source_in_frontmatter:
-                continue
-            remaining = max(sources_count - 1, 0)
-            affected_concepts.append((path.stem, remaining))
+    affected_concepts = _scan_affected_pages(wiki_dir / "concepts", source_file_marker)
 
     concept_deletes = [s for s, r in affected_concepts if r == 0 and not keep_empty_concepts]
     concept_edits = [s for s, r in affected_concepts if r > 0 or keep_empty_concepts]
@@ -897,31 +904,7 @@ def remove(ctx, identifier, keep_raw, keep_empty_concepts, dry_run, yes):
     # Scan entity pages with the same frontmatter logic as concepts. The
     # executor calls ``remove_doc_from_entity_pages``; this only makes the
     # preview/summary truthful about what it will delete vs. edit.
-    affected_entities: list[tuple[str, int]] = []  # (slug, remaining_sources)
-    entities_dir = wiki_dir / "entities"
-    if entities_dir.is_dir():
-        for path in sorted(entities_dir.glob("*.md")):
-            text = path.read_text(encoding="utf-8")
-            if not text.startswith("---"):
-                continue
-            fm_end = text.find("---", 3)
-            if fm_end == -1:
-                continue
-            sources_count = 0
-            source_in_frontmatter = False
-            for line in text[:fm_end].split("\n"):
-                if line.lstrip().startswith("sources:"):
-                    lb = line.find("[")
-                    rb = line.rfind("]")
-                    if lb != -1 and rb != -1 and rb > lb:
-                        items = [s.strip() for s in line[lb + 1:rb].split(",") if s.strip()]
-                        sources_count = len(items)
-                        source_in_frontmatter = source_file_marker in items
-                    break
-            if not source_in_frontmatter:
-                continue
-            remaining = max(sources_count - 1, 0)
-            affected_entities.append((path.stem, remaining))
+    affected_entities = _scan_affected_pages(wiki_dir / "entities", source_file_marker)
 
     entity_deletes = [s for s, r in affected_entities if r == 0 and not keep_empty_concepts]
     entity_edits = [s for s, r in affected_entities if r > 0 or keep_empty_concepts]

tests/test_compiler.py

@@ -603,6 +603,8 @@ def test_update_rebuilds_frontmatter_when_no_closing_delim(self, tmp_path):
         assert text.startswith("---\n")
         assert text.count("---") == 2
         assert "sources:" in text and "summaries/b.md" in text
+        # The PRE-EXISTING source must be preserved, not dropped when rebuilding.
+        assert "summaries/a.md" in text
         assert "type:" in text and "organization" in text
         assert "brief:" in text and "AI lab." in text
         assert "v2 rewritten." in text

tests/test_remove.py

@@ -418,6 +418,31 @@ def test_cli_remove_preview_lists_entity_actions(kb_dir):
     assert (kb_dir / "wiki" / "entities" / "vaswani.md").exists()
 
 
+def test_cli_remove_preview_handles_json_quoted_sources(kb_dir):
+    """Regression: the real compiler writes sources JSON-quoted
+    (sources: ["summaries/x.md"]). The old preview parser comma-split the line
+    keeping the quotes, so the marker never matched and the preview silently
+    reported 0 affected pages even though the executor would delete/edit them."""
+    _seed_two_doc_kb(kb_dir)
+    (kb_dir / "wiki" / "entities").mkdir(parents=True)
+    # JSON-quoted single source (exactly how _yaml_list_line writes it) -> DELETE
+    (kb_dir / "wiki" / "entities" / "vaswani.md").write_text(
+        '---\nsources: ["summaries/attention-h_a.md"]\ntype: person\nbrief: V\n---\n# Vaswani\n',
+        encoding="utf-8",
+    )
+    # JSON-quoted multi-source concept -> MODIFY
+    (kb_dir / "wiki" / "concepts" / "quoted-concept.md").write_text(
+        '---\nsources: ["summaries/attention-h_a.md", "summaries/llm-h_l.md"]\nbrief: Q\n---\n# Q\n',
+        encoding="utf-8",
+    )
+
+    result = _invoke(kb_dir, ["remove", "attention.pdf", "--dry-run"])
+
+    assert result.exit_code == 0, result.output
+    assert "DELETE   wiki/entities/vaswani.md" in result.output
+    assert "MODIFY   wiki/concepts/quoted-concept.md" in result.output
+
+
 def test_cli_remove_yes_executes_full_plan(kb_dir):
     _seed_two_doc_kb(kb_dir)
     result = _invoke(kb_dir, ["remove", "attention.pdf", "--yes"])