fix(compiler): drop non-existent 'related' slugs so they don't create dangling links

The plan's 'related' list is meant to reference existing pages, but the LLM
sometimes lists slugs for pages that don't exist. Those were added to the
wikilink whitelist (so body references survived ghost-stripping) and
back-linked into the summary's Related section, yet no page was ever created
(related items are linked, never generated) — producing a flood of broken
[[concepts/...]] / [[entities/...]] links (esp. on feature-dense docs).
Filter related_items / entity_related to slugs that exist on disk.

Author: KylinMountain

openkb/agent/compiler.py

@@ -1444,6 +1444,22 @@ def _write_v1_summary_stripped() -> None:
     entity_update = entities_plan["update"]
     entity_related = entities_plan["related"]
 
+    # "related" must reference pages that ALREADY exist on disk (the plan
+    # prompt asks for existing slugs). The LLM sometimes lists non-existent
+    # slugs here; keeping them would whitelist [[concepts/...]] /
+    # [[entities/...]] links as valid AND back-link them into the summary, yet
+    # no page is ever created (related items are linked, never generated) —
+    # producing a flood of dangling wikilinks. Drop the non-existent ones so
+    # body references to them are stripped as ghosts instead.
+    related_items = [
+        s for s in related_items
+        if (wiki_dir / "concepts" / f"{_sanitize_concept_name(s)}.md").exists()
+    ]
+    entity_related = [
+        s for s in entity_related
+        if (wiki_dir / "entities" / f"{_sanitize_concept_name(s)}.md").exists()
+    ]
+
     # Distinguish "filters dropped everything" from "LLM emitted an empty plan".
     # Count entity items too, so a plan that emitted only entities — all of
     # which were dropped as malformed — still surfaces the warning.

tests/test_compiler.py

@@ -1773,6 +1773,49 @@ async def fake_llm_async(model, messages, label, **kw):
         assert "(organization)" in index, "index entry was downgraded from (organization) to (other)"
         assert "AI safety lab" in index, "index brief was stripped from the entry"
 
+    @pytest.mark.asyncio
+    async def test_related_to_nonexistent_concept_does_not_create_dangling_links(self, tmp_path, monkeypatch):
+        """A plan 'related' slug whose page does NOT exist must be dropped, not
+        whitelisted+back-linked — otherwise every page gets a dangling
+        [[concepts/<ghost>]] link to a page that is never created."""
+        wiki = tmp_path / "wiki"
+        (wiki / "summaries").mkdir(parents=True)
+        (wiki / "summaries" / "doc.md").write_text(
+            "---\nsources: []\n---\n\n# Doc\n", encoding="utf-8")
+
+        def fake_llm(model, messages, label, **kw):
+            if label == "concepts-plan":
+                return json.dumps({
+                    "concepts": {"create": [{"name": "real-concept", "title": "Real"}],
+                                 "update": [], "related": ["ghost-concept"]},
+                    "entities": {"create": [], "update": [], "related": []},
+                })
+            if label == "summary-rewrite":
+                return "# Doc\n\nSee [[concepts/real-concept]] and [[concepts/ghost-concept]].\n"
+            # concept generation body references the non-existent ghost concept
+            return json.dumps({"brief": "b", "content": "# Real\n\nLinks [[concepts/ghost-concept]].\n"})
+
+        async def fake_llm_async(model, messages, label, **kw):
+            return fake_llm(model, messages, label, **kw)
+
+        monkeypatch.setattr("openkb.agent.compiler._llm_call", fake_llm)
+        monkeypatch.setattr("openkb.agent.compiler._llm_call_async", fake_llm_async)
+
+        from openkb.agent.compiler import _compile_concepts
+        await _compile_concepts(wiki, tmp_path, "m", {"role": "system", "content": "x"},
+                                {"role": "user", "content": "x"}, "summary text", "doc",
+                                max_concurrency=2, doc_type="short", rewrite_summary=True)
+
+        # ghost-concept never existed and was only "related" → never created
+        assert not (wiki / "concepts" / "ghost-concept.md").exists()
+        # ...and no page should link to it (stripped as a ghost, since not whitelisted)
+        real = (wiki / "concepts" / "real-concept.md").read_text(encoding="utf-8")
+        assert "[[concepts/ghost-concept]]" not in real
+        summary = (wiki / "summaries" / "doc.md").read_text(encoding="utf-8")
+        assert "[[concepts/ghost-concept]]" not in summary
+        # the genuinely-created concept must still be linked
+        assert "[[concepts/real-concept]]" in summary
+
 
 # ---------------------------------------------------------------------------
 # Task 9: schema declares entities