feat: add reStructuredText support (.rst parser + multi-dialect prose chunker)

Adds end-to-end .rst handling plus the prose-collateral chunker (P0 of
docs/research/document_collateral_retrieval_2026.md):

- prose_chunker: dialect-agnostic structure-aware parent-child chunker
  (MarkdownDialect + RstDialect behind a ProseDialect Protocol). Heading-
  hierarchy sectioning, atomic code/table blocks, hard token bounding,
  small-sibling merge, faithful line ranges, no-LLM context headers.
- rst_parser: RstParser emits DOCUMENT/SECTION graph entities + CONTAINS,
  mirroring MarkdownParser; adornment headings leveled by encounter order.
- wiring: scanner supports .rst; graph_builder routes .rst -> RstParser;
  parsers package exports RstParser; .rst added to the parser contract.
- tests: prose chunker (17), RST dialect (14), RST parser (8); demo script
  for eyeballing chunk quality on a corpus.
- chore: ignore knowledge.db and .DS_Store.

Validated on a 915-doc corpus: p50=489 tokens, 80% in-band. 123 tests pass;
ruff + mypy clean.

Author: deepakdgupta1

.gitignore

@@ -212,4 +212,8 @@ docs_test/
 #aimodels.yaml
 .knowcode/
 knowcode_index/
-
+knowledge.db
+.DS_Store
+tests/test_sample-prose
+tests/.DS_Store
+.gitignore

scripts/prose_chunk_demo.py

@@ -0,0 +1,101 @@
+"""Demo / profiling runner for the structure-aware prose chunker (P0 prototype).
+
+Runs ``ProseChunker`` over a directory of markdown documents and prints corpus
+statistics plus a sample document's chunks, so chunk quality can be eyeballed.
+
+Usage::
+
+    uv run python scripts/prose_chunk_demo.py [ROOT] [--show-doc PATH] [--limit N]
+
+Example::
+
+    uv run python scripts/prose_chunk_demo.py tests/test_sample-prose
+"""
+
+from __future__ import annotations
+
+import statistics as st
+from pathlib import Path
+
+import click
+
+from knowcode.indexing.prose_chunker import ProseChunker, ProseChunkingConfig
+
+
+def _pct(values: list[int], p: float) -> int:
+    if not values:
+        return 0
+    ordered = sorted(values)
+    return ordered[min(len(ordered) - 1, int(len(ordered) * p))]
+
+
+def _print_corpus_stats(chunker: ProseChunker, files: list[Path], root: Path) -> Path:
+    """Print aggregate stats; return a clean representative doc to display."""
+    max_tokens = chunker.config.max_tokens
+    tokens: list[int] = []
+    per_doc: list[int] = []
+    flagged = total = 0
+    sample: Path | None = None
+
+    for path in files:
+        chunks = chunker.chunk_file(path, doc_id=str(path.relative_to(root)))
+        per_doc.append(len(chunks))
+        total += len(chunks)
+        for chunk in chunks:
+            tokens.append(chunk.token_count)
+            flagged += chunk.is_oversize
+        if sample is None and 6 <= len(chunks) <= 14 and not any(c.is_oversize for c in chunks):
+            sample = path
+
+    in_band = sum(1 for t in tokens if 128 <= t <= max_tokens)
+    click.echo(f"docs={len(files)}  total_chunks={total}")
+    click.echo(
+        f"chunks/doc: mean={st.mean(per_doc):.1f} p50={_pct(per_doc, .5)} "
+        f"p90={_pct(per_doc, .9)} max={max(per_doc)}"
+    )
+    click.echo(
+        f"chunk tokens: mean={st.mean(tokens):.0f} p50={_pct(tokens, .5)} "
+        f"p90={_pct(tokens, .9)} max={max(tokens)}"
+    )
+    click.echo(
+        f"within [128,{max_tokens}]: {in_band / len(tokens) * 100:.1f}%   "
+        f"flagged-oversize: {flagged / total * 100:.1f}% (blob/code that could not split cleanly)"
+    )
+    return sample or files[0]
+
+
+def _print_doc_chunks(chunker: ProseChunker, path: Path, root: Path, limit: int) -> None:
+    chunks = chunker.chunk_file(path, doc_id=str(path.relative_to(root)))
+    click.echo(f"\n=== {path.relative_to(root)}  ({len(chunks)} chunks) ===")
+    for chunk in chunks[:limit]:
+        crumb = " > ".join(chunk.heading_path) or "(preamble)"
+        flag = " [OVERSIZE]" if chunk.is_oversize else ""
+        click.echo(
+            f"\n[L{chunk.level}] {crumb}  | lines {chunk.start_line}-{chunk.end_line} "
+            f"| {chunk.token_count} tok{flag}"
+        )
+        click.echo(f"   ctx:    {chunk.context_header}")
+        click.echo(f"   parent: {chunk.parent_id}")
+        click.echo(f"   text:   {chunk.content[:140].replace(chr(10), ' ')}")
+
+
+@click.command()
+@click.argument("root", type=click.Path(exists=True, path_type=Path), default="tests/test_sample-prose")
+@click.option("--show-doc", type=click.Path(path_type=Path), default=None, help="Display chunks for one document.")
+@click.option("--limit", type=int, default=6, help="Chunks to display for the sample document.")
+@click.option("--target-tokens", type=int, default=ProseChunkingConfig.target_tokens)
+@click.option("--max-tokens", type=int, default=ProseChunkingConfig.max_tokens)
+def main(root: Path, show_doc: Path | None, limit: int, target_tokens: int, max_tokens: int) -> None:
+    chunker = ProseChunker(ProseChunkingConfig(target_tokens=target_tokens, max_tokens=max_tokens))
+    if show_doc is not None:
+        _print_doc_chunks(chunker, show_doc, show_doc.parent, limit)
+        return
+    files = sorted(p for p in root.rglob("*.md"))
+    if not files:
+        raise click.ClickException(f"no .md files under {root}")
+    sample = _print_corpus_stats(chunker, files, root)
+    _print_doc_chunks(chunker, sample, root, limit)
+
+
+if __name__ == "__main__":
+    main()

src/knowcode/indexing/graph_builder.py

@@ -5,7 +5,7 @@
 from typing import Optional
 
 from knowcode.data_models import Entity, ParseResult, Relationship
-from knowcode.parsers import MarkdownParser, PythonParser, YamlParser
+from knowcode.parsers import MarkdownParser, PythonParser, RstParser, YamlParser
 from knowcode.parsers.javascript_parser import JavaScriptParser
 from knowcode.parsers.typescript_parser import TypeScriptParser
 from knowcode.parsers.java_parser import JavaParser
@@ -25,6 +25,7 @@ def __init__(self) -> None:
         """Initialize the graph builder with parsers."""
         self.python_parser = PythonParser()
         self.markdown_parser = MarkdownParser()
+        self.rst_parser = RstParser()
         self.yaml_parser = YamlParser()
         self.js_parser = JavaScriptParser()
         self.ts_parser = TypeScriptParser()
@@ -101,6 +102,8 @@ def _parse_file(self, file_info: FileInfo) -> ParseResult:
             return self.python_parser.parse_file(file_info.path)
         elif file_info.extension == ".md":
             return self.markdown_parser.parse_file(file_info.path)
+        elif file_info.extension == ".rst":
+            return self.rst_parser.parse_file(file_info.path)
         elif file_info.extension in {".yaml", ".yml"}:
             return self.yaml_parser.parse_file(file_info.path)
         elif file_info.extension in {".js", ".jsx"}: