diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index a1f5eb5..e077953 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -34,7 +34,11 @@ session, `--log-dir DIR` writes a transcript per source, `--quiet` drops the liv `--jobs N`/`-j` folds N sources in CONCURRENTLY (default 1 = serial; `CITADEL_JOBS`), `--full-rescan` distrusts the manifest's stat cache and re-hashes every tracked source, `--force ` deliberately re-reads already-ingested sources as a reconcile — it requires -explicit paths and is refused without them), `refresh [--limit N] [--min-age-days D] [--dry-run] [--jobs N]` +explicit paths and is refused without them, `--retry` re-runs everything STUCK without naming +paths: every failed source still on disk plus every ingested source no wiki page cites — the +zero-entry sources the run report flags as `no_pages` and `status` marks `NO PAGES` — as forced +reconciles; it refuses paths/`--force` and exits 0 when nothing is stuck), +`refresh [--limit N] [--min-age-days D] [--dry-run] [--jobs N]` (the THIRD lifecycle: re-verify the least-recently-checked sources — ordered by the manifest's `ingested_at` stamp, oldest/stampless first — through forced reconcile sessions on an explicit per-run budget of N sources; the sustainable alternative to regenerating the wiki after a model @@ -342,13 +346,15 @@ is scoped to that signature (flags actually passed + an auth-shaped message), so problem still fails instead of looping. **Status is the read-only corpus view** (`status.py`, `citadel status`): the manifest + failures -catalog + one stat-only walk (never re-hashes) rendered as a per-source state table — ingested +catalog + one stat-only walk (never re-hashes) + one wiki traversal rendered as a per-source state +table — ingested (model + rules_version, `(stale)` when it predates the current rulebook, `checked YYYY-MM-DD` from the `ingested_at` stamp, the last session's cost when recorded, with copilot's AI credits shown beside the dollars they converted into — with `Recorded LLM cost` / `Recorded AI credits` corpus -totals above the table), failed (reason, attempts), +totals above the table, and a `NO PAGES (nothing cites this source)` marker on every ingested +source the wiki does not cite — a paid session that produced zero entries), failed (reason, attempts), skipped-duplicate, ignored (pattern), oversized (over `CITADEL_MAX_SOURCE_BYTES`, with the size), -pending. +pending — closing with a `citadel ingest --retry` hint whenever anything is failed or uncited. **Other modules:** `okf.py` is the OKF format core (parse/dump, type→folder routing, link math, and the non-negotiable `safe_join` path guard — reuse it for any wiki-relative path). `grammar.py` is diff --git a/CHANGELOG.md b/CHANGELOG.md index fed115a..181a965 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,18 @@ All notable changes to this project are documented here. The format is based on ### Added +- **Zero-page and failed sources are visible — and retryable with one command.** A brand-new + source whose session "succeeds" with zero page changes used to be marked done and never looked + at again, indistinguishable from a healthy ingest. Now: the run report carries a WARNING section + (`IngestReport.no_pages`) plus a `log.md` line for each such source, the live progress renders + its `no changes` verdict in yellow, and `citadel status` marks every ingested source that NO + wiki page cites with `NO PAGES (nothing cites this source)` (also `"uncited"` in `--json`), + ending with a retry hint whenever anything is stuck. The new **`citadel ingest --retry`** + re-runs the whole stuck set without naming paths: every failed source still on disk (errored / + timed-out / unreadable — deliberate `duplicate` skips excluded) plus every ingested-but-uncited + source as a forced reconcile. It prints the computed set first, refuses `--force`/explicit + paths, and exits 0 cleanly when there is nothing to retry. The failed-run report also points at + the retry lane from its Errors section. - **`citadel doctor` warns when `CITADEL_INGEST_MODEL` shadows `COPILOT_MODEL`.** The copilot backend also reads its own `COPILOT_MODEL` env var, and the `--model` flag citadel passes overrides it. A user who points copilot at a BYOK provider (`COPILOT_PROVIDER_BASE_URL`, a diff --git a/CLAUDE.md b/CLAUDE.md index de4c71a..bb59dd6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -31,7 +31,11 @@ session, `--log-dir DIR` writes a transcript per source, `--quiet` drops the liv `--jobs N`/`-j` folds N sources in CONCURRENTLY (default 1 = serial; `CITADEL_JOBS`), `--full-rescan` distrusts the manifest's stat cache and re-hashes every tracked source, `--force ` deliberately re-reads already-ingested sources as a reconcile — it requires -explicit paths and is refused without them), `refresh [--limit N] [--min-age-days D] [--dry-run] [--jobs N]` +explicit paths and is refused without them, `--retry` re-runs everything STUCK without naming +paths: every failed source still on disk plus every ingested source no wiki page cites — the +zero-entry sources the run report flags as `no_pages` and `status` marks `NO PAGES` — as forced +reconciles; it refuses paths/`--force` and exits 0 when nothing is stuck), +`refresh [--limit N] [--min-age-days D] [--dry-run] [--jobs N]` (the THIRD lifecycle: re-verify the least-recently-checked sources — ordered by the manifest's `ingested_at` stamp, oldest/stampless first — through forced reconcile sessions on an explicit per-run budget of N sources; the sustainable alternative to regenerating the wiki after a model @@ -339,13 +343,15 @@ is scoped to that signature (flags actually passed + an auth-shaped message), so problem still fails instead of looping. **Status is the read-only corpus view** (`status.py`, `citadel status`): the manifest + failures -catalog + one stat-only walk (never re-hashes) rendered as a per-source state table — ingested +catalog + one stat-only walk (never re-hashes) + one wiki traversal rendered as a per-source state +table — ingested (model + rules_version, `(stale)` when it predates the current rulebook, `checked YYYY-MM-DD` from the `ingested_at` stamp, the last session's cost when recorded, with copilot's AI credits shown beside the dollars they converted into — with `Recorded LLM cost` / `Recorded AI credits` corpus -totals above the table), failed (reason, attempts), +totals above the table, and a `NO PAGES (nothing cites this source)` marker on every ingested +source the wiki does not cite — a paid session that produced zero entries), failed (reason, attempts), skipped-duplicate, ignored (pattern), oversized (over `CITADEL_MAX_SOURCE_BYTES`, with the size), -pending. +pending — closing with a `citadel ingest --retry` hint whenever anything is failed or uncited. **Other modules:** `okf.py` is the OKF format core (parse/dump, type→folder routing, link math, and the non-negotiable `safe_join` path guard — reuse it for any wiki-relative path). `grammar.py` is diff --git a/citadel/cli.py b/citadel/cli.py index efe0f70..d70729a 100644 --- a/citadel/cli.py +++ b/citadel/cli.py @@ -116,6 +116,15 @@ def build_parser() -> argparse.ArgumentParser: "without them, so a whole-corpus re-read (one agent session per source) can never " "happen by accident.", ) + p_ingest.add_argument( + "--retry", + action="store_true", + help="Retry everything stuck, without naming paths: re-run every FAILED source " + "(errored/timed-out/unreadable, from the failures catalog) that is still on disk, and " + "force-reconcile every ingested source that NO wiki page cites (it produced 0 entries — " + "the `NO PAGES` marker in `citadel status`). Prints the retry set first; refuses " + "explicit paths and --force (use those for a hand-picked re-read).", + ) p_ingest.add_argument( "--jobs", "-j", @@ -413,6 +422,12 @@ def cmd_ingest(args: argparse.Namespace) -> int: agent session per source, so forcing the ENTIRE corpus must never happen by accident — the flag alone is refused with exit 2, before ``ingest.ingest`` is ever called. + ``--retry`` is the no-typing complement: it computes its own bounded set — + ``ingest.retry_candidates()``, the failed sources still on disk plus the ingested ones no + wiki page cites — prints it, and runs those as a FORCED re-read. It therefore refuses + explicit paths and ``--force`` (exit 2): the point of the flag is that citadel picks the + set, and combining the two would blur which one wins. Nothing to retry is a clean exit 0. + ``--jobs N`` is a usage error below 1 (exit 2, like ``--force`` without paths) rather than an exception out of the API layer; omitted, the run takes ``CITADEL_JOBS`` (default 1, serial).""" from . import config, ingest @@ -421,6 +436,15 @@ def cmd_ingest(args: argparse.Namespace) -> int: print(f"error: --jobs must be at least 1 (got {args.jobs}); 1 means the serial default.", file=sys.stderr) return 2 + if args.retry and (args.paths or args.force): + print( + "error: --retry computes its own retry set (every failed source plus every ingested " + "source no wiki page cites) — combine no paths and no --force with it; use " + "`citadel ingest --force ` for a hand-picked re-read instead.", + file=sys.stderr, + ) + return 2 + if args.force and not args.paths: print( "error: --force requires explicit paths (a forced re-read runs one agent session per " @@ -429,6 +453,24 @@ def cmd_ingest(args: argparse.Namespace) -> int: ) return 2 + paths = args.paths or None + force = args.force + if args.retry: + failed, uncited = ingest.retry_candidates() + if not failed and not uncited: + print("Nothing to retry: no failed sources and every ingested source is cited by at least one page.") + return 0 + if failed: + print(f"Retrying {len(failed)} failed source(s):") + for key in failed: + print(f" - {config.display_key(key)}") + if uncited: + print(f"Force-reconciling {len(uncited)} ingested source(s) that no wiki page cites:") + for key in uncited: + print(f" - {config.display_key(key)}") + paths = [str(config.source_path_for_key(key)) for key in failed + uncited] + force = True + if args.verbose: config.LLM_VERBOSE = True # `is not None` (not truthiness) so an explicit `--log-dir ""` is honored as "disable logging" @@ -444,9 +486,7 @@ def cmd_ingest(args: argparse.Namespace) -> int: # CITADEL_LLM_VERBOSE — not just the --verbose flag — also drops the spinner that would # otherwise clobber the streamed transcript. progress = ConsoleProgress(spinner=not config.LLM_VERBOSE) - report = ingest.ingest( - args.paths or None, progress=progress, full_rescan=args.full_rescan, force=args.force, jobs=args.jobs - ) + report = ingest.ingest(paths, progress=progress, full_rescan=args.full_rescan, force=force, jobs=args.jobs) print(report.render()) # Non-zero on a per-source error OR a structural problem left behind (a broken # cross-link the agent introduced) — so ingest gates the wiki's integrity in CI. diff --git a/citadel/ingest.py b/citadel/ingest.py index 96c3083..5a24bdc 100644 --- a/citadel/ingest.py +++ b/citadel/ingest.py @@ -125,6 +125,12 @@ class IngestReport: # segment 1 ("raw/book.txt (segments 1-3 of 7 restored)") — see citadel/resume.py. Recorded # whether or not the resumed source then succeeded: the earlier work was reused either way. resumed: list[str] = field(default_factory=list) + # rel-keys of FRESH sources (a plain ingest — not a reconcile, not a delete cleanup) whose + # session succeeded but changed NOTHING: no page created, updated, or deleted. Suspicious by + # construction — a brand-new source that contributes zero facts is usually a session that + # under-delivered, yet it is marked done and never revisited on its own. Surfaced as a WARNING + # so it is easy to spot and retry (`citadel ingest --retry`, or `--force `). + no_pages: list[str] = field(default_factory=list) # The wiki-history note from wikigit.autocommit ("wiki git: committed ", or a warning # naming what was skipped and why) — empty when the history layer had nothing to say. wiki_git: str = "" @@ -198,12 +204,23 @@ def render(self) -> str: if self.skipped: lines.append("Skipped (already ingested):") lines.extend(f" - {p}" for p in self.skipped) + if self.no_pages: + lines.append("WARNING — ingested but produced NO wiki changes (no page created, updated, or deleted):") + lines.extend(f" - {p}" for p in self.no_pages) + lines.append( + " These sources are marked done and will not be revisited automatically. " + "Retry them with `citadel ingest --retry` (or `citadel ingest --force `)." + ) if self.broken_links: lines.append("WARNING — broken cross-links (run `citadel lint`):") lines.extend(f" - {src} -> {tgt}" for src, tgt in self.broken_links) if self.errors: lines.append("Errors:") lines.extend(f" - {e}" for e in self.errors) + lines.append( + " Failed sources stay in the failures catalog (`citadel status` lists them) and are " + "retried on the next run — or right away with `citadel ingest --retry`." + ) if self.wiki_git: lines.append(self.wiki_git) return "\n".join(lines) @@ -2030,6 +2047,11 @@ class _SourceJob: asserts no reference survived and may legitimately empty the wiki). - ``sha_stat``: the (sha256, stat) discovery already took for the source, threaded into the failures catalog so an unchanged stuck source joins the stat quick check. + - ``warn_no_pages``: True for a FRESH source (a plain ingest of a new key, file or repo) — + a successful session that then changed NOTHING lands on ``report.no_pages`` as a warning. + Deliberately False for reconciles (an unchanged verdict is a legitimate outcome of + re-reading a source, and ``citadel refresh`` would otherwise flag its whole slice) and for + delete cleanups (empty means nothing cited the source — the expected case). """ key: str @@ -2039,6 +2061,7 @@ class _SourceJob: extra_check: Callable[[], list[str]] | None = None allow_emptying: bool = False sha_stat: tuple[str | None, os.stat_result | None] = (None, None) + warn_no_pages: bool = False @dataclass @@ -2160,6 +2183,10 @@ def _record_source_run(run: _JobRun, emit, report: IngestReport, failures_dict, report.pages_updated.extend(outcome.updated) report.pages_written.extend(outcome.created + outcome.updated) report.pages_deleted.extend(outcome.deleted) + if job.warn_no_pages and not (outcome.created or outcome.updated or outcome.deleted): + # A fresh source folded in with zero page changes: marked done below, so without this + # warning it would silently never contribute anything (see IngestReport.no_pages). + report.no_pages.append(job.key) # The manifest stamp covers every session whose work this promote landed — this run's plus # whatever an earlier run already paid for the segments a checkpoint restored — while # ``report.usage`` above stays strictly this run's spend, so nothing is double-counted @@ -2513,6 +2540,48 @@ def _pending_session( return llm.run_ingest_session(rel_key, kind=kind) +def retry_candidates() -> tuple[list[str], list[str]]: + """The source keys ``citadel ingest --retry`` re-runs, as ``(failed, uncited)``. + + - ``failed``: every source in the failures catalog that is still on disk — errored / timed-out + / unreadable records alike (an unchanged unreadable file is re-evaluated for free, and a + fixed one — hydrated placeholder, re-exported document — now ingests). Deliberate skips + stay skipped: a same-basename ``duplicate`` record is a decision, not a failure, and a + ``curate`` record is keyed by a PAGE, not a source (``citadel curate --retry`` owns those). + - ``uncited``: every INGESTED source that no wiki page cites (the same + ``store.citing_pages_map`` verdict as the ``Referenced by`` column of + ``wiki/sources/index.md``, and ``citadel status``'s ``NO PAGES`` marker) — a session was + paid for, the source is marked done, and yet nothing in the wiki carries its facts. These + are re-run as FORCED reconciles. + + Both lists are sorted and disjoint (a key can only be in one catalog); vanished files are + excluded — a retry cannot read what is not there (a vanished INGESTED source is the deletion + sweep's job, not a retry's). Read-only: computing candidates changes nothing. Best-effort + like ``status``'s marker: a wiki that cannot be traversed degrades to an empty ``uncited`` + list instead of taking the recovery command down — the failed sources are still retried, + which is exactly the situation ``--retry`` exists for.""" + failed: list[str] = [] + for key, entry in sorted(failures.load().items()): + if not isinstance(entry, dict): + continue + if entry.get("reason") in (failures.DUPLICATE, failures.CURATE): + continue + if config.source_path_for_key(key).exists(): + failed.append(key) + manifest_dict = manifest.load() + uncited: list[str] = [] + if manifest_dict: + try: + refs = store.citing_pages_map(list(manifest_dict)) + except Exception: # noqa: BLE001 - recovery must degrade, never crash on a broken wiki + refs = None + if refs is not None: + uncited = [ + key for key in sorted(manifest_dict) if not refs.get(key) and config.source_path_for_key(key).exists() + ] + return failed, uncited + + @pagecache.bypass def ingest( paths: list[str] | None = None, @@ -3025,6 +3094,9 @@ def done(usage: llm.SessionUsage | None) -> None: on_success=done, prepare_error="prepare audio transcript" if is_audio else "write source text", sha_stat=sha_stat, + # A brand-new key (not a reconcile of changed/forced bytes) that produces zero page + # changes is worth a warning — see _SourceJob.warn_no_pages. + warn_no_pages=rel_key not in changed_keys, ) # Repo sources: each git repository under raw/ is folded in by ONE session reading a @@ -3065,7 +3137,13 @@ def done(usage: llm.SessionUsage | None) -> None: manifest.save(manifest_dict) report.processed.append(rjob.key) - return _SourceJob(key=rjob.key, build_sessions=build, on_success=done, prepare_error="build digest") + return _SourceJob( + key=rjob.key, + build_sessions=build, + on_success=done, + prepare_error="build digest", + warn_no_pages=rjob.kind == "repo", # a fresh repo digest yielding nothing is suspicious + ) # Deleted sources: a tracked source vanished from disk (full run only). If any page still # cites it, run a `kind="delete"` cleanup session that strips that provenance, gated by a @@ -3162,6 +3240,12 @@ def done(_usage: llm.SessionUsage | None) -> None: f"{len(report.pages_updated)} updated, {len(report.pages_deleted)} deleted " f"(model: {model})" ) + for key in report.no_pages: + store.append_log( + f"ingested {key} but the session produced no wiki changes (no page created, " + f"updated, or deleted); retry with `citadel ingest --retry` or " + f"`citadel ingest --force {key}`" + ) for old_key, new_key in report.moved: store.append_log( f"reorganized {new_key}: same content already ingested as {old_key}; " diff --git a/citadel/progress.py b/citadel/progress.py index 436ad54..82136fc 100644 --- a/citadel/progress.py +++ b/citadel/progress.py @@ -233,7 +233,9 @@ def on_source_done( changes.append(f"{updated} updated") if deleted: changes.append(f"{deleted} deleted") - tail = [(", ".join(changes), "") if changes else ("no changes", "dim")] + # "no changes" renders in YELLOW, not dim: a source that folded in without touching a + # single page is exactly the verdict worth a second look (see IngestReport.no_pages). + tail = [(", ".join(changes), "") if changes else ("no changes", "yellow")] tail.extend(usage_bits(usage, model)) self._finish(source, self._verdict(index, total, "OK", "bold green", source, seconds, tail)) diff --git a/citadel/status.py b/citadel/status.py index 4ad2ce9..f0d263d 100644 --- a/citadel/status.py +++ b/citadel/status.py @@ -6,7 +6,8 @@ - **ingested** — folded into the wiki (with the importing model + the rules-tree hash it ran under, a ``(stale)`` flag when that hash predates the current rulebook — the ``curate --stale-rules`` signal — and the ``checked`` date a model last verified it, the ``citadel - refresh`` ordering); + refresh`` ordering; a ``NO PAGES`` marker calls out a source that NO wiki page cites — it was + ingested but produced zero entries, the ``citadel ingest --retry`` signal); - **failed** — unreadable / errored / timed-out, with the coarse reason and, for a stuck curate- style record, its attempt count; - **skipped-duplicate** — a same-basename twin skipped in favor of another format; @@ -16,8 +17,9 @@ - **pending** — on disk under a raw root, not yet in the manifest or the failures catalog. Built from the manifest + the failures catalog + ONE stat-only discovery walk (reusing ingest's -own walk so repo sources, multi-root layouts, and dead mounts behave identically). It NEVER -re-hashes a byte — that is ingest's job — so it is cheap to run any time. The manifest/failures +own walk so repo sources, multi-root layouts, and dead mounts behave identically) + one wiki +traversal for the ``NO PAGES`` markers. It NEVER re-hashes a byte — that is ingest's job — so it +is cheap to run any time. The manifest/failures files ARE the database; status only reads them. Read-only and defensive: a broken walk degrades to empty pending/ignored rather than raising. """ @@ -28,7 +30,7 @@ from dataclasses import asdict, dataclass, field from pathlib import Path -from . import config, failures, ingest, llm, manifest +from . import config, failures, ingest, llm, manifest, store # How much of a (long) content hash / rules-version / commit id to show in the table. @@ -60,6 +62,12 @@ class SourceState: tokens_in: int | None = None tokens_out: int | None = None aic: float | None = None + # True for an INGESTED source that NO wiki page cites (the ``Referenced by`` column of + # ``wiki/sources/index.md`` is empty for it): a session ran and was paid for, the source is + # marked done, yet it contributed zero entries to the wiki. Rendered as a ``NO PAGES`` marker + # and the `citadel ingest --retry` hint. Always False when the wiki could not be read (the + # marker must never fire on a load error). + uncited: bool = False @dataclass @@ -121,6 +129,10 @@ def render(self) -> str: # Stamped independently, so credits can outlive an unknown dollar figure — show # them rather than silently dropping spend the corpus total already counts. parts.append(f"{llm.format_aic(s.aic)} AIC") + if s.uncited: + # Loud on purpose: "ingested" reads as success, but nothing in the wiki cites + # this source — it produced zero entries. + parts.append("NO PAGES (nothing cites this source)") lines.append(" " + " ".join(parts)) lines.append(f"Failed ({len(self.failed)})") @@ -151,13 +163,27 @@ def render(self) -> str: for key in self.pending: lines.append(f" {key}") + # The one-line call to action: everything stuck — failed sources and the NO PAGES ones — + # is retryable with a single command, so say so instead of leaving the reader to collect + # paths for `--force` by hand. + uncited = sum(1 for s in self.ingested if s.uncited) + if self.failed or uncited: + bits = [] + if self.failed: + bits.append(f"{len(self.failed)} failed source(s)") + if uncited: + bits.append(f"{uncited} NO PAGES source(s)") + lines.append("") + lines.append(f"Retry {' and '.join(bits)} with: citadel ingest --retry") + return "\n".join(lines).rstrip() + "\n" def as_dict(self) -> dict: """The report as one JSON-ready dict (``citadel status --json``): the six buckets plus ``rules_version``, ``cost_usd_total`` and ``aic_total``, each source row a plain dict with only its None fields dropped — - ``attempts: 0`` / ``stale_rules: false`` stay explicit, so scripts get a predictable - shape for 'which sources failed and why' without scraping :meth:`render`'s table. + ``attempts: 0`` / ``stale_rules: false`` / ``uncited: false`` stay explicit, so scripts + get a predictable shape for 'which sources failed and why' (and which produced no pages) + without scraping :meth:`render`'s table. ``oversized`` carries ``{"key", "size_bytes"}`` objects rather than bare strings, since the size is the reason the row exists.""" @@ -263,6 +289,19 @@ def build_status() -> StatusReport: ) ) + # Mark the ingested sources NO wiki page cites (the NO PAGES rows): one wiki traversal + # (store.citing_pages_map — the exact verdict behind sources/index.md's "Referenced by" + # column), best-effort like the walk: a wiki that cannot be read yields no markers rather + # than a failed status. + if report.ingested: + try: + refs = store.citing_pages_map([s.key for s in report.ingested]) + except Exception: # noqa: BLE001 - status is read-only and must degrade, never raise + refs = None + if refs is not None: + for row in report.ingested: + row.uncited = not refs.get(row.key) + for key in sorted(failures_dict): entry = failures_dict[key] if not isinstance(entry, dict): diff --git a/docs/maintenance.md b/docs/maintenance.md index 0f39d5d..699a93b 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -92,12 +92,33 @@ last verified it, the ordering `citadel refresh` works through — and what that cost when the backend reported it, e.g. `$0.05`), **failed** (with the reason and attempt count), **skipped-duplicate**, **ignored** (which pattern matched), **oversized** (past the `CITADEL_MAX_SOURCE_BYTES` discovery ceiling, with the size that explains it), or **pending** (not -yet ingested — the next `citadel ingest` will pick it up). A `Recorded LLM cost` line above the table +yet ingested — the next `citadel ingest` will pick it up). An ingested source that **no wiki page +cites** — a session ran and was paid for, yet it produced zero entries — is marked +`NO PAGES (nothing cites this source)` (`"uncited": true` in `--json`); the same condition is +flagged at ingest time as a `WARNING — ingested but produced NO wiki changes` section on the run +report and a yellow `no changes` verdict in the live progress. A `Recorded LLM cost` line above the table totals the per-source stamps (the maintenance-cost snapshot of the current corpus; `--json` carries it as `cost_usd_total`). It never runs an agent and never re-hashes sources, so it is always cheap to run. An MCP client gets the same table via the read-only `wiki_status` tool (see [mcp.md](mcp.md)). +## Retry + +Everything stuck is retryable with one command: + +```bash +citadel ingest --retry +``` + +It computes its own bounded set and prints it before running: every **failed** source still on +disk (errored / timed-out / unreadable — a deliberate same-basename `duplicate` skip stays +skipped), plus every **ingested source no wiki page cites** (the `NO PAGES` rows above), re-read +as forced reconciles. Failed sources are also retried automatically on every normal `citadel +ingest` run — `--retry` is how you retry them *now*, together with the zero-page sources that a +normal run would never revisit (they are marked done in the manifest). It refuses explicit paths +and `--force` (use `citadel ingest --force ` for a hand-picked re-read) and exits 0 with +`Nothing to retry` when the corpus is healthy. + ## Rules The wiki is built by rules files the ingest agent reads at run time — `schema.md` (the format diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index a86cac6..97a284c 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -109,7 +109,10 @@ CLI (see [configuration — Audio/video sources](configuration.md#audiovideo-sou - Run `citadel status` — the read-only per-source state table shows exactly what happened to each file: ingested, failed, skipped-duplicate, ignored (matched `CITADEL_IGNORE_PATTERNS`), oversized - (over `CITADEL_MAX_SOURCE_BYTES`), or pending. + (over `CITADEL_MAX_SOURCE_BYTES`), or pending. An ingested source that produced **zero entries** + (no wiki page cites it) is marked `NO PAGES`. +- `citadel ingest --retry` re-runs everything stuck in one go: every failed source still on disk + plus every `NO PAGES` source (as a forced reconcile). No paths needed — it prints the set first. - Already-ingested sources are skipped by sha match — that's not a bug. To deliberately re-read one, use `citadel ingest --force `. - Watch a run live with `citadel ingest --verbose` (`-v`), or capture a full transcript per source diff --git a/tests/test_retry.py b/tests/test_retry.py new file mode 100644 index 0000000..959aa3e --- /dev/null +++ b/tests/test_retry.py @@ -0,0 +1,241 @@ +"""Visibility + one-command retry for stuck sources (offline). + +Two ways a source can silently contribute nothing to the wiki: + +- it FAILED (errored / timed-out / unreadable) — persisted in the failures catalog, retried on + the next run; +- it "succeeded" with ZERO page changes — marked done in the manifest, so without help it is + never revisited and no wiki page ever cites it. + +These tests pin the visibility layer (``IngestReport.no_pages`` + its WARNING section, the +``NO PAGES`` marker and retry hint in ``citadel status``) and the retry lane +(``ingest.retry_candidates()`` + ``citadel ingest --retry``). Everything runs on ``tmp_citadel`` +with the shared ``fake_agent``/``seed_page`` fixtures; no CLI, no network. +""" + +from __future__ import annotations + +import pytest + +from citadel import cli, config, failures, ingest, manifest, status + + +def _track(key: str, sha: str) -> None: + """Record one ingested manifest entry for ``key`` (load-modify-save).""" + tracked = manifest.load() + tracked[key] = manifest.make_entry(sha, "claude:sonnet", config.rules_version()) + manifest.save(tracked) + + +def _fail(key: str, reason: str, detail: str = "") -> None: + """Record one failures-catalog entry for ``key`` (load-modify-save).""" + stuck = failures.load() + failures.record(stuck, key, reason, detail) + failures.save(stuck) + + +def _cited_source(cit, seed_page, name: str = "cited") -> str: + """A raw source on disk, tracked in the manifest, WITH a wiki page whose ``resource`` + frontmatter cites it — the healthy baseline the uncited detection must not flag.""" + key = f"raw/{name}.md" + src = cit.raw / f"{name}.md" + src.write_text("body\n", encoding="utf-8") + _track(key, manifest.file_sha256(src)) + seed_page( + f"concepts/{name}.md", + {"type": "Concept", "title": name, "description": "d", "tags": ["t"], "resource": key}, + f"A fact.[^s1]\n\n## Sources\n\n[^s1]: [{key}](../../{key}) - src\n", + ) + return key + + +# --- the run report: IngestReport.no_pages ------------------------------------------------- + + +def test_fresh_ingest_with_no_page_changes_is_flagged(tmp_citadel, fake_agent): + """A brand-new source whose session changes NOTHING lands on ``report.no_pages`` and the + report renders a WARNING with the retry hint — it is marked done, so this warning is the + only trace that the source contributed zero entries.""" + (tmp_citadel.raw / "notes.md").write_text("nothing came of this\n", encoding="utf-8") + fake_agent() # a successful session that writes no pages + + report = ingest.ingest() + assert report.no_pages == ["raw/notes.md"] + assert "raw/notes.md" in report.processed # still marked done — the point of the warning + text = report.render() + assert "produced NO wiki changes" in text + assert "citadel ingest --retry" in text + # The permanent trace survives the console: log.md records the zero-page verdict. + assert "produced no wiki changes" in tmp_citadel.log_path.read_text(encoding="utf-8") + + +def test_reconcile_with_no_changes_is_not_flagged(tmp_citadel, fake_agent, transformer_page): + """A RECONCILE (changed bytes of an already-tracked source) that decides nothing needs to + change is a legitimate verdict — never flagged (else ``citadel refresh`` would warn on its + whole slice).""" + (tmp_citadel.raw / "notes.md").write_text("v1\n", encoding="utf-8") + fake_agent(transformer_page) + assert ingest.ingest().no_pages == [] # created a page: nothing to flag + + (tmp_citadel.raw / "notes.md").write_text("v2 (changed)\n", encoding="utf-8") + agent = fake_agent() # reconcile session: writes nothing + report = ingest.ingest() + assert agent.calls == [("raw/notes.md", "reconcile")] + assert report.no_pages == [] + + +def test_error_report_renders_retry_hint(): + """The Errors section points at the retry lane, so a failed run tells the reader how to try + again instead of leaving the failure to scroll away.""" + report = ingest.IngestReport(processed=[], skipped=[], pages_written=[], errors=["raw/x.md: boom"]) + assert "citadel ingest --retry" in report.render() + + +# --- retry_candidates(): the computed retry set -------------------------------------------- + + +def test_retry_candidates_failed_and_uncited_buckets(tmp_citadel, seed_page): + """``retry_candidates()`` = (failed sources still on disk, ingested sources no page cites). + Excluded: a vanished failure (nothing left to read), a deliberate ``duplicate`` skip, a + curate record (a page, not a source), and every cited healthy source.""" + (tmp_citadel.raw / "bad.md").write_text("errored last run\n", encoding="utf-8") + _fail("raw/bad.md", failures.ERROR, "agent session failed") + _fail("raw/gone.md", failures.ERROR, "vanished since") # no file on disk + (tmp_citadel.raw / "dup.pdf").write_bytes(b"%PDF-fake") + _fail("raw/dup.pdf", failures.DUPLICATE, "same basename as raw/dup.pptx") + _fail("concepts/topic.md", failures.CURATE, "cluster failed") + _cited_source(tmp_citadel, seed_page, "cited") + (tmp_citadel.raw / "empty.md").write_text("ingested to zero entries\n", encoding="utf-8") + _track("raw/empty.md", manifest.file_sha256(tmp_citadel.raw / "empty.md")) + + failed, uncited = ingest.retry_candidates() + assert failed == ["raw/bad.md"] + assert uncited == ["raw/empty.md"] + + +def test_retry_candidates_empty_when_healthy(tmp_citadel, seed_page): + """A corpus with no failures and every source cited has nothing to retry.""" + _cited_source(tmp_citadel, seed_page) + assert ingest.retry_candidates() == ([], []) + + +def test_retry_candidates_degrade_when_wiki_unreadable(tmp_citadel, monkeypatch): + """A wiki that cannot be traversed must not take the recovery command down: the uncited + detection degrades to empty (mirroring ``status``'s best-effort marker) and the failed + sources are still retried — the exact situation ``--retry`` exists for.""" + from citadel import store + + (tmp_citadel.raw / "bad.md").write_text("errored last run\n", encoding="utf-8") + _fail("raw/bad.md", failures.ERROR, "agent session failed") + (tmp_citadel.raw / "empty.md").write_text("tracked\n", encoding="utf-8") + _track("raw/empty.md", manifest.file_sha256(tmp_citadel.raw / "empty.md")) + + def boom(*_a, **_k): + raise OSError("wiki dir unreadable") + + monkeypatch.setattr(store, "citing_pages_map", boom) + assert ingest.retry_candidates() == (["raw/bad.md"], []) + + +# --- citadel ingest --retry ---------------------------------------------------------------- + + +@pytest.mark.parametrize("argv", [["raw/x.md"], ["--force"], ["--force", "raw/x.md"]]) +def test_cli_retry_refuses_paths_and_force(tmp_citadel, capsys, argv): + """``--retry`` computes its own set: explicit paths or ``--force`` alongside it are a usage + error (exit 2), before ``ingest.ingest`` is ever reached.""" + assert cli.main(["ingest", "--quiet", "--retry", *argv]) == 2 + assert "--retry" in capsys.readouterr().err + + +def test_cli_retry_with_nothing_stuck_is_a_clean_noop(tmp_citadel, capsys, monkeypatch): + """No failed and no uncited sources: ``--retry`` says so and exits 0 without a run.""" + + def never(*a, **k): # pragma: no cover - the assertion is that this is never reached + raise AssertionError("ingest.ingest must not run when there is nothing to retry") + + monkeypatch.setattr(ingest, "ingest", never) + assert cli.main(["ingest", "--quiet", "--retry"]) == 0 + assert "Nothing to retry" in capsys.readouterr().out + + +def test_cli_retry_runs_computed_set_as_forced_read(tmp_citadel, seed_page, capsys, monkeypatch): + """``--retry`` prints the retry set and hands exactly those paths to ``ingest.ingest`` with + ``force=True`` — the failed source re-runs, the uncited one re-reads as a forced reconcile.""" + (tmp_citadel.raw / "bad.md").write_text("errored last run\n", encoding="utf-8") + _fail("raw/bad.md", failures.ERROR, "agent session failed") + (tmp_citadel.raw / "empty.md").write_text("ingested to zero entries\n", encoding="utf-8") + _track("raw/empty.md", manifest.file_sha256(tmp_citadel.raw / "empty.md")) + _cited_source(tmp_citadel, seed_page, "cited") # healthy: must NOT be re-read + + captured: dict = {} + + def spy(paths=None, progress=None, **kwargs): + captured["paths"] = paths + captured["kwargs"] = kwargs + return ingest.IngestReport(processed=[], skipped=[], pages_written=[], errors=[]) + + monkeypatch.setattr(ingest, "ingest", spy) + assert cli.main(["ingest", "--quiet", "--retry"]) == 0 + expected = [str(config.source_path_for_key(k)) for k in ("raw/bad.md", "raw/empty.md")] + assert captured["paths"] == expected + assert captured["kwargs"]["force"] is True + out = capsys.readouterr().out + assert "Retrying 1 failed source(s)" in out + assert "Force-reconciling 1 ingested source(s)" in out + + +def test_cli_retry_end_to_end_clears_the_stuck_source(tmp_citadel, fake_agent, transformer_page): + """The whole lane, no spies: a fresh ingest flags the zero-page source, ``--retry`` re-runs + it as a forced reconcile, and once the session produces a citing page the source stops being + a candidate.""" + (tmp_citadel.raw / "notes.md").write_text("worth a page\n", encoding="utf-8") + fake_agent() # first pass under-delivers: no pages + assert ingest.ingest().no_pages == ["raw/notes.md"] + assert ingest.retry_candidates() == ([], ["raw/notes.md"]) + + agent = fake_agent(transformer_page) # the retry writes the page + assert cli.main(["ingest", "--quiet", "--retry"]) == 0 + assert agent.calls == [("raw/notes.md", "reconcile")] + assert ingest.retry_candidates() == ([], []) + + +# --- citadel status: the NO PAGES marker --------------------------------------------------- + + +def test_status_marks_uncited_sources(tmp_citadel, seed_page): + """An ingested source no wiki page cites carries ``uncited`` (rendered as ``NO PAGES`` with + the retry hint, explicit in ``--json``); a cited one does not.""" + _cited_source(tmp_citadel, seed_page, "cited") + (tmp_citadel.raw / "empty.md").write_text("zero entries\n", encoding="utf-8") + _track("raw/empty.md", manifest.file_sha256(tmp_citadel.raw / "empty.md")) + + report = status.build_status() + by_key = {s.key: s for s in report.ingested} + assert by_key["raw/empty.md"].uncited is True + assert by_key["raw/cited.md"].uncited is False + + text = report.render() + empty_line = next(line for line in text.splitlines() if "raw/empty.md" in line) + cited_line = next(line for line in text.splitlines() if "raw/cited.md" in line) + assert "NO PAGES" in empty_line and "NO PAGES" not in cited_line + assert "citadel ingest --retry" in text + + rows = {r["key"]: r for r in report.as_dict()["ingested"]} + assert rows["raw/empty.md"]["uncited"] is True and rows["raw/cited.md"]["uncited"] is False + + +def test_status_healthy_corpus_has_no_retry_hint(tmp_citadel, seed_page): + """Nothing failed and everything cited: no marker, no hint — the table stays quiet.""" + _cited_source(tmp_citadel, seed_page) + text = status.build_status().render() + assert "NO PAGES" not in text + assert "citadel ingest --retry" not in text + + +def test_status_failed_sources_alone_trigger_the_hint(tmp_citadel): + """The retry hint also rides on failures alone (the catalog's sources are retryable too).""" + _fail("raw/bad.md", failures.ERROR, "agent session failed") + text = status.build_status().render() + assert "1 failed source(s)" in text + assert "citadel ingest --retry" in text