feat: new system maintenance features

Author: Specthram

server/routers/system.py

@@ -189,6 +189,15 @@ def cleanup() -> dict:
     return maintenance.cleanup_report()
 
 
+@router.get("/cleanup/{category}")
+def cleanup_category(category: str) -> dict:
+    """Return one category's ``{count, bytes}`` (per-row lazy loader)."""
+    try:
+        return maintenance.category_report(category)
+    except ValueError as exc:
+        raise HTTPException(status_code=404, detail=str(exc)) from exc
+
+
 @router.post("/cleanup/{category}")
 def cleanup_purge(category: str) -> dict:
     """Purge one cleanup category; return what was removed and reclaimed."""

src/maintenance.py

@@ -27,11 +27,24 @@
 
 from src import crops, db, thumbnails, wm_compose
 from src import sqlite_store as store
+from src.constants import WATERMARK_BACKUPS_DIR
 
 logger = logging.getLogger(__name__)
 
 # The cleanup categories, in the order the System view lists them.
-CATEGORIES = ("media", "captions", "patches", "thumbs")
+CATEGORIES = (
+    "media",
+    "captions",
+    "dataset_captions",
+    "claims",
+    "quality",
+    "embeddings",
+    "index",
+    "crops",
+    "patches",
+    "wm_backups",
+    "thumbs",
+)
 
 
 def _safe_size(path: Path) -> int:
@@ -103,12 +116,118 @@ def purge_orphan_media() -> dict:
 
 def unused_caption_report() -> dict:
     """Return ``{count, bytes}`` for the superseded caption revisions."""
-    return {"count": store.unused_revision_count(), "bytes": 0}
+    return {
+        "count": store.unused_revision_count(),
+        "bytes": store.unused_revision_bytes(),
+    }
 
 
 def purge_unused_captions() -> dict:
     """Prune the superseded caption revisions; return ``{purged, bytes}``."""
-    return {"purged": store.prune_unused_revisions(), "bytes": 0}
+    freed = store.unused_revision_bytes()
+    return {"purged": store.prune_unused_revisions(), "bytes": freed}
+
+
+# -- captions of media in no dataset ------------------------------------------
+
+
+def unlinked_caption_report() -> dict:
+    """Return ``{count, bytes}`` for captions of media in no dataset."""
+    return store.unlinked_caption_report()
+
+
+def purge_unlinked_captions() -> dict:
+    """Delete those caption histories; return ``{purged, bytes}``."""
+    freed = store.unlinked_caption_report()["bytes"]
+    return {"purged": store.purge_unlinked_captions(), "bytes": freed}
+
+
+# -- caption grounding (claims) history ---------------------------------------
+
+
+def claims_report() -> dict:
+    """Return ``{count, bytes}`` for the stored caption-claim history."""
+    return store.grounding_history_report()
+
+
+def purge_claims() -> dict:
+    """Clear the whole grounding history; return ``{purged, bytes}``."""
+    freed = store.grounding_history_report()["bytes"]
+    return {"purged": store.purge_grounding_history(), "bytes": freed}
+
+
+# -- index data (quality / embeddings / dims+hashes) --------------------------
+
+
+def quality_report() -> dict:
+    """Return ``{count, bytes}`` for the stored quality scores."""
+    return store.quality_scores_report()
+
+
+def purge_quality() -> dict:
+    """Delete every quality score; return ``{purged, bytes}``."""
+    freed = store.quality_scores_report()["bytes"]
+    return {"purged": store.purge_quality_scores(), "bytes": freed}
+
+
+def embeddings_report() -> dict:
+    """Return ``{count, bytes}`` for the stored embedding vectors."""
+    return store.embeddings_report()
+
+
+def purge_embeddings() -> dict:
+    """Delete every embedding vector; return ``{purged, bytes}``."""
+    freed = store.embeddings_report()["bytes"]
+    return {"purged": store.purge_embeddings(), "bytes": freed}
+
+
+def media_index_report() -> dict:
+    """Return ``{count, bytes}`` for the per-media index columns."""
+    return store.media_index_report()
+
+
+def purge_media_index() -> dict:
+    """Reset the per-media index columns; return ``{purged, bytes}``."""
+    freed = store.media_index_report()["bytes"]
+    return {"purged": store.purge_media_index(), "bytes": freed}
+
+
+# -- rendered-crop cache ------------------------------------------------------
+
+
+def crops_cache_report() -> dict:
+    """Return ``{count, bytes}`` for the rendered-crop cache."""
+    count, total = _tree_size(crops.get_crops_dir())
+    return {"count": count, "bytes": total}
+
+
+def purge_crops_cache() -> dict:
+    """Empty the rendered-crop cache; return ``{purged, bytes}``.
+
+    Crops are virtual (a rectangle over the parent media), so the PNGs are
+    re-materialized on demand the next time something reads them.
+    """
+    removed, freed = _empty_tree(crops.get_crops_dir())
+    return {"purged": removed, "bytes": freed}
+
+
+# -- watermark backups --------------------------------------------------------
+
+
+def wm_backup_report() -> dict:
+    """Return ``{count, bytes}`` for the pre-patch original backups."""
+    count, total = _tree_size(WATERMARK_BACKUPS_DIR)
+    return {"count": count, "bytes": total}
+
+
+def purge_wm_backups() -> dict:
+    """Empty the watermark backups; return ``{purged, bytes}``.
+
+    Loses the "restore original" option of already-patched media; the
+    patched pixels themselves are untouched.
+    """
+    removed, freed = _empty_tree(WATERMARK_BACKUPS_DIR)
+    return {"purged": removed, "bytes": freed}
 
 
 # -- orphan patch / composite / crop cache files ----------------------------
@@ -201,20 +320,42 @@ def purge_thumbnail_cache() -> dict:
 _REPORTS = {
     "media": orphan_media_report,
     "captions": unused_caption_report,
+    "dataset_captions": unlinked_caption_report,
+    "claims": claims_report,
+    "quality": quality_report,
+    "embeddings": embeddings_report,
+    "index": media_index_report,
+    "crops": crops_cache_report,
     "patches": orphan_patch_report,
+    "wm_backups": wm_backup_report,
     "thumbs": thumbnail_report,
 }
 
 _PURGES = {
     "media": purge_orphan_media,
     "captions": purge_unused_captions,
+    "dataset_captions": purge_unlinked_captions,
+    "claims": purge_claims,
+    "quality": purge_quality,
+    "embeddings": purge_embeddings,
+    "index": purge_media_index,
+    "crops": purge_crops_cache,
     "patches": purge_orphan_patches,
+    "wm_backups": purge_wm_backups,
     "thumbs": purge_thumbnail_cache,
 }
 
 # Only the categories that delete database rows benefit from a VACUUM; the
 # file-cache sweeps free their space on unlink.
-_VACUUM_AFTER = {"media", "captions"}
+_VACUUM_AFTER = {
+    "media",
+    "captions",
+    "dataset_captions",
+    "claims",
+    "quality",
+    "embeddings",
+    "index",
+}
 
 
 def vacuum() -> None:
@@ -228,6 +369,19 @@ def cleanup_report() -> dict:
     return {category: report() for category, report in _REPORTS.items()}
 
 
+def category_report(category: str) -> dict:
+    """Return one category's ``{count, bytes}`` (the per-row loader).
+
+    The System view fetches each category independently so a slow scan
+    (patch orphans, big cache trees) never blocks the cheap ones. Raises
+    ``ValueError`` on an unknown category.
+    """
+    report = _REPORTS.get(category)
+    if report is None:
+        raise ValueError(f"unknown cleanup category: {category!r}")
+    return report()
+
+
 def run_cleanup(category: str) -> dict:
     """Purge one :data:`CATEGORIES` category.
 

src/sqlite_store/__init__.py

@@ -101,7 +101,10 @@
     unused_caption_count,
     delete_unused_captions,
     unused_revision_count,
+    unused_revision_bytes,
     prune_unused_revisions,
+    unlinked_caption_report,
+    purge_unlinked_captions,
 )
 
 from src.sqlite_store.media import (
@@ -326,6 +329,17 @@
     tag_grounding_for_media,
     tag_groundings_bulk,
     delete_tag_grounding,
+    grounding_history_report,
+    purge_grounding_history,
+)
+
+from src.sqlite_store.sweeps import (
+    quality_scores_report,
+    purge_quality_scores,
+    embeddings_report,
+    purge_embeddings,
+    media_index_report,
+    purge_media_index,
 )
 
 __all__ = [
@@ -531,6 +545,17 @@
     "prune_sub_libraries",
     "prune_uncategorized_if_empty",
     "prune_unused_revisions",
+    "unused_revision_bytes",
+    "unlinked_caption_report",
+    "purge_unlinked_captions",
+    "grounding_history_report",
+    "purge_grounding_history",
+    "quality_scores_report",
+    "purge_quality_scores",
+    "embeddings_report",
+    "purge_embeddings",
+    "media_index_report",
+    "purge_media_index",
     "purge_media",
     "read_caption",
     "reconcile_resolutions",

src/sqlite_store/captions.py

@@ -485,3 +485,57 @@ def prune_unused_revisions() -> int:
                 f"WHERE {_UNUSED_REVISION_WHERE}"
             )
             return cursor.rowcount
+
+
+def unused_revision_bytes() -> int:
+    """Return the text weight (bytes) of the prunable caption revisions."""
+    row = _query_one(
+        "SELECT COALESCE(SUM(LENGTH(content)), 0) AS b "
+        f"FROM caption_revision WHERE {_UNUSED_REVISION_WHERE}"
+    )
+    return row["b"] if row else 0
+
+
+# Captions belonging to a media that sits in no dataset: browsable nowhere
+# in a dataset context, so their whole history is prunable. The media row
+# itself stays (it belongs to a library).
+_UNLINKED_CAPTION_WHERE = (
+    "media_id NOT IN (SELECT media_id FROM dataset_media)"
+)
+
+
+def unlinked_caption_report() -> dict:
+    """Return ``{count, bytes}`` for captions of media in no dataset.
+
+    ``count`` is caption rows (media × type); ``bytes`` the text weight of
+    their entire revision histories.
+    """
+    row = _query_one(
+        "SELECT COUNT(DISTINCT c.id) AS n, "
+        "COALESCE(SUM(LENGTH(r.content)), 0) AS b FROM caption c "
+        "LEFT JOIN caption_revision r ON r.caption_id = c.id "
+        f"WHERE c.{_UNLINKED_CAPTION_WHERE}"
+    )
+    if row is None:
+        return {"count": 0, "bytes": 0}
+    return {"count": row["n"], "bytes": row["b"]}
+
+
+def purge_unlinked_captions() -> int:
+    """Delete the caption histories of media in no dataset; return count.
+
+    Ancestry links are cleared first (``parent_revision_id`` carries no
+    cascade); the caption rows then cascade their revisions, whose review /
+    grounding / score children cascade in turn.
+    """
+    with closing(db.connect()) as conn:
+        with conn:
+            conn.execute(
+                "UPDATE caption_revision SET parent_revision_id = NULL "
+                "WHERE caption_id IN "
+                f"(SELECT id FROM caption WHERE {_UNLINKED_CAPTION_WHERE})"
+            )
+            cursor = conn.execute(
+                f"DELETE FROM caption WHERE {_UNLINKED_CAPTION_WHERE}"
+            )
+            return cursor.rowcount

src/sqlite_store/grounding.py

@@ -256,3 +256,28 @@ def delete_tag_grounding(media_id: int, tag_id: int) -> None:
         "DELETE FROM media_tag_grounding WHERE media_id = ? AND tag_id = ?",
         (media_id, tag_id),
     )
+
+
+def grounding_history_report() -> dict:
+    """Return ``{count, bytes}`` for the stored caption-claim history.
+
+    ``count`` is the claim rows across every grounded revision; ``bytes``
+    their text weight (the scores themselves are a few bytes each).
+    """
+    row = _query_one(
+        "SELECT COUNT(*) AS n, COALESCE(SUM(LENGTH(text)), 0) AS b "
+        "FROM caption_grounding_claim"
+    )
+    if row is None:
+        return {"count": 0, "bytes": 0}
+    return {"count": row["n"], "bytes": row["b"]}
+
+
+def purge_grounding_history() -> int:
+    """Delete every caption grounding (claims cascade); return the claims."""
+    row = _query_one("SELECT COUNT(*) AS n FROM caption_grounding_claim")
+    claims = row["n"] if row else 0
+    with closing(db.connect()) as conn:
+        with conn:
+            conn.execute("DELETE FROM caption_grounding")
+    return claims