|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""Generate CATALOG.md — the migrated-dataset registry — from the sidecar |
| 3 | +manifests in lectures/*.yml. |
| 4 | +
|
| 5 | +CATALOG.md is a GENERATED file: the per-dataset manifests are the source of |
| 6 | +truth, and this script is the only thing that should write the catalog. Run it |
| 7 | +after adding or editing any manifest: |
| 8 | +
|
| 9 | + python scripts/build_catalog.py |
| 10 | +
|
| 11 | +CI asserts the catalog is current with `git diff --exit-code CATALOG.md` after |
| 12 | +regenerating, so a stale catalog fails the build (PLAN Phase 5). |
| 13 | +
|
| 14 | +Scope: migrated-only. A dataset appears here once it has a manifest; files in |
| 15 | +lectures/ without a manifest are not yet migrated and are tracked in PLAN |
| 16 | +Phase 9, not here. |
| 17 | +""" |
| 18 | +from __future__ import annotations |
| 19 | + |
| 20 | +import sys |
| 21 | +from pathlib import Path |
| 22 | + |
| 23 | +import yaml |
| 24 | + |
| 25 | +REPO = Path(__file__).resolve().parent.parent |
| 26 | +LECTURES = REPO / "lectures" |
| 27 | +OUT = REPO / "CATALOG.md" |
| 28 | + |
| 29 | +# Interim URL form (AGENTS.md); swaps to data.quantecon.org/lectures/ at Phase 4. |
| 30 | +RAW = "https://github.com/QuantEcon/data-lectures/raw/main/lectures" |
| 31 | + |
| 32 | + |
| 33 | +def human_size(n: int) -> str: |
| 34 | + size = float(n) |
| 35 | + for unit in ("B", "KB", "MB", "GB"): |
| 36 | + if size < 1024 or unit == "GB": |
| 37 | + return f"{int(size)} {unit}" if unit == "B" else f"{size:.1f} {unit}" |
| 38 | + size /= 1024 |
| 39 | + return f"{size:.1f} GB" |
| 40 | + |
| 41 | + |
| 42 | +def oneline(text) -> str: |
| 43 | + """Collapse a folded/multiline scalar to a single trimmed line.""" |
| 44 | + if text is None: |
| 45 | + return "" |
| 46 | + return " ".join(str(text).split()) |
| 47 | + |
| 48 | + |
| 49 | +def load_manifests(): |
| 50 | + manifests = [] |
| 51 | + for path in sorted(LECTURES.glob("*.yml")): |
| 52 | + with path.open(encoding="utf-8") as f: |
| 53 | + m = yaml.safe_load(f) |
| 54 | + if not isinstance(m, dict) or "filename" not in m: |
| 55 | + print(f"skip (not a manifest): {path.name}", file=sys.stderr) |
| 56 | + continue |
| 57 | + m["_manifest_path"] = path |
| 58 | + data_file = LECTURES / m["filename"] |
| 59 | + m["_size"] = data_file.stat().st_size if data_file.exists() else None |
| 60 | + manifests.append(m) |
| 61 | + return manifests |
| 62 | + |
| 63 | + |
| 64 | +def fmt_consumers(consumers) -> str: |
| 65 | + if not consumers: |
| 66 | + return "—" |
| 67 | + parts = [] |
| 68 | + for c in consumers: |
| 69 | + repo = c.get("repo", "") |
| 70 | + short = repo.split("/")[-1] if repo else "?" |
| 71 | + file = c.get("file", "") |
| 72 | + stem = file.split("/")[-1] if file else "" |
| 73 | + if repo and file: |
| 74 | + parts.append(f"[{short} · {stem}](https://github.com/{repo}/blob/main/{file})") |
| 75 | + else: |
| 76 | + parts.append(f"{short} · {stem}".strip(" ·")) |
| 77 | + return "<br>".join(parts) |
| 78 | + |
| 79 | + |
| 80 | +def fmt_source(source) -> str: |
| 81 | + if not isinstance(source, dict): |
| 82 | + return oneline(source) |
| 83 | + name = oneline(source.get("name", "")) |
| 84 | + url = source.get("url") |
| 85 | + return f"[{name}]({url})" if url else name |
| 86 | + |
| 87 | + |
| 88 | +def fmt_redist(license) -> str: |
| 89 | + if not isinstance(license, dict): |
| 90 | + return "?" |
| 91 | + r = license.get("redistribution", "?") |
| 92 | + return {"permitted": "✅ permitted", "restricted": "⚠️ restricted"}.get(r, str(r)) |
| 93 | + |
| 94 | + |
| 95 | +def fmt_integrity(integrity) -> str: |
| 96 | + if not isinstance(integrity, dict): |
| 97 | + return "?" |
| 98 | + up = integrity.get("upstream") or {} |
| 99 | + status = up.get("status", "?") |
| 100 | + mark = {"verified": "✅", "spot-checked": "◑", "unverifiable": "⚠️", |
| 101 | + "unverified": "…", "failing": "❌"}.get(status, "") |
| 102 | + return f"{mark} {status}".strip() |
| 103 | + |
| 104 | + |
| 105 | +def fmt_builder(m) -> str: |
| 106 | + if m.get("class") == "verbatim": |
| 107 | + return "n/a (verbatim)" |
| 108 | + bs = m.get("builder_status", "?") |
| 109 | + return {"committed": "✅ committed", "unrecovered": "⚠️ unrecovered", |
| 110 | + "not-applicable": "n/a"}.get(bs, str(bs)) |
| 111 | + |
| 112 | + |
| 113 | +def build(manifests) -> str: |
| 114 | + total = sum(m["_size"] or 0 for m in manifests) |
| 115 | + permitted = sum(1 for m in manifests |
| 116 | + if (m.get("license") or {}).get("redistribution") == "permitted") |
| 117 | + restricted = len(manifests) - permitted |
| 118 | + |
| 119 | + lines = [] |
| 120 | + lines.append("<!-- GENERATED FILE — do not edit by hand. -->") |
| 121 | + lines.append("<!-- Regenerate: python scripts/build_catalog.py -->") |
| 122 | + lines.append("<!-- Source of truth: the per-dataset manifests at lectures/<file>.yml -->") |
| 123 | + lines.append("") |
| 124 | + lines.append("# Dataset catalog — `QuantEcon/data-lectures`") |
| 125 | + lines.append("") |
| 126 | + lines.append( |
| 127 | + "The migrated-dataset registry, **auto-generated** from the sidecar " |
| 128 | + "manifests (`lectures/*.yml`). Do not edit by hand — run " |
| 129 | + "`python scripts/build_catalog.py`. A dataset appears here once it has a " |
| 130 | + "manifest; files not yet migrated are tracked in " |
| 131 | + "[PLAN.md](PLAN.md) Phase 9." |
| 132 | + ) |
| 133 | + lines.append("") |
| 134 | + lines.append( |
| 135 | + f"**{len(manifests)} datasets migrated** · {human_size(total)} total · " |
| 136 | + f"{permitted} permitted / {restricted} restricted redistribution" |
| 137 | + ) |
| 138 | + lines.append("") |
| 139 | + lines.append("| Dataset | Class | Source | Licence | Redist. | Integrity | Builder | Size | Used by |") |
| 140 | + lines.append("| --- | --- | --- | --- | --- | --- | --- | --- | --- |") |
| 141 | + for m in manifests: |
| 142 | + fn = m["filename"] |
| 143 | + title = oneline(m.get("title", "")) |
| 144 | + dataset = f"[**{fn}**]({RAW}/{fn})" |
| 145 | + if title: |
| 146 | + dataset += f"<br><sub>{title}</sub>" |
| 147 | + license = m.get("license") or {} |
| 148 | + row = [ |
| 149 | + dataset, |
| 150 | + m.get("class", "?"), |
| 151 | + fmt_source(m.get("source")), |
| 152 | + oneline(license.get("name", "?")), |
| 153 | + fmt_redist(license), |
| 154 | + fmt_integrity(m.get("integrity")), |
| 155 | + fmt_builder(m), |
| 156 | + human_size(m["_size"]) if m["_size"] is not None else "—", |
| 157 | + fmt_consumers(m.get("consumers")), |
| 158 | + ] |
| 159 | + lines.append("| " + " | ".join(row) + " |") |
| 160 | + lines.append("") |
| 161 | + lines.append("---") |
| 162 | + lines.append("") |
| 163 | + lines.append( |
| 164 | + "**Legend** — *Integrity* is the `integrity.upstream.status` (is this " |
| 165 | + "what the source says?): ✅ verified · ◑ spot-checked · ⚠️ unverifiable · " |
| 166 | + "… unverified · ❌ failing. *Redist.* ⚠️ restricted files are cached as " |
| 167 | + "inherited exposures and tracked for licence review " |
| 168 | + "([workspace-lectures#20](https://github.com/QuantEcon/workspace-lectures/issues/20)). " |
| 169 | + "*Builder* ⚠️ unrecovered marks a constructed dataset whose builder was " |
| 170 | + "never committed (PLAN Phase 9)." |
| 171 | + ) |
| 172 | + lines.append("") |
| 173 | + return "\n".join(lines) |
| 174 | + |
| 175 | + |
| 176 | +def main() -> int: |
| 177 | + manifests = load_manifests() |
| 178 | + if not manifests: |
| 179 | + print("no manifests found under lectures/*.yml", file=sys.stderr) |
| 180 | + return 1 |
| 181 | + OUT.write_text(build(manifests), encoding="utf-8") |
| 182 | + print(f"wrote {OUT.relative_to(REPO)} — {len(manifests)} datasets") |
| 183 | + return 0 |
| 184 | + |
| 185 | + |
| 186 | +if __name__ == "__main__": |
| 187 | + raise SystemExit(main()) |
0 commit comments