Merge pull request #150 from ContextLab/fix-hf-daily-papers

fix(hf-daily-papers): default to yesterday + retry-with-fallback + earlier cron

Author: jeremymanning

.github/workflows/hf-daily-papers.yml

@@ -1,6 +1,6 @@
 name: HF Daily Papers
 
-# Daily @ 23:59 UTC: submit the top-5 most-upvoted Hugging Face daily-papers
+# Daily @ 08:00 UTC: submit the top-5 most-upvoted Hugging Face daily-papers
 # (https://huggingface.co/papers/date/YYYY-MM-DD) as `human-submission` /
 # `new-paper` GitHub issues. The hourly `submission-intake.yml` workflow
 # then routes each one through the same code path the website's
@@ -10,13 +10,19 @@ name: HF Daily Papers
 # excluded from the contributor leaderboard by `_is_bot_submitter` in
 # `src/llmxive/web_data.py` (the paper's *authors* still get credited).
 #
+# Timing: the implementation defaults to (today_utc - 1 day) because HF's
+# daily-papers buckets are populated in the late afternoon UTC by HF's
+# editorial pipeline. Running at 08:00 UTC fetches yesterday's bucket,
+# which is guaranteed-published by then. The fallback chain in
+# _fetch_daily_json walks further back if even that bucket is empty.
+#
 # Security: workflow_dispatch inputs are untrusted — they're surfaced to
 # the shell via env: vars (never interpolated directly into run: steps)
 # and validated by Python's argparse before reaching `gh`.
 
 on:
   schedule:
-    - cron: "59 23 * * *"   # 23:59 UTC daily
+    - cron: "0 8 * * *"   # 08:00 UTC daily
   workflow_dispatch:
     inputs:
       date:

.omc/project-memory.json

@@ -211,6 +211,12 @@
       "lastAccessed": 1778845312910,
       "type": "file"
     },
+    {
+      "path": "src/llmxive/hf_daily_papers.py",
+      "accessCount": 10,
+      "lastAccessed": 1778852271916,
+      "type": "file"
+    },
     {
       "path": "src/llmxive/agents/personality.py",
       "accessCount": 7,
@@ -259,6 +265,12 @@
       "lastAccessed": 1778844885482,
       "type": "file"
     },
+    {
+      "path": "tests/unit/test_hf_daily_papers.py",
+      "accessCount": 4,
+      "lastAccessed": 1778857293016,
+      "type": "file"
+    },
     {
       "path": "specs/005-librarian-agent/spec.md",
       "accessCount": 3,

src/llmxive/hf_daily_papers.py

@@ -73,8 +73,8 @@ class SubmissionResult:
 # HF daily-papers fetch
 # ────────────────────────────────────────────────────────────────────────────
 
-def _fetch_daily_json(date: str, *, timeout: float = 30.0) -> list[dict[str, Any]]:
-    """Hit the public HF daily-papers endpoint for a given YYYY-MM-DD.
+def _fetch_daily_json_one(date: str, *, timeout: float = 30.0) -> list[dict[str, Any]]:
+    """Hit the public HF daily-papers endpoint for a given YYYY-MM-DD (one attempt).
 
     Returns the raw JSON list. Raises urllib.error.HTTPError on non-2xx.
     """
@@ -88,6 +88,47 @@ def _fetch_daily_json(date: str, *, timeout: float = 30.0) -> list[dict[str, Any
     return data
 
 
+def _fetch_daily_json(date: str, *, timeout: float = 30.0,
+                      fallback_days: int = 1) -> tuple[str, list[dict[str, Any]]]:
+    """Hit HF daily-papers for `date`; on 400 (date bucket not yet populated)
+    fall back to the previous UTC day up to `fallback_days` times.
+
+    Returns (effective_date, payload). Raises urllib.error.HTTPError on
+    persistent failure after the fallback chain.
+
+    Rationale: GitHub Actions cron jobs scheduled at ~23:59 UTC routinely
+    fire 5-30 minutes late. Once we cross midnight UTC, `_today_utc()` returns
+    the *new* day whose HF bucket isn't published yet, and the endpoint
+    returns HTTP 400. We must retry against the previous day so the cron
+    isn't a roulette wheel keyed on Actions queue depth.
+    """
+    from datetime import datetime, timedelta
+
+    try:
+        ts = datetime.strptime(date, "%Y-%m-%d")
+    except ValueError:
+        # Date didn't parse — try a single fetch and let urlopen raise.
+        return date, _fetch_daily_json_one(date, timeout=timeout)
+
+    attempts: list[str] = [date]
+    for i in range(1, fallback_days + 1):
+        attempts.append((ts - timedelta(days=i)).strftime("%Y-%m-%d"))
+
+    last_err: urllib.error.HTTPError | None = None
+    for d in attempts:
+        try:
+            return d, _fetch_daily_json_one(d, timeout=timeout)
+        except urllib.error.HTTPError as exc:
+            if exc.code in (400, 404):
+                # Date bucket not yet populated — try the previous day.
+                last_err = exc
+                continue
+            raise
+    # All attempts failed with 400/404 — re-raise the last one.
+    assert last_err is not None
+    raise last_err
+
+
 def _parse_paper(entry: dict[str, Any]) -> Paper | None:
     """Pull the fields we need from one HF entry; None if it's malformed."""
     paper = entry.get("paper") if isinstance(entry, dict) else None
@@ -105,19 +146,33 @@ def _parse_paper(entry: dict[str, Any]) -> Paper | None:
     return Paper(arxiv_id=arxiv_id, title=title, summary=summary, upvotes=upvotes)
 
 
-def fetch_top_papers(date: str, *, limit: int = 5, raw_json: list[dict[str, Any]] | None = None) -> list[Paper]:
-    """Return the top-N HF daily papers for ``date`` (YYYY-MM-DD), sorted by
-    upvotes desc. ``raw_json`` lets tests inject without monkey-patching
-    urllib.
+def fetch_top_papers(
+    date: str,
+    *,
+    limit: int = 5,
+    raw_json: list[dict[str, Any]] | None = None,
+) -> tuple[str, list[Paper]]:
+    """Return (effective_date, top-N HF daily papers) sorted by upvotes desc.
+
+    The effective_date may differ from the requested `date` when the HF
+    endpoint returns 400/404 for `date` (bucket not yet populated) and the
+    fallback chain in `_fetch_daily_json` succeeds against an earlier day.
+
+    ``raw_json`` lets tests inject without monkey-patching urllib. When
+    raw_json is supplied, the effective_date equals the requested date.
     """
-    data = raw_json if raw_json is not None else _fetch_daily_json(date)
+    if raw_json is not None:
+        effective = date
+        data = raw_json
+    else:
+        effective, data = _fetch_daily_json(date)
     parsed: list[Paper] = []
     for entry in data:
         p = _parse_paper(entry)
         if p is not None:
             parsed.append(p)
     parsed.sort(key=lambda p: (-p.upvotes, p.arxiv_id))
-    return parsed[: max(0, int(limit))]
+    return effective, parsed[: max(0, int(limit))]
 
 
 # ────────────────────────────────────────────────────────────────────────────
@@ -231,7 +286,14 @@ def submit_top_papers(
     when ``dry_run``). Idempotent across same-day retries: an arXiv URL
     already present in any open or closed `new-paper` issue is skipped.
     """
-    papers = fetch_top_papers(date, limit=limit, raw_json=raw_json)
+    effective_date, papers = fetch_top_papers(date, limit=limit, raw_json=raw_json)
+    if effective_date != date:
+        print(
+            f"[hf-daily-papers] HF bucket for {date} not yet populated; "
+            f"falling back to {effective_date}",
+            file=sys.stderr,
+        )
+    date = effective_date  # record the date that actually returned data
     filed: list[dict[str, Any]] = []
     skipped: list[dict[str, Any]] = []
     seen: set[str] = set() if dry_run else _list_recent_paper_issue_urls(repo)
@@ -268,9 +330,22 @@ def submit_top_papers(
 
 
 def _today_utc() -> str:
-    """YYYY-MM-DD in UTC — the HF daily-papers feed is UTC-bucketed."""
-    from datetime import datetime, timezone
-    return datetime.now(timezone.utc).strftime("%Y-%m-%d")
+    """Default date for the HF daily-papers fetch — YESTERDAY in UTC.
+
+    HF's daily-papers feed is bucketed by *publication* day (UTC). A bucket
+    is populated when HF's editorial pipeline finishes that day's roundup.
+    Empirically that happens in the late afternoon UTC; running before that
+    against today's date returns HTTP 400.
+
+    We default to (today_utc - 1 day) so the cron is robust against
+    schedule drift (Actions can fire 5-30 minutes late past midnight UTC)
+    AND against HF publishing slowly. The fallback in `_fetch_daily_json`
+    will walk further back if even yesterday's bucket is unpopulated.
+
+    Tests / manual runs can still pass an explicit `--date YYYY-MM-DD`.
+    """
+    from datetime import datetime, timedelta, timezone
+    return (datetime.now(timezone.utc) - timedelta(days=1)).strftime("%Y-%m-%d")
 
 
 def cli_main(argv: Sequence[str] | None = None) -> int:

tests/unit/test_hf_daily_papers.py

@@ -39,19 +39,20 @@ class TestFetchTopPapers:
     def test_sorts_by_upvotes_desc(self) -> None:
         raw = [_entry("2605.01000", "low", 1), _entry("2605.02000", "high", 100),
                _entry("2605.03000", "mid", 50)]
-        out = fetch_top_papers("2026-05-13", limit=3, raw_json=raw)
+        effective_date, out = fetch_top_papers("2026-05-13", limit=3, raw_json=raw)
+        assert effective_date == "2026-05-13"  # raw_json passed → no fallback
         assert [p.upvotes for p in out] == [100, 50, 1]
         assert out[0].arxiv_id == "2605.02000"
 
     def test_ties_break_by_arxiv_id(self) -> None:
         # Deterministic tie-break so reruns + tests are stable.
         raw = [_entry("2605.09000", "B", 10), _entry("2605.01000", "A", 10)]
-        out = fetch_top_papers("2026-05-13", limit=2, raw_json=raw)
+        _, out = fetch_top_papers("2026-05-13", limit=2, raw_json=raw)
         assert [p.arxiv_id for p in out] == ["2605.01000", "2605.09000"]
 
     def test_limit_truncates(self) -> None:
         raw = [_entry(f"2605.{i:05d}", f"t{i}", i) for i in range(20)]
-        out = fetch_top_papers("2026-05-13", limit=5, raw_json=raw)
+        _, out = fetch_top_papers("2026-05-13", limit=5, raw_json=raw)
         assert len(out) == 5
         # newest upvotes first
         assert out[0].upvotes == 19
@@ -65,12 +66,12 @@ def test_malformed_entries_skipped(self) -> None:
             {"paper": {"id": "2605.02000", "title": "", "upvotes": 99}},  # missing title
             "not a dict",                              # type error
         ]
-        out = fetch_top_papers("2026-05-13", limit=10, raw_json=raw)
+        _, out = fetch_top_papers("2026-05-13", limit=10, raw_json=raw)
         assert [p.arxiv_id for p in out] == ["2605.01000"]
 
     def test_non_int_upvotes_coerced_to_zero(self) -> None:
         raw = [{"paper": {"id": "2605.0", "title": "weird", "upvotes": "many"}}]
-        out = fetch_top_papers("2026-05-13", limit=1, raw_json=raw)
+        _, out = fetch_top_papers("2026-05-13", limit=1, raw_json=raw)
         assert out[0].upvotes == 0
 
 
@@ -131,3 +132,108 @@ def test_no_papers_to_file_yields_empty_result(self) -> None:
         assert result.fetched == 0
         assert result.filed == []
         assert result.skipped == []
+
+
+class TestDateFallback:
+    """Tests for the date-bucket fallback chain added in the post-spec-010 fix.
+
+    HF daily-papers buckets are populated by HF's editorial pipeline in the
+    late afternoon UTC. Running at 23:59 UTC + cron schedule drift can put
+    the request past midnight (the new day's bucket is empty → HTTP 400).
+    """
+
+    def test_today_utc_defaults_to_yesterday(self) -> None:
+        from datetime import datetime, timedelta, timezone
+
+        from llmxive.hf_daily_papers import _today_utc
+
+        expected = (datetime.now(timezone.utc) - timedelta(days=1)).strftime("%Y-%m-%d")
+        assert _today_utc() == expected
+
+    def test_fetch_falls_back_on_400(self, monkeypatch) -> None:
+        """When the requested date returns 400, walk back one day and retry."""
+        import urllib.error
+
+        from llmxive.hf_daily_papers import _fetch_daily_json
+
+        calls: list[str] = []
+
+        def fake_one(date: str, *, timeout: float = 30.0):
+            calls.append(date)
+            if date == "2026-05-15":
+                raise urllib.error.HTTPError(
+                    url="x", code=400, msg="Bad Request", hdrs=None, fp=None,
+                )
+            return [{"paper": {"id": "2605.fallback", "title": "ok", "upvotes": 1}}]
+
+        monkeypatch.setattr(
+            "llmxive.hf_daily_papers._fetch_daily_json_one", fake_one
+        )
+        effective, data = _fetch_daily_json("2026-05-15", fallback_days=1)
+        assert effective == "2026-05-14"
+        assert calls == ["2026-05-15", "2026-05-14"]
+        assert data[0]["paper"]["id"] == "2605.fallback"
+
+    def test_fetch_falls_back_on_404(self, monkeypatch) -> None:
+        """404 (alternative HF response shape) also triggers fallback."""
+        import urllib.error
+
+        from llmxive.hf_daily_papers import _fetch_daily_json
+
+        def fake_one(date: str, *, timeout: float = 30.0):
+            if date == "2026-05-15":
+                raise urllib.error.HTTPError(
+                    url="x", code=404, msg="Not Found", hdrs=None, fp=None,
+                )
+            return []
+
+        monkeypatch.setattr(
+            "llmxive.hf_daily_papers._fetch_daily_json_one", fake_one
+        )
+        effective, data = _fetch_daily_json("2026-05-15", fallback_days=1)
+        assert effective == "2026-05-14"
+        assert data == []
+
+    def test_fetch_does_not_swallow_5xx(self, monkeypatch) -> None:
+        """5xx errors should propagate — they indicate transient HF issues,
+        not a date-bucket problem."""
+        import urllib.error
+
+        import pytest
+
+        from llmxive.hf_daily_papers import _fetch_daily_json
+
+        def fake_one(date: str, *, timeout: float = 30.0):
+            raise urllib.error.HTTPError(
+                url="x", code=503, msg="Service Unavailable", hdrs=None, fp=None,
+            )
+
+        monkeypatch.setattr(
+            "llmxive.hf_daily_papers._fetch_daily_json_one", fake_one
+        )
+        with pytest.raises(urllib.error.HTTPError) as excinfo:
+            _fetch_daily_json("2026-05-15", fallback_days=1)
+        assert excinfo.value.code == 503
+
+    def test_fetch_raises_when_all_fallback_attempts_400(
+        self, monkeypatch
+    ) -> None:
+        """If every date in the fallback chain returns 400, raise the last
+        HTTPError so the caller can surface a meaningful failure."""
+        import urllib.error
+
+        import pytest
+
+        from llmxive.hf_daily_papers import _fetch_daily_json
+
+        def fake_one(date: str, *, timeout: float = 30.0):
+            raise urllib.error.HTTPError(
+                url="x", code=400, msg="Bad Request", hdrs=None, fp=None,
+            )
+
+        monkeypatch.setattr(
+            "llmxive.hf_daily_papers._fetch_daily_json_one", fake_one
+        )
+        with pytest.raises(urllib.error.HTTPError) as excinfo:
+            _fetch_daily_json("2026-05-15", fallback_days=2)
+        assert excinfo.value.code == 400