fix: unbreak scheduled PR pipeline after action parsing change

lroolle · claude · lroolle · commit 58be5e632916 · 2026-07-16T01:52:41.000+08:00
Since ~June 14 every scheduled run pushed a docs branch, failed silently on 'gh pr create', barked a notification, and exited green: claude-code-action@main changed claude_args parsing so 'Bash(gh pr:*)' split on the space into two invalid permission rules. Result: 31 orphan branches, zero PRs, three weeks of stale main. - Move tool permissions to .claude/settings.json (JSON survives any arg parsing; settingSources=project already loads it in CI) - Pin claude-code-action to v1.0.168 instead of the moving @main - Add a Verify outcome tripwire: pushed branch without a PR, or uncommitted changes after the agent step, now fail the run - Bark protocol: notify only after a real PR URL exists; bark a failure alert otherwise - fetcher: record failures in .metadata.json (they were discarded), print per-host failure breakdown, support JINA_API_KEY (r.jina.ai rate limiting caused 574 of 731 failures), skip terraform pages that serve no .md variant (the other 157) Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
diff --git a/.claude/settings.json b/.claude/settings.json
@@ -0,0 +1,13 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(gh pr:*)",
+      "Bash(git:*)",
+      "Bash(npm:*)",
+      "Bash(jq:*)",
+      "Bash(date:*)",
+      "Bash(./scripts/create-docs-pr.sh:*)",
+      "mcp__barkme__notify"
+    ]
+  }
+}
diff --git a/.github/workflows/fetch-claude-docs.yml b/.github/workflows/fetch-claude-docs.yml
@@ -2,7 +2,7 @@ name: Fetch Claude Code Docs
 
 on:
   schedule:
-    # Every 6 hours: 00:00, 06:00, 12:00, 18:00 UTC
+    # Four times daily: 01:00, 06:00, 11:00, 16:00 UTC (aligned with hn-digest)
     - cron: "0 1,6,11,16 * * *"
   workflow_dispatch: # Allow manual trigger
 
@@ -45,6 +45,7 @@ jobs:
       - name: Fetch all Anthropic docs
         env:
           GITHUB_TOKEN: ${{ steps.app-token.outputs.token }}
+          JINA_API_KEY: ${{ secrets.JINA_API_KEY }}
         run: uv run scripts/fetcher.py
 
       - name: Setup GitHub CLI and Git
@@ -55,7 +56,11 @@ jobs:
           git config --global user.email "claude-yolo@lroole.com"
 
       - name: Claude handles everything
-        uses: anthropics/claude-code-action@main
+        # Pinned: @main changed claude_args parsing in June 2026 and silently
+        # broke 'Bash(gh pr:*)' (split on the space into two invalid rules)
+        # -> 3 weeks of pushed branches with no PRs. Tool permissions now live
+        # in .claude/settings.json, which survives arg-parsing changes.
+        uses: anthropics/claude-code-action@v1.0.168
         env:
           GITHUB_TOKEN: ${{ steps.app-token.outputs.token }}
         with:
@@ -64,7 +69,6 @@ jobs:
           allowed_bots: "claude-yolo[bot]"
           claude_args: |
             --model claude-sonnet-4-6
-            --allowedTools Bash(gh pr:*),Bash(git:*),Bash(npm:*),Bash(jq:*),Bash(date:*),mcp__barkme__notify
             --mcp-config '{
               "mcpServers": {
                 "barkme": {
@@ -178,13 +182,39 @@ jobs:
             - Highlight the interesting bits
 
             ## BARKME PROTOCOL (nightshift notifications)
-            Always barkme when creating PR (one simple notification):
+            Bark AFTER the PR exists, never before:
+            - Only notify once gh pr create / create-docs-pr.sh returned a real PR URL
             - Title: "📦 Claude Code v{version}"
             - Body: "{key feature}"
-            - URL: Link to PR
+            - URL: the actual PR link (verify it is non-empty before barking)
+
+            If push or PR creation fails: bark a failure alert instead
+            ("🚨 docs pipeline: {step} failed - needs human") and exit non-zero.
+            A notification without a PR behind it is worse than no notification.
 
             Keep it SHORT - day shift will do detailed analysis later.
 
             Remember: You're not a changelog generator. You're a smart filter that understands WHY changes matter to humans using Claude Code.
 
             [>be me >11:45pm Pacific docs checker >Anthropic devs are asleep >catching their fresh updates >day shift will handle the rest]
+
+      # Tripwire: a pushed branch without a PR, or changes left uncommitted,
+      # means the agent step silently failed. Fail loudly instead of lying green.
+      # (June 2026 incident: 31 orphan branches, zero PRs, all runs green.)
+      - name: Verify outcome
+        env:
+          GITHUB_TOKEN: ${{ steps.app-token.outputs.token }}
+        run: |
+          BRANCH=$(git branch --show-current)
+          if [ -n "$(git status --porcelain)" ]; then
+            echo "::error::Workspace has uncommitted changes after agent step"
+            git status --short | head -20
+            exit 1
+          fi
+          if [ "$BRANCH" != "main" ] && [ -n "$BRANCH" ]; then
+            PRS=$(gh pr list --head "$BRANCH" --json number --jq 'length')
+            if [ "$PRS" = "0" ]; then
+              echo "::error::Branch $BRANCH pushed but no PR exists - PR creation silently failed"
+              exit 1
+            fi
+          fi
diff --git a/.gitignore b/.gitignore
@@ -15,8 +15,11 @@ dist/
 # Claude Code
 .claude-trace/
 
-# Local development
-.claude/
+# Local development (root only; keep settings.json tracked: CI reads tool
+# permissions from it via settingSources=project. Note: content/github/**
+# mirrors still ship .claude/ dirs we currently do not archive.)
+/.claude/*
+!/.claude/settings.json
 worktree/
 WIP/
 references/
diff --git a/scripts/fetcher.py b/scripts/fetcher.py
@@ -107,11 +107,20 @@ async def fetch_text(self, session: aiohttp.ClientSession, url: str) -> str:
             r.raise_for_status()
             return await r.text()
 
-    async def fetch_bytes(self, session: aiohttp.ClientSession, url: str) -> bytes:
-        async with session.get(url) as r:
+    async def fetch_bytes(
+        self, session: aiohttp.ClientSession, url: str,
+        headers: Optional[Dict] = None,
+    ) -> bytes:
+        async with session.get(url, headers=headers) as r:
             r.raise_for_status()
             return await r.read()
 
+    def _jina_headers(self) -> Optional[Dict]:
+        # Without a key, r.jina.ai rate-limits hard (~95% failures at 10
+        # concurrent). Set JINA_API_KEY to lift blog/support success rates.
+        key = os.environ.get("JINA_API_KEY")
+        return {"Authorization": f"Bearer {key}"} if key else None
+
     def extract_sitemap_urls(self, xml: str, must_contain: str = "") -> List[str]:
         urls = []
         for line in xml.split("\n"):
@@ -211,7 +220,8 @@ async def download_via_jina(self, session, url, output_subdir, semaphore) -> Dic
                 self.stats["skipped"] += 1
                 return {"url": url, "status": "skipped"}
             try:
-                content = await self.fetch_bytes(session, f"https://r.jina.ai/{url}")
+                content = await self.fetch_bytes(
+                    session, f"https://r.jina.ai/{url}", headers=self._jina_headers())
                 output_path.parent.mkdir(parents=True, exist_ok=True)
                 async with aiofiles.open(output_path, "wb") as f:
                     await f.write(content)
@@ -233,7 +243,8 @@ async def download_support_article(self, session, url, semaphore) -> Dict:
                 self.stats["skipped"] += 1
                 return {"url": url, "status": "skipped"}
             try:
-                content = await self.fetch_bytes(session, f"https://r.jina.ai/{url}")
+                content = await self.fetch_bytes(
+                    session, f"https://r.jina.ai/{url}", headers=self._jina_headers())
                 output_path.parent.mkdir(parents=True, exist_ok=True)
                 async with aiofiles.open(output_path, "wb") as f:
                     await f.write(content)
@@ -342,8 +353,13 @@ async def fetch_all(self):
                 print("Source: platform.claude.com/sitemap.xml")
                 xml = await self.fetch_text(session, self.platform_sitemap_url)
                 urls = self.extract_sitemap_urls(xml, "/docs/en/")
+                # Terraform provider reference serves no .md variant (404s)
+                terraform = [u for u in urls if "/api/terraform/" in u]
+                urls = [u for u in urls if "/api/terraform/" not in u]
                 counts["platform"] = len(urls)
-                print(f"  {len(urls)} docs")
+                print(f"  {len(urls)} docs"
+                      + (f" ({len(terraform)} terraform pages skipped, no .md)"
+                         if terraform else ""))
                 for url in urls:
                     tasks.append(self.download_doc(session, url, semaphore))
 
@@ -413,9 +429,24 @@ async def fetch_all(self):
             if tasks:
                 results = await tqdm_asyncio.gather(*tasks, desc="Fetching", unit="file")
                 await self._save_metadata(results)
+                self._print_failures(results)
 
         self._print_summary()
 
+    def _print_failures(self, results: List[Dict]):
+        failed = [r for r in results if r.get("status") == "failed"]
+        if not failed:
+            return
+        by_host = defaultdict(int)
+        for r in failed:
+            by_host[r["url"].split("/")[2]] += 1
+        print("\nFailed by host:")
+        for host, n in sorted(by_host.items(), key=lambda kv: -kv[1]):
+            print(f"  {host}: {n}")
+        print("Sample errors:")
+        for r in failed[:3]:
+            print(f"  {r['url']}: {str(r.get('error', ''))[:120]}")
+
     async def _fetch_meta(self, session):
         print("Meta: NPM manifest + CHANGELOG")
         try:
@@ -446,6 +477,10 @@ async def _save_metadata(self, results: List[Dict]):
                 "section": self.section or "all",
             },
             "items": [r for r in results if r.get("status") == "success"],
+            "failures": [
+                {"url": r["url"], "error": str(r.get("error", ""))[:200]}
+                for r in results if r.get("status") == "failed"
+            ],
             "summary": {
                 "total": self.stats["total"],
                 "downloaded": self.stats["downloaded"],
