crawl_docs_v3.py

# -*- coding: utf-8 -*-
"""
Pico XR 文档全量爬虫 V3
- 覆盖 picoxr.com 和 pico-interactive.com 两个域名
- 先用 Crawl4AI 渲染导航页自动发现子页面 URL
- 更宽松的 wait_for 策略，修复 design 页面超时
- 增量爬取，跳过已爬取的 URL
"""

import sys
import io
import os

# Force Crawl4AI to use project-local data directory (avoids sandbox permission issues)
# CRAWL4_AI_BASE_DIRECTORY is the env var crawl4ai respects; it appends ".crawl4ai" to it
_CRAWL4AI_BASE = os.path.dirname(os.path.abspath(__file__))
os.environ["CRAWL4_AI_BASE_DIRECTORY"] = _CRAWL4AI_BASE
# Pre-create the directories crawl4ai expects
os.makedirs(os.path.join(_CRAWL4AI_BASE, ".crawl4ai", "robots"), exist_ok=True)
os.makedirs(os.path.join(_CRAWL4AI_BASE, ".crawl4ai", "cache"), exist_ok=True)
os.makedirs(os.path.join(_CRAWL4AI_BASE, ".crawl4ai", "models"), exist_ok=True)

sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace')
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace')

import asyncio
import json
import re
import time
from pathlib import Path
from urllib.parse import urlparse, urljoin

from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode

DATA_DIR = Path(__file__).parent / "data"
DATA_DIR.mkdir(exist_ok=True)
DISCOVERY_ROUNDS = 3

# 导航/发现页面 - 用于自动提取子链接
DISCOVERY_URLS = [
    "https://developer-cn.picoxr.com/document/",
    "https://developer-cn.picoxr.com/document/unity/overview",
    "https://developer-cn.picoxr.com/document/unreal/overview",
    "https://developer-cn.picoxr.com/document/native/overview",
    "https://developer-cn.picoxr.com/document/openxr/openxr-overview",
    "https://developer-cn.picoxr.com/document/discover/xr-devices",
    "https://developer-cn.picoxr.com/document/design/game-genres",
    "https://developer-cn.picoxr.com/document/design/visual-design",
    "https://developer-cn.picoxr.com/document/design/locomotion",
    "https://developer-cn.picoxr.com/document/design/dock-icon-and-album-cover",
    "https://developer-cn.picoxr.com/document/distribute/app-distribution-overview",
    "https://developer-cn.picoxr.com/document/distribute/app-review-overview",
    "https://developer-cn.picoxr.com/document/safe-info",
    # pico-interactive.com 导航页
    "https://developer-cn.pico-interactive.com/document/unity/splash-screen",
    "https://developer-cn.pico-interactive.com/document/unity/input-mapping",
    "https://developer-cn.pico-interactive.com/document/unity/seethrough",
    "https://developer-cn.pico-interactive.com/document/unity/spatial-audio",
    "https://developer-cn.pico-interactive.com/document/unity/metrics-hud",
]

# 确定需要爬取的详细页面（作为种子 URL，发现阶段还会扩展）
SEED_URLS = [
    # === Unity SDK (picoxr.com) ===
    "https://developer-cn.picoxr.com/document/unity/overview",
    "https://developer-cn.picoxr.com/document/unity/splash-screen",
    "https://developer-cn.picoxr.com/document/unity/input-mapping",
    "https://developer-cn.picoxr.com/document/unity/seethrough",
    "https://developer-cn.picoxr.com/document/unity/spatial-audio",
    "https://developer-cn.picoxr.com/document/unity/metrics-hud",
    "https://developer-cn.picoxr.com/document/unity/renderdoc-for-pico",
    "https://developer-cn.picoxr.com/document/unity/command-line-utility",
    "https://developer-cn.picoxr.com/document/unity/242767",
    # Unity 平台服务
    "https://developer-cn.picoxr.com/document/unity/accounts-and-friends",
    "https://developer-cn.picoxr.com/document/unity/interaction",
    "https://developer-cn.picoxr.com/document/unity/achievements",
    "https://developer-cn.picoxr.com/document/unity/matchmaking",
    "https://developer-cn.picoxr.com/document/unity/leaderboard",
    "https://developer-cn.picoxr.com/document/unity/in-app-purchase",
    "https://developer-cn.picoxr.com/document/unity/downloadable-content",
    "https://developer-cn.picoxr.com/document/unity/subscription",
    "https://developer-cn.picoxr.com/document/unity/cloud-storage",
    "https://developer-cn.picoxr.com/document/unity/rtc",
    "https://developer-cn.picoxr.com/document/unity/avatar",
    "https://developer-cn.picoxr.com/document/unity/body-tracking",
    "https://developer-cn.picoxr.com/document/unity/hand-tracking",
    "https://developer-cn.picoxr.com/document/unity/eye-tracking",
    "https://developer-cn.picoxr.com/document/unity/stereo-rendering",
    "https://developer-cn.picoxr.com/document/unity/foveated-rendering",
    "https://developer-cn.picoxr.com/document/unity/multi-app",
    "https://developer-cn.picoxr.com/document/unity/pico-webview",
    "https://developer-cn.picoxr.com/document/unity/custom-home",
    "https://developer-cn.picoxr.com/document/unity/spatial-anchors",
    "https://developer-cn.picoxr.com/document/unity/scene-mesh",
    "https://developer-cn.picoxr.com/document/unity/auto-stereoscopic",
    "https://developer-cn.picoxr.com/document/unity/pxrc-guide",
    "https://developer-cn.picoxr.com/document/unity/pico-profile",
    # === Unreal SDK ===
    "https://developer-cn.picoxr.com/document/unreal/unreal-sdk-overview",
    "https://developer-cn.picoxr.com/document/unreal/splash-screen",
    "https://developer-cn.picoxr.com/document/unreal/input-mapping",
    "https://developer-cn.picoxr.com/document/unreal/seethrough",
    "https://developer-cn.picoxr.com/document/unreal/spatial-audio",
    "https://developer-cn.picoxr.com/document/unreal/metrics-hud",
    "https://developer-cn.picoxr.com/document/unreal/rendering",
    "https://developer-cn.picoxr.com/document/unreal/platform-service",
    "https://developer-cn.picoxr.com/document/unreal/interaction",
    "https://developer-cn.picoxr.com/document/unreal/achievements",
    "https://developer-cn.picoxr.com/document/unreal/leaderboard",
    "https://developer-cn.picoxr.com/document/unreal/in-app-purchase",
    "https://developer-cn.picoxr.com/document/unreal/cloud-storage",
    "https://developer-cn.picoxr.com/document/unreal/matchmaking",
    "https://developer-cn.picoxr.com/document/unreal/subscription",
    # === Native SDK ===
    "https://developer-cn.picoxr.com/document/native/native-sdk-overview",
    "https://developer-cn.picoxr.com/document/native/rendering",
    "https://developer-cn.picoxr.com/document/native/input",
    "https://developer-cn.picoxr.com/document/native/mixed-reality",
    "https://developer-cn.picoxr.com/document/native/spatial-audio",
    "https://developer-cn.picoxr.com/document/native/performance",
    "https://developer-cn.picoxr.com/document/native/platform-service",
    # === OpenXR ===
    "https://developer-cn.picoxr.com/document/openxr/openxr-overview",
    "https://developer-cn.picoxr.com/document/openxr/rendering",
    "https://developer-cn.picoxr.com/document/openxr/input",
    "https://developer-cn.picoxr.com/document/openxr/mixed-reality",
    "https://developer-cn.picoxr.com/document/openxr/spatial-audio",
    # === 设备介绍 ===
    "https://developer-cn.picoxr.com/document/discover/xr-devices",
    # === 设计指南 ===
    "https://developer-cn.picoxr.com/document/design/game-genres",
    "https://developer-cn.picoxr.com/document/design/visual-design",
    "https://developer-cn.picoxr.com/document/design/locomotion",
    "https://developer-cn.picoxr.com/document/design/dock-icon-and-album-cover",
    # === 分发上架 ===
    "https://developer-cn.picoxr.com/document/distribute/app-distribution-overview",
    "https://developer-cn.picoxr.com/document/distribute/app-review-overview",
    "https://developer-cn.picoxr.com/document/distribute/create-a-developer-account",
    "https://developer-cn.picoxr.com/document/distribute/create-an-app",
    "https://developer-cn.picoxr.com/document/distribute/app-information-review",
    "https://developer-cn.picoxr.com/document/distribute/formal-app",
    "https://developer-cn.picoxr.com/document/distribute/analytics-IAP",
    # === 安全信息 ===
    "https://developer-cn.picoxr.com/document/safe-info",
    # === pico-interactive.com 文档 ===
    "https://developer-cn.pico-interactive.com/document/unity/splash-screen",
    "https://developer-cn.pico-interactive.com/document/unity/input-mapping",
    "https://developer-cn.pico-interactive.com/document/unity/seethrough",
    "https://developer-cn.pico-interactive.com/document/unity/spatial-audio",
    "https://developer-cn.pico-interactive.com/document/unity/metrics-hud",
    "https://developer-cn.pico-interactive.com/document/unity/242767",
    "https://developer-cn.pico-interactive.com/document/unity/renderdoc-for-pico",
    "https://developer-cn.pico-interactive.com/document/unity/command-line-utility",
    "https://developer-cn.pico-interactive.com/document/unity/accounts-and-friends",
    "https://developer-cn.pico-interactive.com/document/unity/interaction",
    "https://developer-cn.pico-interactive.com/document/unity/achievements",
    "https://developer-cn.pico-interactive.com/document/unity/matchmaking",
    "https://developer-cn.pico-interactive.com/document/unity/leaderboard",
    "https://developer-cn.pico-interactive.com/document/unity/in-app-purchase",
    "https://developer-cn.pico-interactive.com/document/unity/downloadable-content",
    "https://developer-cn.pico-interactive.com/document/unity/subscription",
    "https://developer-cn.pico-interactive.com/document/unity/cloud-storage",
    "https://developer-cn.pico-interactive.com/document/unity/rtc",
]


def get_category(url: str) -> str:
    path = urlparse(url).path
    if "unity" in path or "/ffr" in path or "/stereo-rendering" in path or "/spatial-anchor" in path or "/scene-mesh" in path:
        return "Unity SDK"
    elif "unreal" in path:
        return "Unreal SDK"
    elif "native" in path:
        return "Native SDK"
    elif "openxr" in path:
        return "OpenXR"
    elif "/discover/" in path:
        return "设备介绍"
    elif "/design/" in path or path in ["/document/locomotion", "/document/game-genres", "/document/visual-design"]:
        return "设计指南"
    elif "/distribute/" in path:
        return "分发上架"
    elif "/safe-info" in path:
        return "安全信息"
    else:
        return "其他"


def normalize_url(url: str) -> str:
    """Normalize URL: remove fragment, trailing slash"""
    url = url.split("#")[0].rstrip("/")
    return url


def clean_markdown(text: str) -> str:
    if not text:
        return ""
    text = re.sub(r"\n{4,}", "\n\n\n", text)
    return text.strip()


def strip_pico_page_shell(markdown: str, title: str = "") -> str:
    """Remove Pico site navigation/header shell from captured markdown while preserving doc body."""
    markdown = clean_markdown(markdown)
    if not markdown:
        return ""

    shell_markers = ["[博客]", "[开发资源]", "[文档]", "[开发者平台]"]
    has_shell = sum(1 for marker in shell_markers if marker in markdown[:1200]) >= 3
    if not has_shell:
        return strip_pico_footer(markdown)

    candidates = []
    if title:
        title_index = markdown.find(title)
        if 0 <= title_index <= 2000:
            candidates.append(title_index)

    lines = markdown.splitlines()
    offset = 0
    for line in lines:
        stripped = line.strip()
        if stripped and not stripped.startswith("[](") and not stripped.startswith("![](") and not stripped.startswith("···") and not stripped.startswith("["):
            if stripped not in ("重试", "## 出错了！"):
                if offset <= 2000:
                    candidates.append(offset)
                break
        offset += len(line) + 1

    if not candidates:
        return markdown
    return strip_pico_footer(markdown[min(candidates):])


def extract_doc_markdown(result) -> str:
    """Return markdown, preferring the rendered Pico document content over the page shell."""
    html = getattr(result, "html", "") or ""
    matches = re.findall(r'<(?:div|section)[^>]+class=["\'][^"\']*DocArcosite_content[^"\']*["\'][^>]*>[\s\S]*?(?=<body|</body>|</html>|$)', html)
    if matches:
        body_html = max(matches, key=len)
        body_html = re.sub(r"<script[\s\S]*?</script>", "", body_html, flags=re.I)
        body_html = re.sub(r"<style[\s\S]*?</style>", "", body_html, flags=re.I)
        body_html = re.sub(r"<img[^>]+src=[\"']([^\"']+)[\"'][^>]*>", r"\n![](\1)\n", body_html, flags=re.I)
        body_html = re.sub(r"</(h[1-6]|p|div|section|li|tr|table|ul|ol)>", "\n", body_html, flags=re.I)
        body_html = re.sub(r"<br\s*/?>", "\n", body_html, flags=re.I)
        text = re.sub(r"<[^>]+>", "", body_html)
        text = text.replace("&nbsp;", " ").replace("&amp;", "&").replace("&lt;", "<").replace("&gt;", ">")
        return strip_pico_page_shell(text)
    if hasattr(result.markdown, "raw_markdown"):
        markdown = strip_pico_page_shell(result.markdown.raw_markdown)
        if not is_placeholder_doc(markdown):
            return markdown
    if hasattr(result.markdown, "fit_markdown") and result.markdown.fit_markdown:
        return strip_pico_page_shell(result.markdown.fit_markdown)
    return strip_pico_page_shell(str(result.markdown))


def is_placeholder_doc(markdown: str) -> bool:
    """Return True for rendered Pico placeholder pages, not real docs."""
    if not markdown:
        return True
    if "文档不存在" in markdown or "Document does not exist" in markdown:
        return True
    # Crawl4AI can capture only the global Pico shell/nav when the SPA content is missed.
    shell_markers = ["博客", "开发资源", "开发者平台", "硬件设备"]
    return len(markdown) < 1800 and all(marker in markdown for marker in shell_markers)


def canonical_doc_url(url: str) -> str:
    """Map invalid old section URLs to current flat Pico document URLs when known."""
    mappings = {
        "/document/unity/avatar": "/document/unity-avatar",
        "/document/unity/multi-app": "/document/multi-application",
        "/document/unity/custom-home": "/document/custom-home",
        "/document/unity/spatial-anchors": "/document/spatial-anchor",
        "/document/unity/scene-mesh": "/document/scene-mesh",
        "/document/unity/pico-webview": "/document/pico-webview",
        "/document/unity/pxrc-guide": "/document/pxrc-overview",
        "/document/unity/pico-profile": "/document/pico-profile",
        "/document/unity/auto-stereoscopic": "/document/auto-stereoscopic-overview",
        "/document/unity/foveated-rendering": "/document/ffr",
        "/document/unity/stereo-rendering": "/document/stereo-rendering",
        "/document/native/input": "/document/native-input",
        "/document/native/rendering": "/document/native-rendering",
        "/document/native/mixed-reality": "/document/native-mr",
        "/document/native/spatial-audio": "/document/native-spatial-audio",
        "/document/native/platform-service": "/document/native-platform-services",
        "/document/openxr/input": "/document/openxr-input",
        "/document/openxr/rendering": "/document/openxr-rendering",
        "/document/openxr/mixed-reality": "/document/openxr-mr",
        "/document/openxr/spatial-audio": "/document/openxr-spatial-audio",
        "/document/unreal/input-mapping": "/document/unreal-input",
        "/document/unreal/rendering": "/document/unreal-rendering",
        "/document/unreal/platform-service": "/document/unreal-platform-services",
        "/document/design/locomotion": "/document/locomotion",
        "/document/design/game-genres": "/document/game-genres",
        "/document/design/visual-design": "/document/visual-design",
    }
    parsed = urlparse(url)
    path = parsed.path.rstrip("/")
    if path in mappings:
        return f"{parsed.scheme}://{parsed.netloc}{mappings[path]}"
    if parsed.netloc == "developer-cn.picoxr.com" and path.startswith("/document/unity/"):
        legacy_slugs = {
            "242767", "accounts-and-friends", "achievements", "cloud-storage",
            "command-line-utility", "downloadable-content", "in-app-purchase",
            "input-mapping", "interaction", "leaderboard", "matchmaking",
            "metrics-hud", "renderdoc-for-pico", "rtc", "seethrough",
            "spatial-audio", "splash-screen", "subscription",
        }
        slug = path.rsplit("/", 1)[-1]
        if slug in legacy_slugs:
            return f"https://developer-cn.pico-interactive.com/document/unity/{slug}"
    return url


def dedupe_docs(docs: list[dict]) -> list[dict]:
    """Keep one doc per URL and drop placeholder pages."""
    by_url = {}
    for doc in docs:
        url = normalize_url(doc.get("url", ""))
        if not url or not is_content_doc_url(url) or is_placeholder_doc(doc.get("markdown", "")):
            continue
        previous = by_url.get(url)
        if previous is None or len(doc.get("markdown", "")) > len(previous.get("markdown", "")):
            by_url[url] = doc
    return sorted(by_url.values(), key=lambda d: d.get("url", ""))


def load_existing_docs() -> dict:
    """Load existing docs, return {normalized_url: doc} dict"""
    docs_file = DATA_DIR / "pico_docs_full.json"
    if not docs_file.exists():
        return {}
    try:
        with open(docs_file, "r", encoding="utf-8") as f:
            docs = json.load(f)
        return {normalize_url(canonical_doc_url(d["url"])): d for d in docs if d.get("url") and not is_placeholder_doc(d.get("markdown", ""))}
    except Exception:
        return {}


def save_docs(docs: list[dict]):
    for doc in docs:
        doc["markdown"] = strip_pico_page_shell(doc.get("markdown", ""), doc.get("title", ""))
        if not doc.get("title"):
            doc["title"] = title_from_markdown(doc.get("markdown", ""))
    docs = dedupe_docs(docs)
    with open(DATA_DIR / "pico_docs_full.json", "w", encoding="utf-8") as f:
        json.dump(docs, f, ensure_ascii=False, indent=2)
    
    index = [
        {
            "id": doc["id"],
            "url": doc["url"],
            "title": doc["title"],
            "category": doc["category"],
            "summary": doc["markdown"][:300] + "..." if len(doc["markdown"]) > 300 else doc["markdown"],
        }
        for doc in docs
    ]
    with open(DATA_DIR / "pico_docs_index.json", "w", encoding="utf-8") as f:
        json.dump(index, f, ensure_ascii=False, indent=2)
    
    print(f"  [SAVE] {len(docs)} docs saved")


def is_doc_url(url: str) -> bool:
    parsed = urlparse(normalize_url(url))
    if parsed.netloc not in ("developer-cn.picoxr.com", "developer-cn.pico-interactive.com"):
        return False
    path = parsed.path.rstrip("/")
    if not path.startswith("/document"):
        return False
    skip_exts = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", ".pdf", ".zip", ".apk")
    return not path.lower().endswith(skip_exts)


def is_content_doc_url(url: str) -> bool:
    path = urlparse(normalize_url(url)).path.rstrip("/")
    return is_doc_url(url) and path != "/document"


def title_from_markdown(markdown: str) -> str:
    for line in markdown.splitlines():
        title = line.strip().lstrip("#").strip()
        if not title:
            continue
        if title.startswith(("[](", "![](", "[", "···")):
            continue
        if title in ("上一篇", "下一篇", "关闭", "重试", "## 出错了！"):
            continue
        if len(title) > 80:
            continue
        return title
    return ""


def strip_pico_footer(markdown: str) -> str:
    markers = ["\n上一篇\n", "\n关闭\n", "\n硬件设备\n", "\n工具\n", "\nAll rights reserved."]
    cut_points = []
    for marker in markers:
        index = markdown.find(marker)
        if index > 500:
            cut_points.append(index)
    if cut_points:
        markdown = markdown[:min(cut_points)]
    return clean_markdown(markdown)


def extract_document_links(result) -> set:
    """Extract absolute Pico document links from rendered HTML and Crawl4AI link metadata."""
    hrefs = set(re.findall(r'href=["\\\']([^"\\\']+)', getattr(result, "html", "") or ""))
    links = getattr(result, "links", {}) or {}
    hrefs.update(link.get("href", "") for link in links.get("internal", []))
    urls = set()
    for href in hrefs:
        if not href or href.startswith(("javascript:", "mailto:", "tel:")):
            continue
        absolute = normalize_url(urljoin(result.url, href))
        if is_content_doc_url(absolute):
            urls.add(absolute)
    return urls


async def discover_urls(crawler: AsyncWebCrawler) -> set:
    """Render document pages recursively to discover all reachable Pico document URLs."""
    print("\n" + "=" * 60)
    print("[PHASE 1] Discovering URLs from document pages...")
    print("=" * 60)

    discovered = set(normalize_url(url) for url in DISCOVERY_URLS + SEED_URLS if is_content_doc_url(url))
    frontier = set(normalize_url(url) for url in DISCOVERY_URLS + SEED_URLS if is_doc_url(url))
    scanned = set()

    discovery_config = CrawlerRunConfig(
        cache_mode=CacheMode.BYPASS,
        page_timeout=45000,
        remove_overlay_elements=True,
        wait_for="css:body",
        delay_before_return_html=6,
    )

    batch_size = 4
    for round_num in range(1, DISCOVERY_ROUNDS + 1):
        round_urls = sorted(frontier - scanned)
        if not round_urls:
            break
        print(f"\n  [DISCOVER] round {round_num}/{DISCOVERY_ROUNDS}: {len(round_urls)} pages")
        for i in range(0, len(round_urls), batch_size):
            batch = round_urls[i:i+batch_size]
            print(f"    [DISCOVER] batch {i//batch_size+1}: {len(batch)} pages")
            try:
                results = await crawler.arun_many(urls=batch, config=discovery_config)
                for result in results:
                    scanned.add(normalize_url(getattr(result, "url", "")))
                    if not result.success:
                        print(f"      [SKIP] {result.url}")
                        continue
                    links = extract_document_links(result)
                    before = len(discovered)
                    discovered.update(links)
                    frontier.update(links)
                    print(f"      [OK] {result.url} -> +{len(discovered) - before}, total: {len(discovered)}")
            except Exception as e:
                print(f"      [ERR] {e}")
            await asyncio.sleep(1)
    return discovered


async def crawl_all():
    browser_config = BrowserConfig(
        headless=True,
        viewport_width=1920,
        viewport_height=1080,
    )
    
    # Content crawl config - wait for the SPA shell, then extract the rendered Pico doc body when present.
    content_config = CrawlerRunConfig(
        cache_mode=CacheMode.BYPASS,
        page_timeout=60000,
        remove_overlay_elements=True,
        exclude_external_links=True,
        excluded_tags=["nav", "footer", "header"],
        css_selector="[class*=DocArcosite_content], body",
        wait_for="css:body",
        delay_before_return_html=8,
    )
    
    all_docs = []
    failed_urls = []
    
    # Load existing docs for incremental crawling
    existing = {} if os.environ.get("PICO_FORCE_RECRAWL") == "1" else load_existing_docs()
    print(f"[INFO] Found {len(existing)} existing docs")
    
    async with AsyncWebCrawler(config=browser_config) as crawler:
        # Phase 1: Discover URLs from navigation pages
        discovered_urls = await discover_urls(crawler)
        
        # Merge discovered URLs with seed URLs
        all_urls = set()
        for url in SEED_URLS:
            all_urls.add(normalize_url(canonical_doc_url(url)))
        for url in discovered_urls:
            all_urls.add(normalize_url(canonical_doc_url(url)))
        
        # Remove already crawled URLs
        to_crawl = [url for url in all_urls if url not in existing]
        
        print("\n" + "=" * 60)
        print(f"[PHASE 2] Crawling {len(to_crawl)} pages (skipping {len(existing)} already crawled)")
        print(f"[INFO] Total unique URLs: {len(all_urls)}")
        print("=" * 60)
        
        # Convert to sorted list for deterministic order
        to_crawl = sorted(to_crawl)
        
        batch_size = 3
        
        for i in range(0, len(to_crawl), batch_size):
            batch = to_crawl[i:i+batch_size]
            batch_num = i // batch_size + 1
            total_batches = (len(to_crawl) + batch_size - 1) // batch_size
            print(f"\n[BATCH] {batch_num}/{total_batches}: {len(batch)} pages")
            
            try:
                results = await crawler.arun_many(urls=batch, config=content_config)
                
                for result in results:
                    url = normalize_url(getattr(result, 'url', '?'))
                    
                    if not result.success:
                        failed_urls.append(url)
                        print(f"  [ERR] {url}: {getattr(result, 'error_message', 'unknown')}")
                        continue
                    
                    title = result.metadata.get("title", "") or ""
                    title = re.sub(r"\s*[-|—]\s*PICO.*$", "", title, flags=re.IGNORECASE).strip()
                    title = re.sub(r"\s*[-|—]\s*Pico.*$", "", title, flags=re.IGNORECASE).strip()
                    
                    markdown = extract_doc_markdown(result)
                    title = title or title_from_markdown(markdown)

                    if len(markdown) < 100 or is_placeholder_doc(markdown):
                        print(f"  [SKIP] placeholder/too short: {url} ({len(markdown)} chars)")
                        continue
                    
                    path_parts = [p for p in urlparse(url).path.strip("/").split("/") if p]
                    doc_id = "_".join(path_parts[-3:]) if len(path_parts) >= 3 else "_".join(path_parts)
                    
                    doc = {
                        "id": doc_id,
                        "url": url,
                        "title": title,
                        "category": get_category(url),
                        "markdown": markdown,
                        "crawled_at": time.strftime("%Y-%m-%dT%H:%M:%S"),
                    }
                    all_docs.append(doc)
                    print(f"  [OK] [{len(all_docs)}] {title[:50]} ({len(markdown)} chars)")
                
            except Exception as e:
                print(f"  [ERR] batch error: {e}")
                failed_urls.extend(batch)
            
            # Save intermediate results every 3 batches
            if all_docs and batch_num % 3 == 0:
                combined = list(existing.values()) + all_docs
                save_docs(combined)
            
            await asyncio.sleep(1)
    
    # Final save: merge with existing docs
    combined = list(existing.values()) + all_docs
    save_docs(combined)
    
    # Save discovered URLs for reference
    with open(DATA_DIR / "discovered_urls.json", "w", encoding="utf-8") as f:
        json.dump(sorted(all_urls), f, ensure_ascii=False, indent=2)
    
    if failed_urls:
        with open(DATA_DIR / "failed_urls.json", "w", encoding="utf-8") as f:
            json.dump(failed_urls, f, ensure_ascii=False, indent=2)
        print(f"\n[WARN] {len(failed_urls)} pages failed")
    
    # Print summary
    print("\n" + "=" * 60)
    print(f"[DONE] Crawled {len(all_docs)} new docs")
    print(f"[DONE] Total docs in knowledge base: {len(combined)}")
    
    cats = {}
    total_chars = 0
    for d in combined:
        c = d.get("category", "other")
        cats[c] = cats.get(c, 0) + 1
        total_chars += len(d.get("markdown", ""))
    
    print(f"[DONE] Total markdown chars: {total_chars:,}")
    print("[DONE] Categories:")
    for c in sorted(cats.keys()):
        print(f"  {c}: {cats[c]} docs")
    
    print(f"[SAVE] Data directory: {DATA_DIR}")
    print("=" * 60)


if __name__ == "__main__":
    asyncio.run(crawl_all())