feat: search_engine_indexing usage category — free for allowed crawlers

- New UsageCategory.SEARCH_ENGINE_INDEXING (price 0 when allowed)
- Config: FAIRFETCH_SEARCH_ENGINES_ALLOWED, FAIRFETCH_SEARCH_ENGINES_BLOCKED
- Default allowlist: Googlebot, Bingbot, DuckDuckBot, Slurp, Baidu, Yandex, Sogou, Exabot, etc.
- core/search_engine: is_allowed_search_engine(); middleware free pass for allowed UA
- Non-allowed crawlers pay base (1x) for search_engine_indexing
- Docs: README, PUBLISHER_GUIDE, CONCEPTS, AI_AGENT_GUIDE
- Tests: test_search_engine_indexing.py (category, parse_list_env, allow/block, 200 free, 402)

Author: arun-fairfetch

README.md

@@ -305,6 +305,7 @@ You get back a `402 Payment Required` response — this is not an error, it's a
   },
   "available_tiers": {
     "summary":    { "price": "1000",  "compliance_level": "standard" },
+    "search_engine_indexing": { "price": "0", "compliance_level": "standard" },
     "rag":        { "price": "2000",  "compliance_level": "standard" },
     "research":   { "price": "3000",  "compliance_level": "elevated" },
     "training":   { "price": "5000",  "compliance_level": "strict" },
@@ -412,6 +413,7 @@ Not all content usage is equal. Fairfetch defines **usage categories** that cont
 | Category | Compliance | Price Multiplier | Use Case |
 |----------|-----------|-----------------|----------|
 | `summary` | Standard | 1x | Display a short summary or snippet |
+| `search_engine_indexing` | Standard | 0x (free) | Search engine crawling for indexing; free when publisher allows (see [config](#-configuration)) |
 | `rag` | Standard | 2x | Retrieval-Augmented Generation / search grounding |
 | `research` | Elevated | 3x | Academic or internal research use |
 | `training` | Strict | 5x | Model fine-tuning or pre-training |
@@ -689,6 +691,8 @@ fairfetch/
 | `FAIRFETCH_SIGNING_KEY` | *(generated)* | Ed25519 private key (b64) |
 | `FAIRFETCH_LICENSE_TYPE` | `publisher-terms` | Default license |
 | `FAIRFETCH_DEFAULT_USAGE_CATEGORY` | `summary` | Default usage tier for pricing |
+| `FAIRFETCH_SEARCH_ENGINES_ALLOWED` | *(built-in list)* | Comma-separated User-Agent substrings for search engines allowed **free** indexing (e.g. Googlebot, Bingbot, DuckDuckBot). Overrides default. |
+| `FAIRFETCH_SEARCH_ENGINES_BLOCKED` | *(empty)* | Comma-separated User-Agent substrings never given free indexing (takes precedence over allowed). |
 | `FAIRFETCH_ENABLE_GRANTS` | `true` | Issue Usage Grants |
 | `FAIRFETCH_PREFERRED_ACCESS` | `true` | Inject bot-steering headers |
 | `LITELLM_MODEL` | `gpt-4o-mini` | LLM for summarization |

api/dependencies.py

@@ -19,6 +19,20 @@
 from payments.mock_facilitator import MockFacilitator
 from payments.mock_license_facilitator import MockLicenseFacilitator, MockLicenseProvider
 
+# Default User-Agent substrings for search engines allowed free indexing (publisher can override)
+_DEFAULT_SEARCH_ENGINES_ALLOWED = (
+    "Googlebot",
+    "Bingbot",
+    "DuckDuckBot",
+    "Slurp",  # Yahoo
+    "Baiduspider",
+    "YandexBot",
+    "Sogou",
+    "Exabot",
+    "facebookexternalhit",
+    "ia_archiver",  # Alexa
+)
+
 
 class FairFetchConfig(BaseModel):
     """Centralized runtime configuration — loaded entirely from env vars.
@@ -47,6 +61,15 @@ class FairFetchConfig(BaseModel):
     enable_usage_grants: bool = Field(default=True)
     enable_preferred_access: bool = Field(default=True)
 
+    search_engines_allowed: list[str] = Field(
+        default_factory=lambda: list(_DEFAULT_SEARCH_ENGINES_ALLOWED),
+        description="User-Agent substrings for search engines allowed free indexing",
+    )
+    search_engines_blocked: list[str] = Field(
+        default_factory=list,
+        description="User-Agent substrings for search engines never given free indexing",
+    )
+
     @classmethod
     def from_env(cls) -> FairFetchConfig:
         return cls(
@@ -72,6 +95,10 @@ def from_env(cls) -> FairFetchConfig:
             enable_preferred_access=(
                 os.getenv("FAIRFETCH_PREFERRED_ACCESS", "true").lower() == "true"
             ),
+            search_engines_allowed=_parse_list_env(
+                "FAIRFETCH_SEARCH_ENGINES_ALLOWED", _DEFAULT_SEARCH_ENGINES_ALLOWED
+            ),
+            search_engines_blocked=_parse_list_env("FAIRFETCH_SEARCH_ENGINES_BLOCKED", ()),
         )
 
 
@@ -113,6 +140,17 @@ def _is_valid_price_string(s: str) -> bool:
     return isinstance(s, str) and len(s) <= 20 and s.isdigit()
 
 
+def _parse_list_env(
+    key: str,
+    default: tuple[str, ...] | list[str],
+) -> list[str]:
+    """Parse comma-separated env var into list of stripped strings; use default if unset."""
+    raw = os.getenv(key, "").strip()
+    if not raw:
+        return list(default)
+    return [s.strip() for s in raw.split(",") if s.strip()]
+
+
 def _parse_price_by_route(raw: str) -> dict[str, str]:
     """Parse FAIRFETCH_PRICE_BY_ROUTE JSON (path prefix -> price). Only numeric prices kept."""
     if not raw or not raw.strip():

api/main.py

@@ -91,6 +91,8 @@ def get_requirement(url: str) -> PaymentRequirement:
         get_requirement=get_requirement,
         license_provider=license_provider,
         wallet_ledger=wallet_ledger,
+        search_engines_allowed=config.search_engines_allowed,
+        search_engines_blocked=config.search_engines_blocked,
         paid_path_prefixes=["/content/"],
         exempt_paths=[
             "/health",

core/search_engine.py

@@ -0,0 +1,19 @@
+"""Search engine allow/block list for free indexing (usage category search_engine_indexing)."""
+
+from __future__ import annotations
+
+
+def is_allowed_search_engine(
+    user_agent: str,
+    allowed: list[str],
+    blocked: list[str],
+) -> bool:
+    """True if UA matches an allowed search engine and is not in the blocklist."""
+    ua = (user_agent or "").strip()
+    if not ua:
+        return False
+    ua_lower = ua.lower()
+    for sub in blocked:
+        if sub.strip().lower() in ua_lower:
+            return False
+    return any(sub.strip().lower() in ua_lower for sub in allowed)

docs/AI_AGENT_GUIDE.md

@@ -180,6 +180,7 @@ curl -s "http://localhost:8402/content/fetch?url=https://example.com"
   },
   "available_tiers": {
     "summary":    { "price": "1000",  "compliance_level": "standard" },
+    "search_engine_indexing": { "price": "0", "compliance_level": "standard" },
     "rag":        { "price": "2000",  "compliance_level": "standard" },
     "research":   { "price": "3000",  "compliance_level": "elevated" },
     "training":   { "price": "5000",  "compliance_level": "strict" },

docs/CONCEPTS.md

@@ -246,6 +246,7 @@ what it costs. Here's what each field means:
   },
   "available_tiers": {
     "summary":    { "price": "1000",  "compliance_level": "standard" },
+    "search_engine_indexing": { "price": "0", "compliance_level": "standard" },
     "rag":        { "price": "2000",  "compliance_level": "standard" },
     "research":   { "price": "3000",  "compliance_level": "elevated" },
     "training":   { "price": "5000",  "compliance_level": "strict" },

docs/PUBLISHER_GUIDE.md

@@ -163,6 +163,8 @@ LITELLM_MODEL=gpt-4o-mini
 | `FAIRFETCH_PUBLISHER_DOMAIN` | Your website’s domain (no `https://`) | `newstoday.com` |
 | `FAIRFETCH_CONTENT_PRICE` | Default price per request in smallest USDC unit (1000 ≈ $0.001). Used when no route rule matches. | `1000` |
 | `FAIRFETCH_PRICE_BY_ROUTE` | *(Optional)* JSON map of **path prefix → price** so different sections have different prices. Longest matching path wins. E.g. `{"": "1000", "/business": "2000", "/sports": "500"}` makes `/business` cost 2000, `/sports` 500, and everything else 1000. | (omit for one price site-wide) |
+| `FAIRFETCH_SEARCH_ENGINES_ALLOWED` | *(Optional)* Comma-separated User-Agent substrings for search engines allowed **free** indexing (e.g. `Googlebot,Bingbot,DuckDuckBot`). Default includes Google, Bing, DuckDuckGo, Yahoo, Baidu, Yandex, Sogou, Exabot, and others. | (omit to use default) |
+| `FAIRFETCH_SEARCH_ENGINES_BLOCKED` | *(Optional)* Comma-separated User-Agent substrings never given free indexing (overrides allowed list). | (omit for none) |
 | `FAIRFETCH_LICENSE_TYPE` | Legal terms you offer: `publisher-terms`, `commercial`, or `research-only` | `publisher-terms` |
 | `FAIRFETCH_SIGNING_KEY` | Leave empty at first; we’ll generate a key next. | (empty) |
 | `LITELLM_MODEL` | Model used to generate summaries (needs an API key in production) | `gpt-4o-mini` |
@@ -182,6 +184,9 @@ Here, `/business` (and `/business/...`) is 2000, `/sports` is 500, and all other
 
 **Behavior and limits:** Prices must be numeric (digits only); non-numeric values are ignored. The content URL path is normalized (percent-encoding decoded, `.` and `..` segments collapsed) so route matching cannot be bypassed. At most 256 route entries are used; extra entries are ignored.
 
+**Search engine indexing (free for allowed crawlers)**  
+The usage category `search_engine_indexing` lets search engines (Google, Bing, DuckDuckGo, etc.) index your site for **free** when you allow them. Set `FAIRFETCH_SEARCH_ENGINES_ALLOWED` to a comma-separated list of User-Agent substrings (e.g. `Googlebot,Bingbot,DuckDuckBot`). The default allowlist includes Googlebot, Bingbot, DuckDuckBot, Slurp, Baiduspider, YandexBot, Sogou, Exabot, and a few others. Set `FAIRFETCH_SEARCH_ENGINES_BLOCKED` to block specific crawlers from free access (takes precedence over the allowlist). Crawlers not on the allowlist that request `usage=search_engine_indexing` pay the base price (1x).
+
 ### 3.2 Generate a signing key (recommended for production)
 
 This key lets AI agents (and you) verify that content really came from your server.

interfaces/facilitator.py

@@ -44,7 +44,7 @@ class PaymentRequirement(BaseModel):
     description: str = Field(default="Content access fee")
     usage_category: str = Field(
         default=UsageCategory.SUMMARY,
-        description="Intended usage: summary, rag, research, training, commercial",
+        description="Intended usage: summary, search_engine_indexing, rag, research, training, commercial",  # noqa: E501
     )
     extra: dict[str, str] = Field(default_factory=dict)
 

interfaces/license_provider.py

@@ -30,13 +30,15 @@ class UsageCategory(StrEnum):
 
     Categories are ordered by escalating compliance requirements and pricing:
       - SUMMARY:   Display a short summary or snippet (lowest tier)
+      - SEARCH_ENGINE_INDEXING: Search engine crawling for indexing (free when allowed by publisher)
       - RAG:       Retrieval-Augmented Generation / search grounding
       - RESEARCH:  Academic or internal research use
       - TRAINING:  Model fine-tuning or pre-training (highest tier)
       - COMMERCIAL: Redistribution or commercial derivative works
     """
 
     SUMMARY = "summary"
+    SEARCH_ENGINE_INDEXING = "search_engine_indexing"
     RAG = "rag"
     RESEARCH = "research"
     TRAINING = "training"
@@ -58,6 +60,12 @@ class ComplianceLevel(StrEnum):
         "requires_audit_trail": False,
         "description": "Short summary or snippet display",
     },
+    UsageCategory.SEARCH_ENGINE_INDEXING: {
+        "compliance_level": ComplianceLevel.STANDARD,
+        "price_multiplier": 0,
+        "requires_audit_trail": False,
+        "description": "Search engine crawling for indexing (free when publisher allows)",
+    },
     UsageCategory.RAG: {
         "compliance_level": ComplianceLevel.STANDARD,
         "price_multiplier": 2,
@@ -110,7 +118,7 @@ class UsageGrant(BaseModel):
     license_type: str = Field(default="publisher-terms")
     usage_category: str = Field(
         default=UsageCategory.SUMMARY,
-        description="Permitted use: summary, rag, research, training, commercial",
+        description="Permitted use: summary, search_engine_indexing, rag, research, training, commercial",  # noqa: E501
     )
     granted_to: str = Field(default="", description="Payer wallet or agent identifier")
     granted_at: str = Field(default_factory=lambda: datetime.now(UTC).isoformat())

payments/x402.py

@@ -23,6 +23,7 @@
 from fastapi.responses import JSONResponse
 from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
 
+from core.search_engine import is_allowed_search_engine
 from interfaces.facilitator import BaseFacilitator, PaymentRequirement
 from interfaces.license_provider import BaseLicenseProvider, UsageCategory
 from payments.wallet_ledger import WalletLedger
@@ -52,6 +53,8 @@ def __init__(
         get_requirement: Callable[[str], PaymentRequirement],
         license_provider: BaseLicenseProvider | None = None,
         wallet_ledger: WalletLedger | None = None,
+        search_engines_allowed: list[str] | None = None,
+        search_engines_blocked: list[str] | None = None,
         paid_path_prefixes: list[str] | None = None,
         exempt_paths: list[str] | None = None,
     ) -> None:
@@ -60,6 +63,8 @@ def __init__(
         self._get_requirement = get_requirement
         self._license_provider = license_provider
         self._wallet_ledger = wallet_ledger
+        self._search_engines_allowed = search_engines_allowed or []
+        self._search_engines_blocked = search_engines_blocked or []
         self._paid_prefixes = paid_path_prefixes or ["/content/"]
         self._exempt = set(exempt_paths or ["/health", "/openapi.json", "/docs", "/redoc"])
 
@@ -80,6 +85,30 @@ def _resolve_usage_category(
                 pass
         return default_requirement.usage_category
 
+    def _402_body_with_price(
+        self,
+        req_for_category: PaymentRequirement,
+        usage_cat: str,
+        effective_price: int,
+        **extra: object,
+    ) -> dict[str, object]:
+        """Build 402 body; override displayed price when search_engine_indexing and not free."""
+        body = {**req_for_category.to_402_body(), **extra}
+        if usage_cat == UsageCategory.SEARCH_ENGINE_INDEXING.value and effective_price != 0:
+            accepts = body.get("accepts")
+            if isinstance(accepts, dict):
+                body["accepts"] = {**accepts, "price": str(effective_price)}
+            if "available_tiers" in body and isinstance(body["available_tiers"], dict):
+                tiers = dict(body["available_tiers"])
+                if "search_engine_indexing" in tiers and isinstance(
+                    tiers["search_engine_indexing"], dict
+                ):
+                    tier = dict(tiers["search_engine_indexing"])
+                    tier["price"] = str(effective_price)
+                    tiers["search_engine_indexing"] = tier
+                    body["available_tiers"] = tiers
+        return body
+
     async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -> Response:
         if not self._is_paid_route(request.url.path):
             return await call_next(request)
@@ -91,6 +120,30 @@ async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -
         request.state.payment_requirement = req_for_category
         effective_price = int(req_for_category.effective_price())
 
+        # --- Search engine indexing: free for allowed crawlers, 1x base for others ---
+        if usage_cat == UsageCategory.SEARCH_ENGINE_INDEXING.value:
+            user_agent = request.headers.get("user-agent", "") or ""
+            if is_allowed_search_engine(
+                user_agent,
+                self._search_engines_allowed,
+                self._search_engines_blocked,
+            ):
+                effective_price = 0
+            else:
+                effective_price = int(requirement.price)
+
+        # --- Free pass: no payment required (e.g. allowed search engine) ---
+        if effective_price == 0:
+            payer = request.headers.get("user-agent", "search_engine") or "search_engine"
+            request.state.payment_result = None
+            request.state.payment_payer = payer
+            request.state.usage_category = usage_cat
+            response = await call_next(request)
+            response.headers["X-FairFetch-Payment-Method"] = "free"
+            response.headers[RECEIPT_HEADER] = "free"
+            await self._issue_grant(request, response, usage_cat, payer)
+            return response
+
         wallet_token = request.headers.get(WALLET_TOKEN_HEADER)
         payment_header = request.headers.get(PAYMENT_HEADER)
 
@@ -140,18 +193,20 @@ async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -
             )
             return JSONResponse(
                 status_code=402,
-                content={
-                    **req_for_category.to_402_body(),
-                    "wallet_error": "insufficient_balance",
-                    "wallet_balance": balance,
-                    "amount_required": effective_price,
-                    "shortfall": effective_price - balance,
-                },
+                content=self._402_body_with_price(
+                    req_for_category,
+                    usage_cat,
+                    effective_price,
+                    wallet_error="insufficient_balance",
+                    wallet_balance=balance,
+                    amount_required=effective_price,
+                    shortfall=effective_price - balance,
+                ),
             )
 
         # --- Standard path: x402 one-time payment ---
         if not payment_header:
-            body = req_for_category.to_402_body()
+            body = self._402_body_with_price(req_for_category, usage_cat, effective_price)
             body["hint"] = (
                 "For faster access without 402 round-trips, use a pre-funded wallet. "
                 "Send X-WALLET-TOKEN header instead of X-PAYMENT. "
@@ -164,16 +219,24 @@ async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -
                 headers={"X-Payment-Required": "true"},
             )
 
-        result = await self._facilitator.settle(payment_header, req_for_category)
+        # When search_engine_indexing but not free, settle at base price (1x)
+        req_for_settlement = req_for_category
+        if usage_cat == UsageCategory.SEARCH_ENGINE_INDEXING.value and effective_price != 0:
+            req_for_settlement = requirement.model_copy(
+                update={"usage_category": UsageCategory.SUMMARY.value}
+            )
+        result = await self._facilitator.settle(payment_header, req_for_settlement)
 
         if not result.valid:
             logger.warning("Payment rejected for %s: %s", request.url.path, result.error)
             return JSONResponse(
                 status_code=402,
-                content={
-                    **req_for_category.to_402_body(),
-                    "verification_error": result.error,
-                },
+                content=self._402_body_with_price(
+                    req_for_category,
+                    usage_cat,
+                    effective_price,
+                    verification_error=result.error,
+                ),
             )
 
         logger.info(