diff --git a/.env.example b/.env.example index c23dc84..b8d299c 100644 --- a/.env.example +++ b/.env.example @@ -5,8 +5,14 @@ OPENAI_API_KEY=your-openrouter-or-openai-key-here # Default model for every agent (override per role at the bottom of this file). OPENAI_MODEL=google/gemini-3-flash-preview -# Web search via serper.dev. Required: the agents research every section through it. +# Web search and page fetch run through a deterministic round-robin with failover across whichever +# providers have a key set. Serper is the baseline (search + fetch); Exa and Tavily add search + +# fetch, Firecrawl and Diffbot add fetch only. Set at least one search provider so research works. SERPER_API_KEY=your-serper-api-key-here +EXA_API_KEY= +TAVILY_API_KEY= +FIRECRAWL_API_KEY= +DIFFBOT_API_KEY= # Email delivery via Resend. Only needed when actually sending (python src/app.py run --send). RESEND_API_KEY= diff --git a/CLAUDE.md b/CLAUDE.md index c9ff607..7742051 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ Guidance for working in this repository. ## What this is -`agentic-mediapulse` is an agentic newsletter generator. Given a subject (a stock ticker, company name, or industry theme), a newsroom of focused agents researches, writes, and edits a locale-aware briefing across five editorial sections, with every claim traced to a real source. It is built on the Microsoft Agent Framework (`agent-framework`) plus Serper for web search. +`agentic-mediapulse` is an agentic newsletter generator. Given a subject (a stock ticker, company name, or industry theme), a newsroom of focused agents researches, writes, and edits a locale-aware briefing across five editorial sections, with every claim traced to a real source. It is built on the Microsoft Agent Framework (`agent-framework`) plus a round-robin web toolbelt (Serper, Exa, Tavily, Firecrawl, Diffbot) for search and page fetch. ## Layout @@ -13,7 +13,7 @@ All application code lives under `src/`. Packages keep their top-level names (`a - `src/api.py` — FastAPI service. `POST /run` (full campaign) and `POST /test` (one user). Both run in the background, return `202`, and default to dry-run. Auth via the `X-API-Key` header matched against `SECRET_KEY`. - `src/app.py` — local CLI mirroring the API (`run`, `test`), for testing without HTTP. - `src/agents/orchestrator.py` — the pipeline: analyst → 5 parallel beat desks (researcher → writer → editor) → managing-editor gap roundtable → masthead → reviewer → deterministic clean/assemble/dedupe. Most non-agent logic (citation gating, URL/article validation, dedupe, subject-name canonicalization, prose humanizing) lives here. -- `src/agents/` — one module per agent (`analyst`, `researcher`, `writer`, `editor`, `managing_editor`, `reviewer`), plus `beats.py` (beat desks), `campaign.py` (top-level run over subscriptions), `sections.py` (the five editorial beats), `providers/` (subject-memory and ticker-profile context providers), and `tools/` (Serper search, web fetch). +- `src/agents/` — one module per agent (`analyst`, `researcher`, `writer`, `editor`, `managing_editor`, `reviewer`), plus `beats.py` (beat desks), `campaign.py` (top-level run over subscriptions, which skips and marks `failed` any newsletter that comes out with zero sections rather than sending it), `sections.py` (the five editorial beats), `providers/` (subject-memory and ticker-profile context providers), and `tools/` (the `web_search` and `web_fetch` tools over a round-robin + failover provider package in `tools/providers/`: Serper, Exa, Tavily, Firecrawl, Diffbot, selected deterministically and hidden from the LLM). - `src/agents/runtime/` — agent plumbing shared by every agent: `chat_client.py` (per-role chat client plus the `SKILLS` provider), `make_agent.py` (the factory that wires generic activity tracking into every agent), `guardrails.py` (guardrail/citation middleware), and `tracking.py` (the generic `ActivityTracker`/`ToolTracker` middleware, `newsletter_scope`, and run context vars). New agents are built via `make_agent(...)` so tracking is automatic. - `src/agents/skills/` — `SKILL.md` files that control agent behavior (`subject-profile`, `section-research`, `newsletter-format`). Prefer editing these over code when changing how agents research or write. - `src/db/` — all database access. `mediapulse.py` reads subscriptions and ticker profiles from the upstream MediaPulse Postgres (`MEDIAPULSE_DATABASE_URL`, read-only, raw psycopg). The app's own Postgres (`DATABASE_URL`, SQLModel) uses `engine.py` for the shared engine, with each table's model alongside its operations: `newsletters.py` (archives each newsletter as markdown plus JSONB metadata, with a `pending`/`complete`/`failed` lifecycle via `create_newsletter`/`finalize_newsletter`), `memory.py` (subject-brief agent memory), and `agent_activity.py` (one row per agent run and tool call, tied to its `newsletter_id`, recording status, duration, model, and token usage). The schema is owned by Alembic migrations, not `create_all` (see Migrations). @@ -101,7 +101,8 @@ The suite covers the deterministic logic, not the LLM agents: orchestrator text/ ## External services -- `SERPER_API_KEY` — web search. +- `SERPER_API_KEY` — web search and page fetch (the baseline provider). +- `EXA_API_KEY`, `TAVILY_API_KEY`, `FIRECRAWL_API_KEY`, `DIFFBOT_API_KEY` — optional extra providers; when their key is set they join a deterministic round-robin with failover (Exa and Tavily also search, Firecrawl and Diffbot fetch only). `web_search`/`web_fetch` keep an identical signature, so the LLM never sees which provider served a call. - `MEDIAPULSE_DATABASE_URL` — read-only Postgres for subscriptions and ticker data; schema is defined in the upstream [MediaPulse](https://github.com/hyperjumptech/mediapulse) repo. - `DATABASE_URL` — the app's own read-write Postgres for archived newsletters and agent memory. Tables are auto-created on first use, separate from `MEDIAPULSE_DATABASE_URL`. - `SECRET_KEY` — API auth. diff --git a/src/agents/analyst.py b/src/agents/analyst.py index ccdcb42..e21717f 100644 --- a/src/agents/analyst.py +++ b/src/agents/analyst.py @@ -3,7 +3,7 @@ from agents.runtime.chat_client import SKILLS, chat_client from agents.runtime.guardrails import SubjectGuardrail from agents.runtime.make_agent import make_agent -from agents.tools import search, web_fetch +from agents.tools import web_fetch, web_search analyst = make_agent( name="analyst", @@ -13,7 +13,7 @@ "You are a research analyst. The subject may be a ticker, a company, or an industry/theme. " "Use the subject-profile skill to turn the subject into a brief." ), - tools=[search, web_fetch], + tools=[web_search, web_fetch], context_providers=[SKILLS, TickerProfileProvider(), SubjectMemoryProvider()], middleware=[SubjectGuardrail()], ) diff --git a/src/agents/campaign.py b/src/agents/campaign.py index 59c0cc1..7d46355 100644 --- a/src/agents/campaign.py +++ b/src/agents/campaign.py @@ -6,7 +6,7 @@ from db.mediapulse import fetch_subscriptions from db.newsletters import create_newsletter, finalize_newsletter from emails.mailer import send_email -from emails.templates.newsletter import newsletter_sources, render_newsletter_email +from emails.templates.newsletter import has_sections, newsletter_sources, render_newsletter_email CONCURRENCY = 3 SEND_INTERVAL = 1.0 # seconds between emails @@ -32,6 +32,15 @@ async def run_campaign(*, subscriptions: list[dict] | None = None, send: bool = async def deliver(ticker: str, recipients: list[dict], markdown: str, newsletter_id: int | None) -> None: nonlocal last_send + + if not has_sections(markdown): + finalize_newsletter( + newsletter_id, content=markdown, metadata={"ticker": ticker, "sources": []}, status="failed" + ) + log(f"skipped {ticker}: newsletter has no sections, not sending") + + return + email = render_newsletter_email(markdown, ticker=ticker) finalize_newsletter( newsletter_id, content=markdown, metadata={"ticker": ticker, "sources": newsletter_sources(markdown)} diff --git a/src/agents/researcher.py b/src/agents/researcher.py index ec72d10..edcad16 100644 --- a/src/agents/researcher.py +++ b/src/agents/researcher.py @@ -4,7 +4,7 @@ from agents.runtime.guardrails import RecordSources, SourceRegistry from agents.runtime.make_agent import make_agent from agents.sections import Section -from agents.tools import search +from agents.tools import web_search def make_researcher(section: Section, registry: SourceRegistry) -> Agent: @@ -20,7 +20,7 @@ def make_researcher(section: Section, registry: SourceRegistry) -> Agent: "matters, written in English. The writer needs at least 2 and up to 5 strong, distinct stories. " "You may query in the subject's local language. Never invent a URL. Output only the list, no preamble." ), - tools=[search], + tools=[web_search], context_providers=[SKILLS], middleware=[RecordSources(registry)], ) diff --git a/src/agents/skills/section-research/SKILL.md b/src/agents/skills/section-research/SKILL.md index 4f10fe5..16f8c8d 100644 --- a/src/agents/skills/section-research/SKILL.md +++ b/src/agents/skills/section-research/SKILL.md @@ -15,7 +15,7 @@ Research one newsletter beat, find the strongest recent articles, and output the ## Search instructions 1. Build simple queries: the subject's short common name on its own first, then add at most one topic word to narrow if needed. Never use long multi-keyword queries, `site:` operators, or mixed-language queries — they return nothing. -2. Call `search` with kind="news" first; use kind="web" only for background. Always pass the `gl` and `hl` codes from the brief's Locale line. Searches default to the past week; widen to recency="month" if results are thin, narrow to recency="day" for breaking news. +2. Call `web_search` with kind="news" first; use kind="web" only for background. Always pass the `gl` and `hl` codes from the brief's Locale line. Searches default to the past week; widen to recency="month" if results are thin, narrow to recency="day" for breaking news. 3. Keep the subject the lead: most items should be about it. You may cover named competitors and industry context secondarily, but only from the subject's home market. 4. Select the strongest, most relevant recent items — at least 2, at most 5. Prefer substantive developments: products, strategy, expansion, deals, regulation, technology, operations, leadership. Skip stock-index roundups (IHSG levels), "top gainers/losers", analyst price targets, and technical analysis unless a price move is itself the news. Never invent an article or URL. diff --git a/src/agents/skills/subject-profile/SKILL.md b/src/agents/skills/subject-profile/SKILL.md index 5c47b0a..67c4a76 100644 --- a/src/agents/skills/subject-profile/SKILL.md +++ b/src/agents/skills/subject-profile/SKILL.md @@ -13,7 +13,7 @@ The subject string the user provided. If verified exchange listing details are a ## Instructions -1. If the subject is not obvious and no verified listing details are provided, call `search` (kind="web" or kind="news", recency="") to confirm what it is. +1. If the subject is not obvious and no verified listing details are provided, call `web_search` (kind="web" or kind="news", recency="") to confirm what it is. 2. Output a compact brief using exactly the labelled lines in the Output section below — nothing else. ## Output diff --git a/src/agents/tools/__init__.py b/src/agents/tools/__init__.py index 009c69d..ffc078f 100644 --- a/src/agents/tools/__init__.py +++ b/src/agents/tools/__init__.py @@ -1,4 +1,4 @@ -from agents.tools.search import search from agents.tools.web_fetch import web_fetch +from agents.tools.web_search import web_search -__all__ = ["search", "web_fetch"] +__all__ = ["web_fetch", "web_search"] diff --git a/src/agents/tools/providers/__init__.py b/src/agents/tools/providers/__init__.py new file mode 100644 index 0000000..1fb987c --- /dev/null +++ b/src/agents/tools/providers/__init__.py @@ -0,0 +1,14 @@ +from agents.tools.providers import diffbot, exa, firecrawl, serper, tavily +from agents.tools.providers.dispatch import AllProvidersFailed, Provider, dispatch, reset_cursor + +__all__ = [ + "AllProvidersFailed", + "Provider", + "diffbot", + "dispatch", + "exa", + "firecrawl", + "reset_cursor", + "serper", + "tavily", +] diff --git a/src/agents/tools/providers/diffbot.py b/src/agents/tools/providers/diffbot.py new file mode 100644 index 0000000..898408c --- /dev/null +++ b/src/agents/tools/providers/diffbot.py @@ -0,0 +1,16 @@ +import os + +import httpx + + +def fetch(url: str) -> str: + """Adapter: Diffbot article extraction for `url`, raising on HTTP error.""" + response = httpx.get( + "https://api.diffbot.com/v3/article", + params={"token": os.environ["DIFFBOT_API_KEY"], "url": url}, + timeout=30.0, + ) + response.raise_for_status() + objects = response.json().get("objects", []) + + return objects[0].get("text", "") if objects else "" diff --git a/src/agents/tools/providers/dispatch.py b/src/agents/tools/providers/dispatch.py new file mode 100644 index 0000000..926c66f --- /dev/null +++ b/src/agents/tools/providers/dispatch.py @@ -0,0 +1,72 @@ +import os +from collections.abc import Callable + + +class AllProvidersFailed(Exception): + """Raised when every configured provider for a capability errored on one call.""" + + def __init__(self, capability: str, failures: list[tuple[str, str]]) -> None: + self.capability = capability + self.failures = failures + detail = "; ".join(f"{name}={reason}" for name, reason in failures) or "no providers configured" + + super().__init__(f"all {capability} providers failed: {detail}") + + +class Provider: + """A named web provider, active only when its API-key env var is set.""" + + def __init__(self, name: str, env: str, fn: Callable) -> None: + self.name = name + self.env = env + self.fn = fn + + def available(self) -> bool: + return bool(os.getenv(self.env)) + + +# Round-robin cursor per capability ("search"/"fetch"). Lock-free: a benign race only skews load. +_cursor: dict[str, int] = {} + + +def reset_cursor() -> None: + """Clear the round-robin cursor so tests start from a known position.""" + _cursor.clear() + + +def _rotate(capability: str, active: list[Provider]): + start = _cursor.get(capability, 0) + _cursor[capability] = start + 1 + + for offset in range(len(active)): + yield active[(start + offset) % len(active)] + + +def dispatch(capability: str, providers: list[Provider], call: Callable, accept: Callable) -> object: + """Round-robin across the available providers, failing over on error or unusable result. + + Returns the first result accepted by `accept`. If providers respond but none is usable (for + example all empty), returns the last such result. If every configured provider errors, raises + AllProvidersFailed naming each one. + """ + active = [provider for provider in providers if provider.available()] + errors: list[tuple[str, str]] = [] + empty_result, saw_empty = None, False + + for provider in _rotate(capability, active): + try: + result = call(provider.fn) + except Exception as error: + errors.append((provider.name, f"{type(error).__name__}: {error}")) + + continue + + if accept(result): + return result + + empty_result, saw_empty = result, True + + if saw_empty: + return empty_result + + raise AllProvidersFailed(capability, errors) diff --git a/src/agents/tools/providers/exa.py b/src/agents/tools/providers/exa.py new file mode 100644 index 0000000..0088334 --- /dev/null +++ b/src/agents/tools/providers/exa.py @@ -0,0 +1,61 @@ +import os +from datetime import datetime, timedelta, timezone + +import httpx + +_RECENCY_DAYS = {"day": 1, "week": 7, "month": 30} + + +def _start_date(recency: str) -> str | None: + days = _RECENCY_DAYS.get(recency) + + if not days: + return None + + return (datetime.now(timezone.utc) - timedelta(days=days)).isoformat() + + +def search(query: str, kind: str, gl: str, hl: str, recency: str) -> list[dict]: + """Adapter: Exa neural/keyword search, mapped to the Serper result shape.""" + body = {"query": query, "numResults": 10, "type": "auto", "contents": {"text": {"maxCharacters": 500}}} + + if kind == "news": + body["category"] = "news" + + start_date = _start_date(recency) + + if start_date: + body["startPublishedDate"] = start_date + + response = httpx.post( + "https://api.exa.ai/search", + headers={"x-api-key": os.environ["EXA_API_KEY"]}, + json=body, + timeout=30.0, + ) + response.raise_for_status() + results = response.json().get("results", []) + + return [ + { + "title": item.get("title", ""), + "link": item.get("url", ""), + "snippet": " ".join((item.get("text") or "").split())[:300], + "date": item.get("publishedDate", "") or "", + } + for item in results + ] + + +def fetch(url: str) -> str: + """Adapter: Exa page contents for `url`, raising on HTTP error.""" + response = httpx.post( + "https://api.exa.ai/contents", + headers={"x-api-key": os.environ["EXA_API_KEY"]}, + json={"urls": [url], "text": True}, + timeout=30.0, + ) + response.raise_for_status() + results = response.json().get("results", []) + + return results[0].get("text", "") if results else "" diff --git a/src/agents/tools/providers/firecrawl.py b/src/agents/tools/providers/firecrawl.py new file mode 100644 index 0000000..d5f992a --- /dev/null +++ b/src/agents/tools/providers/firecrawl.py @@ -0,0 +1,17 @@ +import os + +import httpx + + +def fetch(url: str) -> str: + """Adapter: Firecrawl scrape for `url` as markdown, raising on HTTP error.""" + response = httpx.post( + "https://api.firecrawl.dev/v1/scrape", + headers={"Authorization": f"Bearer {os.environ['FIRECRAWL_API_KEY']}"}, + json={"url": url, "formats": ["markdown"]}, + timeout=30.0, + ) + response.raise_for_status() + data = response.json().get("data", {}) + + return data.get("markdown", "") or "" diff --git a/src/agents/tools/providers/serper.py b/src/agents/tools/providers/serper.py new file mode 100644 index 0000000..8833f29 --- /dev/null +++ b/src/agents/tools/providers/serper.py @@ -0,0 +1,49 @@ +import os + +import httpx + +_RECENCY = {"day": "qdr:d", "week": "qdr:w", "month": "qdr:m"} + + +def serper(endpoint: str, query: str, gl: str, hl: str, tbs: str = "") -> list[dict]: + body = {"q": query} + + if gl: + body["gl"] = gl + + if hl: + body["hl"] = hl + + if tbs: + body["tbs"] = tbs + + response = httpx.post( + f"https://google.serper.dev/{endpoint}", + headers={"X-API-KEY": os.environ["SERPER_API_KEY"]}, + json=body, + timeout=30.0, + ) + response.raise_for_status() + + return response.json().get("news" if endpoint == "news" else "organic", []) + + +def search(query: str, kind: str, gl: str, hl: str, recency: str) -> list[dict]: + """Adapter: map the unified search call onto Serper's news/web endpoints.""" + endpoint = "news" if kind == "news" else "search" + + return serper(endpoint, query, gl, hl, _RECENCY.get(recency, "")) + + +def fetch(url: str) -> str: + """Adapter: scrape readable text for `url` via Serper, raising on HTTP error.""" + response = httpx.post( + "https://scrape.serper.dev", + headers={"X-API-KEY": os.environ["SERPER_API_KEY"]}, + json={"url": url}, + timeout=30.0, + ) + response.raise_for_status() + data = response.json() + + return data.get("text") or data.get("markdown") or "" diff --git a/src/agents/tools/providers/tavily.py b/src/agents/tools/providers/tavily.py new file mode 100644 index 0000000..ca27969 --- /dev/null +++ b/src/agents/tools/providers/tavily.py @@ -0,0 +1,44 @@ +import os + +import httpx + + +def search(query: str, kind: str, gl: str, hl: str, recency: str) -> list[dict]: + """Adapter: Tavily search, mapped to the Serper result shape.""" + body = {"query": query, "max_results": 10, "topic": "news" if kind == "news" else "general"} + + if recency in ("day", "week", "month"): + body["time_range"] = recency + + response = httpx.post( + "https://api.tavily.com/search", + headers={"Authorization": f"Bearer {os.environ['TAVILY_API_KEY']}"}, + json=body, + timeout=30.0, + ) + response.raise_for_status() + results = response.json().get("results", []) + + return [ + { + "title": item.get("title", ""), + "link": item.get("url", ""), + "snippet": " ".join((item.get("content") or "").split())[:300], + "date": item.get("published_date", "") or "", + } + for item in results + ] + + +def fetch(url: str) -> str: + """Adapter: Tavily extract for `url`, raising on HTTP error.""" + response = httpx.post( + "https://api.tavily.com/extract", + headers={"Authorization": f"Bearer {os.environ['TAVILY_API_KEY']}"}, + json={"urls": [url]}, + timeout=30.0, + ) + response.raise_for_status() + results = response.json().get("results", []) + + return results[0].get("raw_content", "") if results else "" diff --git a/src/agents/tools/search.py b/src/agents/tools/search.py deleted file mode 100644 index 196861f..0000000 --- a/src/agents/tools/search.py +++ /dev/null @@ -1,15 +0,0 @@ -from agents.tools.serper import format_results, serper - -_RECENCY = {"day": "qdr:d", "week": "qdr:w", "month": "qdr:m"} - - -def search(query: str, kind: str = "news", gl: str = "", hl: str = "", recency: str = "week") -> str: - """Search for the query and return titles, URLs, and snippets. - - kind is 'news' (default, recent articles) or 'web' (general results). gl/hl are the - Serper country/language codes that localize results to the subject's home market. - recency limits results by age: 'day' (last 24h), 'week' (default), 'month', or '' for any time. - """ - endpoint = "news" if kind == "news" else "search" - - return format_results(serper(endpoint, query, gl, hl, _RECENCY.get(recency, ""))) diff --git a/src/agents/tools/serper.py b/src/agents/tools/serper.py deleted file mode 100644 index 200569c..0000000 --- a/src/agents/tools/serper.py +++ /dev/null @@ -1,42 +0,0 @@ -import os - -import httpx - - -def serper(endpoint: str, query: str, gl: str, hl: str, tbs: str = "") -> list[dict]: - body = {"q": query} - - if gl: - body["gl"] = gl - - if hl: - body["hl"] = hl - - if tbs: - body["tbs"] = tbs - - response = httpx.post( - f"https://google.serper.dev/{endpoint}", - headers={"X-API-KEY": os.environ["SERPER_API_KEY"]}, - json=body, - timeout=30.0, - ) - response.raise_for_status() - - return response.json().get("news" if endpoint == "news" else "organic", []) - - -def format_results(results: list[dict]) -> str: - if not results: - return "No results found." - - blocks = [] - - for item in results[:10]: - title = item.get("title", "") - link = item.get("link", "") - snippet = item.get("snippet", "") - date = item.get("date", "") - blocks.append(f"{title}\n{link}\n{date} {snippet}".strip()) - - return "\n\n".join(blocks) diff --git a/src/agents/tools/web_fetch.py b/src/agents/tools/web_fetch.py index 2aca400..a792ca3 100644 --- a/src/agents/tools/web_fetch.py +++ b/src/agents/tools/web_fetch.py @@ -1,6 +1,16 @@ -import os +from agents.tools.providers import AllProvidersFailed, Provider, diffbot, dispatch, exa, firecrawl, serper, tavily -import httpx +FETCH_PROVIDERS = [ + Provider("serper", "SERPER_API_KEY", serper.fetch), + Provider("exa", "EXA_API_KEY", exa.fetch), + Provider("tavily", "TAVILY_API_KEY", tavily.fetch), + Provider("firecrawl", "FIRECRAWL_API_KEY", firecrawl.fetch), + Provider("diffbot", "DIFFBOT_API_KEY", diffbot.fetch), +] + + +def _nonempty(value: str) -> bool: + return bool(value and value.strip()) def web_fetch(url: str) -> str: @@ -9,17 +19,8 @@ def web_fetch(url: str) -> str: return "Not a fetchable URL — pass a full http(s) article link." try: - response = httpx.post( - "https://scrape.serper.dev", - headers={"X-API-KEY": os.environ["SERPER_API_KEY"]}, - json={"url": url}, - timeout=30.0, - ) - response.raise_for_status() - except httpx.HTTPError: + text = dispatch("fetch", FETCH_PROVIDERS, lambda fn: fn(url), accept=_nonempty) + except AllProvidersFailed: return f"Could not fetch this URL; summarize from the candidate's title and snippet instead: {url}" - data = response.json() - text = data.get("text") or data.get("markdown") or "" - return text[:6000] if text else "No content." diff --git a/src/agents/tools/web_search.py b/src/agents/tools/web_search.py new file mode 100644 index 0000000..a542ef5 --- /dev/null +++ b/src/agents/tools/web_search.py @@ -0,0 +1,35 @@ +from agents.tools.providers import Provider, dispatch, exa, serper, tavily + +SEARCH_PROVIDERS = [ + Provider("serper", "SERPER_API_KEY", serper.search), + Provider("exa", "EXA_API_KEY", exa.search), + Provider("tavily", "TAVILY_API_KEY", tavily.search), +] + + +def format_results(results: list[dict]) -> str: + if not results: + return "No results found." + + blocks = [] + + for item in results[:10]: + title = item.get("title", "") + link = item.get("link", "") + snippet = item.get("snippet", "") + date = item.get("date", "") + blocks.append(f"{title}\n{link}\n{date} {snippet}".strip()) + + return "\n\n".join(blocks) + + +def web_search(query: str, kind: str = "news", gl: str = "", hl: str = "", recency: str = "week") -> str: + """Search for the query and return titles, URLs, and snippets. + + kind is 'news' (default, recent articles) or 'web' (general results). gl/hl are the + country/language codes that localize results to the subject's home market. + recency limits results by age: 'day' (last 24h), 'week' (default), 'month', or '' for any time. + """ + results = dispatch("search", SEARCH_PROVIDERS, lambda fn: fn(query, kind, gl, hl, recency), accept=bool) + + return format_results(results) diff --git a/src/emails/templates/newsletter.py b/src/emails/templates/newsletter.py index 474125c..38c6840 100644 --- a/src/emails/templates/newsletter.py +++ b/src/emails/templates/newsletter.py @@ -83,6 +83,13 @@ def newsletter_sources(markdown: str) -> list[str]: return sources +def has_sections(markdown: str) -> bool: + """Return True if the newsletter has at least one editorial item (a section with a Read link).""" + _title, _summary, sections = _parse(markdown) + + return any(items for _name, items in sections) + + @lru_cache(maxsize=1) def _template() -> str: """Load the tokenized HTML template generated by email-playground.""" diff --git a/tests/test_campaign.py b/tests/test_campaign.py index ee449e5..ed80841 100644 --- a/tests/test_campaign.py +++ b/tests/test_campaign.py @@ -1,12 +1,18 @@ import agents.campaign as campaign -def _patch_pipeline(monkeypatch, *, fail_for=()): +def _patch_pipeline(monkeypatch, *, fail_for=(), empty_for=()): async def fake_run_newsletter(ticker): if ticker in fail_for: raise RuntimeError(f"boom {ticker}") - return f"# {ticker} markdown" + if ticker in empty_for: + return f"# {ticker} Pulse\n\nStandfirst only, no sections.\n" + + return ( + f"# {ticker} Pulse\n\nStandfirst.\n\n## Quick Hits\n" + f"Something happened.\n[Read: Source](https://example.com/{ticker})\n---\n" + ) sent = [] diff --git a/tests/test_email_template.py b/tests/test_email_template.py index 8eabc07..d41c975 100644 --- a/tests/test_email_template.py +++ b/tests/test_email_template.py @@ -78,3 +78,11 @@ def test_newsletter_sources_dedupes_and_preserves_order(): def test_newsletter_sources_empty_without_links(): assert email_template.newsletter_sources("# Title\n\nNo links here.") == [] + + +def test_has_sections_true_for_cited_newsletter(): + assert email_template.has_sections(_MARKDOWN) is True + + +def test_has_sections_false_without_items(): + assert email_template.has_sections("# ACME Pulse: Big Week\n\nStandfirst only, no sections.\n") is False diff --git a/tests/test_tools.py b/tests/test_tools.py index 5e34b6e..74a8eb4 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -3,10 +3,16 @@ import httpx import pytest -# Import via importlib: agents/tools/__init__.py re-exports `search` and `web_fetch` -# as names on the package, which would shadow the submodules under `import ... as`. -search_module = importlib.import_module("agents.tools.search") -serper_module = importlib.import_module("agents.tools.serper") +from agents.tools.providers.dispatch import AllProvidersFailed, Provider, dispatch, reset_cursor + +# Import the provider/tool modules via importlib: `agents.tools` re-exports `web_search`/`web_fetch` +# as function names that would shadow the submodules under `import ... as`. +serper_module = importlib.import_module("agents.tools.providers.serper") +exa_module = importlib.import_module("agents.tools.providers.exa") +tavily_module = importlib.import_module("agents.tools.providers.tavily") +firecrawl_module = importlib.import_module("agents.tools.providers.firecrawl") +diffbot_module = importlib.import_module("agents.tools.providers.diffbot") +web_search_module = importlib.import_module("agents.tools.web_search") web_fetch_module = importlib.import_module("agents.tools.web_fetch") @@ -23,8 +29,11 @@ def json(self): return self._payload +# --- format_results (now lives in web_search) --- + + def test_format_results_empty(): - assert serper_module.format_results([]) == "No results found." + assert web_search_module.format_results([]) == "No results found." def test_format_results_builds_blocks(): @@ -32,17 +41,20 @@ def test_format_results_builds_blocks(): {"title": "T1", "link": "https://a.com/1", "snippet": "s1", "date": "2024"}, {"title": "T2", "link": "https://a.com/2", "snippet": "s2", "date": ""}, ] - out = serper_module.format_results(results) + out = web_search_module.format_results(results) assert "T1\nhttps://a.com/1\n2024 s1" in out - assert "\n\n" in out # blocks separated by a blank line + assert "\n\n" in out def test_format_results_caps_at_ten(): results = [{"title": f"T{index}", "link": f"https://a.com/{index}"} for index in range(15)] - out = serper_module.format_results(results) + out = web_search_module.format_results(results) assert out.count("https://a.com/") == 10 +# --- Serper low-level client + adapter --- + + def test_serper_news_endpoint_reads_news_key(monkeypatch): captured = {} @@ -63,45 +75,254 @@ def test_serper_other_endpoint_reads_organic_key(monkeypatch): assert serper_module.serper("search", "q", "", "") == [{"title": "O"}] -def test_search_maps_recency_and_endpoint(monkeypatch): +def test_serper_search_maps_recency_and_endpoint(monkeypatch): captured = {} def fake_serper(endpoint, query, gl, hl, tbs): captured.update(endpoint=endpoint, tbs=tbs) return [] - monkeypatch.setattr(search_module, "serper", fake_serper) - search_module.search("q", kind="news", recency="day") + monkeypatch.setattr(serper_module, "serper", fake_serper) + serper_module.search("q", "news", "", "", "day") assert captured == {"endpoint": "news", "tbs": "qdr:d"} - search_module.search("q", kind="web", recency="bogus") + serper_module.search("q", "web", "", "", "bogus") assert captured == {"endpoint": "search", "tbs": ""} +@pytest.mark.parametrize("recency,expected", [("day", "qdr:d"), ("week", "qdr:w"), ("month", "qdr:m")]) +def test_recency_table(recency, expected): + assert serper_module._RECENCY[recency] == expected + + +# --- Exa adapter --- + + +def test_exa_search_maps_to_serper_shape(monkeypatch): + monkeypatch.setenv("EXA_API_KEY", "exa-key") + captured = {} + item = {"title": "T", "url": "https://a.com/x", "publishedDate": "2026-01-01", "text": "hello world"} + + def fake_post(url, headers, json, timeout): + captured["url"] = url + captured["headers"] = headers + captured["body"] = json + return FakeResponse({"results": [item]}) + + monkeypatch.setattr(exa_module.httpx, "post", fake_post) + results = exa_module.search("apple", "news", "id", "id", "week") + assert captured["url"] == "https://api.exa.ai/search" + assert captured["headers"]["x-api-key"] == "exa-key" + assert captured["body"]["category"] == "news" + assert "startPublishedDate" in captured["body"] + assert results == [{"title": "T", "link": "https://a.com/x", "snippet": "hello world", "date": "2026-01-01"}] + + +def test_exa_fetch_returns_text(monkeypatch): + monkeypatch.setenv("EXA_API_KEY", "exa-key") + monkeypatch.setattr(exa_module.httpx, "post", lambda *a, **k: FakeResponse({"results": [{"text": "body"}]})) + assert exa_module.fetch("https://a.com/x") == "body" + + +def test_exa_fetch_empty_without_results(monkeypatch): + monkeypatch.setenv("EXA_API_KEY", "exa-key") + monkeypatch.setattr(exa_module.httpx, "post", lambda *a, **k: FakeResponse({"results": []})) + assert exa_module.fetch("https://a.com/x") == "" + + +# --- Tavily adapter --- + + +def test_tavily_search_maps_to_serper_shape(monkeypatch): + monkeypatch.setenv("TAVILY_API_KEY", "tav-key") + captured = {} + item = {"title": "T", "url": "https://a.com/y", "content": "snip", "published_date": "2026-02-02"} + + def fake_post(url, headers, json, timeout): + captured["url"] = url + captured["headers"] = headers + captured["body"] = json + return FakeResponse({"results": [item]}) + + monkeypatch.setattr(tavily_module.httpx, "post", fake_post) + results = tavily_module.search("apple", "news", "", "", "week") + assert captured["url"] == "https://api.tavily.com/search" + assert captured["headers"]["Authorization"] == "Bearer tav-key" + assert captured["body"]["topic"] == "news" + assert captured["body"]["time_range"] == "week" + assert results == [{"title": "T", "link": "https://a.com/y", "snippet": "snip", "date": "2026-02-02"}] + + +def test_tavily_fetch_returns_raw_content(monkeypatch): + monkeypatch.setenv("TAVILY_API_KEY", "tav-key") + + def fake_post(*a, **k): + return FakeResponse({"results": [{"raw_content": "raw"}]}) + + monkeypatch.setattr(tavily_module.httpx, "post", fake_post) + assert tavily_module.fetch("https://a.com/y") == "raw" + + +# --- Firecrawl adapter --- + + +def test_firecrawl_fetch_returns_markdown(monkeypatch): + monkeypatch.setenv("FIRECRAWL_API_KEY", "fc-key") + captured = {} + + def fake_post(url, headers, json, timeout): + captured["url"] = url + captured["headers"] = headers + return FakeResponse({"success": True, "data": {"markdown": "# md"}}) + + monkeypatch.setattr(firecrawl_module.httpx, "post", fake_post) + out = firecrawl_module.fetch("https://a.com/z") + assert captured["url"] == "https://api.firecrawl.dev/v1/scrape" + assert captured["headers"]["Authorization"] == "Bearer fc-key" + assert out == "# md" + + +# --- Diffbot adapter --- + + +def test_diffbot_fetch_returns_text(monkeypatch): + monkeypatch.setenv("DIFFBOT_API_KEY", "db-key") + captured = {} + + def fake_get(url, params, timeout): + captured["url"] = url + captured["params"] = params + return FakeResponse({"objects": [{"text": "article"}]}) + + monkeypatch.setattr(diffbot_module.httpx, "get", fake_get) + out = diffbot_module.fetch("https://a.com/article") + assert captured["url"] == "https://api.diffbot.com/v3/article" + assert captured["params"] == {"token": "db-key", "url": "https://a.com/article"} + assert out == "article" + + +# --- dispatch engine (fake providers) --- + + +def test_dispatch_round_robin_rotates(monkeypatch): + monkeypatch.setenv("A_KEY", "1") + monkeypatch.setenv("B_KEY", "1") + reset_cursor() + providers = [Provider("a", "A_KEY", lambda: "A"), Provider("b", "B_KEY", lambda: "B")] + assert dispatch("search", providers, lambda fn: fn(), accept=bool) == "A" + assert dispatch("search", providers, lambda fn: fn(), accept=bool) == "B" + assert dispatch("search", providers, lambda fn: fn(), accept=bool) == "A" + + +def test_dispatch_fails_over_on_error(monkeypatch): + monkeypatch.setenv("A_KEY", "1") + monkeypatch.setenv("B_KEY", "1") + reset_cursor() + + def boom(): + raise RuntimeError("down") + + providers = [Provider("a", "A_KEY", boom), Provider("b", "B_KEY", lambda: "B")] + assert dispatch("search", providers, lambda fn: fn(), accept=bool) == "B" + + +def test_dispatch_all_errored_raises_named(monkeypatch): + monkeypatch.setenv("A_KEY", "1") + monkeypatch.setenv("B_KEY", "1") + reset_cursor() + + def boom_a(): + raise RuntimeError("a-down") + + def boom_b(): + raise ValueError("b-down") + + providers = [Provider("a", "A_KEY", boom_a), Provider("b", "B_KEY", boom_b)] + + with pytest.raises(AllProvidersFailed) as excinfo: + dispatch("search", providers, lambda fn: fn(), accept=bool) + + message = str(excinfo.value) + assert "all search providers failed" in message + assert "a=" in message + assert "b=" in message + + +def test_dispatch_returns_empty_result_without_raising(monkeypatch): + monkeypatch.setenv("A_KEY", "1") + reset_cursor() + providers = [Provider("a", "A_KEY", lambda: [])] + assert dispatch("search", providers, lambda fn: fn(), accept=bool) == [] + + +def test_dispatch_skips_unconfigured_providers(monkeypatch): + monkeypatch.delenv("A_KEY", raising=False) + monkeypatch.setenv("B_KEY", "1") + reset_cursor() + providers = [Provider("a", "A_KEY", lambda: "A"), Provider("b", "B_KEY", lambda: "B")] + assert dispatch("search", providers, lambda fn: fn(), accept=bool) == "B" + + +def test_dispatch_no_providers_raises(monkeypatch): + monkeypatch.delenv("A_KEY", raising=False) + reset_cursor() + providers = [Provider("a", "A_KEY", lambda: "A")] + + with pytest.raises(AllProvidersFailed): + dispatch("search", providers, lambda fn: fn(), accept=bool) + + +# --- web_search / web_fetch routing (Serper active via conftest) --- + + +def test_web_search_routes_and_formats(monkeypatch): + reset_cursor() + + def fake_serper(endpoint, query, gl, hl, tbs): + return [{"title": "T", "link": "https://a.com/1", "snippet": "s", "date": "2026"}] + + monkeypatch.setattr(serper_module, "serper", fake_serper) + out = web_search_module.web_search("apple") + assert "https://a.com/1" in out + + def test_web_fetch_rejects_non_http(): assert "Not a fetchable URL" in web_fetch_module.web_fetch("ftp://x") def test_web_fetch_returns_truncated_text(monkeypatch): - monkeypatch.setattr(web_fetch_module.httpx, "post", lambda *a, **k: FakeResponse({"text": "x" * 7000})) + reset_cursor() + monkeypatch.setattr(serper_module.httpx, "post", lambda *a, **k: FakeResponse({"text": "x" * 7000})) out = web_fetch_module.web_fetch("https://a.com/article") assert len(out) == 6000 def test_web_fetch_handles_empty_content(monkeypatch): - monkeypatch.setattr(web_fetch_module.httpx, "post", lambda *a, **k: FakeResponse({})) + reset_cursor() + monkeypatch.setattr(serper_module.httpx, "post", lambda *a, **k: FakeResponse({})) assert web_fetch_module.web_fetch("https://a.com/article") == "No content." -def test_web_fetch_falls_back_on_http_error(monkeypatch): - def fake_post(*args, **kwargs): +def test_web_fetch_falls_back_when_all_providers_fail(monkeypatch): + reset_cursor() + + def boom(*a, **k): raise httpx.ConnectError("down") - monkeypatch.setattr(web_fetch_module.httpx, "post", fake_post) + monkeypatch.setattr(serper_module.httpx, "post", boom) out = web_fetch_module.web_fetch("https://a.com/article") assert "Could not fetch this URL" in out -@pytest.mark.parametrize("recency,expected", [("day", "qdr:d"), ("week", "qdr:w"), ("month", "qdr:m")]) -def test_recency_table(recency, expected): - assert search_module._RECENCY[recency] == expected +def test_web_fetch_fails_over_to_next_provider(monkeypatch): + monkeypatch.setenv("EXA_API_KEY", "exa-key") + reset_cursor() + + def fake_post(url, *a, **k): + if "serper" in url: + raise httpx.ConnectError("serper down") + + return FakeResponse({"results": [{"text": "exa body"}]}) + + monkeypatch.setattr(httpx, "post", fake_post) + assert web_fetch_module.web_fetch("https://a.com/article") == "exa body"