diff --git a/.env.example b/.env.example index 42071a4..c23dc84 100644 --- a/.env.example +++ b/.env.example @@ -1,29 +1,29 @@ -# Leave OPENAI_BASE_URL empty for OpenAI direct, or point it at any OpenAI-compatible gateway. -# OpenRouter (cheap access to Gemini, DeepSeek, Qwen, ...): https://openrouter.ai/api/v1 +# LLM gateway base URL. Leave empty for OpenAI direct, or point at any OpenAI-compatible gateway. OPENAI_BASE_URL=https://openrouter.ai/api/v1 # An OpenAI key for direct use, or an OpenRouter key (sk-or-...) for the gateway above. OPENAI_API_KEY=your-openrouter-or-openai-key-here +# Default model for every agent (override per role at the bottom of this file). OPENAI_MODEL=google/gemini-3-flash-preview -SERPER_API_KEY=your-serper-api-key-here -# Subject memory (Redis). Defaults to a local server; recall is skipped if it's unreachable. -REDIS_URL=redis://localhost:6379/0 -SUBJECT_TTL=604800 +# Web search via serper.dev. Required: the agents research every section through it. +SERPER_API_KEY=your-serper-api-key-here -# Email delivery (Resend). Only needed when sending: python main.py GOTO you@example.com +# Email delivery via Resend. Only needed when actually sending (python src/app.py run --send). RESEND_API_KEY= +# Sender shown on outgoing newsletters. EMAIL_FROM=MediaPulse -# Subscriber + ticker database (Postgres/Supabase, read-only). Used by the campaign job and to -# enrich the analyst with each ticker's listing profile (sector, business, HQ). Skipped if unset. +# Upstream MediaPulse Postgres (read-only): subscribers and each ticker's listing profile. MEDIAPULSE_DATABASE_URL= -# API auth. Callers must send this as the `X-API-Key` header. Generate: python -c "import secrets;print(secrets.token_urlsafe(32))" +# The app's own Postgres (read-write): archived newsletters and agent memory. +DATABASE_URL= + +# API auth. Callers send this as the `X-API-Key` header. Generate one with: +# python -c "import secrets; print(secrets.token_urlsafe(32))" SECRET_KEY= -# Optional per-agent model overrides (fall back to OPENAI_MODEL): -# All roles default to one model. To cut cost, set a cheaper model (e.g. google/gemini-3.1-flash-lite) -# for the high-volume researcher and writer roles. +# Optional per-agent model overrides (each falls back to OPENAI_MODEL). ANALYST_MODEL=google/gemini-3-flash-preview RESEARCHER_MODEL=google/gemini-3-flash-preview WRITER_MODEL=google/gemini-3-flash-preview diff --git a/.gitignore b/.gitignore index c08f09a..0816367 100644 --- a/.gitignore +++ b/.gitignore @@ -27,7 +27,7 @@ env/ .subject_cache.json # Generated email templates (built from email-playground, fresh each deploy/CI run) -/src/templates/*.html +/src/emails/templates/*.html # Node (email-playground) node_modules/ diff --git a/CLAUDE.md b/CLAUDE.md index 6798ea4..82ec4fc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,15 +8,17 @@ Guidance for working in this repository. ## Layout -All application code lives under `src/`. Packages keep their top-level names (`agents`, `utils`), so imports are `from agents...` / `from utils...`, with `src` on the path (pytest sets `pythonpath = ["src"]`; the Docker image runs uvicorn with `--app-dir src`). Run entrypoints from the repo root so `load_dotenv()` finds `.env`. +All application code lives under `src/`. Packages keep their top-level names (`agents`, `db`, `utils`), so imports are `from agents...` / `from db...` / `from utils...`, with `src` on the path (pytest sets `pythonpath = ["src"]`; the Docker image runs uvicorn with `--app-dir src`). Run entrypoints from the repo root so `load_dotenv()` finds `.env`. - `src/api.py` — FastAPI service. `POST /run` (full campaign) and `POST /test` (one user). Both run in the background, return `202`, and default to dry-run. Auth via the `X-API-Key` header matched against `SECRET_KEY`. - `src/app.py` — local CLI mirroring the API (`run`, `test`), for testing without HTTP. - `src/agents/orchestrator.py` — the pipeline: analyst → 5 parallel beat desks (researcher → writer → editor) → managing-editor gap roundtable → masthead → reviewer → deterministic clean/assemble/dedupe. Most non-agent logic (citation gating, URL/article validation, dedupe, subject-name canonicalization, prose humanizing) lives here. -- `src/agents/` — one module per agent (`analyst`, `researcher`, `writer`, `editor`, `managing_editor`, `reviewer`), plus `beats.py` (beat desks), `campaign.py` (top-level run over subscriptions), and `tools/` (Serper search, web fetch). +- `src/agents/` — one module per agent (`analyst`, `researcher`, `writer`, `editor`, `managing_editor`, `reviewer`), plus `beats.py` (beat desks), `campaign.py` (top-level run over subscriptions), `providers/` (subject-memory and ticker-profile context providers), and `tools/` (Serper search, web fetch). - `src/agents/skills/` — `SKILL.md` files that control agent behavior (`subject-profile`, `section-research`, `newsletter-format`). Prefer editing these over code when changing how agents research or write. -- `src/utils/` — `db.py` (subscriptions/tickers from the MediaPulse Postgres), `memory.py`, `client.py`, `guardrails.py`, `sections.py`, `mailer.py`, `email_template.py`, `ticker.py`. `email_template.py` no longer hand-builds HTML: it parses the newsletter markdown and fills the tokenized template at `src/templates/newsletter.html`. -- `email-playground/` — a standalone React Email (TypeScript) project that is the visual source-of-truth for MediaPulse email templates (the newsletter is the first one). `npm run build:templates` renders each template to a tokenized `src/templates/.html` that the Python side consumes (`email_template.py` for the newsletter). Those HTML files are gitignored build artifacts, regenerated fresh in CI and the Docker build. Restyle emails there, not in Python. See `email-playground/README.md`. +- `src/db/` — all database access. `mediapulse.py` reads subscriptions and ticker profiles from the upstream MediaPulse Postgres (`MEDIAPULSE_DATABASE_URL`, read-only, raw psycopg). The app's own Postgres (`DATABASE_URL`, SQLModel) uses `engine.py` for the shared engine, with each table's model alongside its operations in `newsletters.py` (archives each generated newsletter as markdown plus JSONB metadata) and `memory.py` (subject-brief agent memory). Tables are auto-created on first write. +- `src/emails/` — everything email-related. `mailer.py` sends via Resend. `templates/` pairs each template's renderer with its tokenized HTML: `templates/newsletter.py` parses the newsletter markdown and fills `templates/newsletter.html` (the gitignored build artifact from `email-playground`). +- `src/utils/` — `client.py`, `guardrails.py`, `sections.py`. +- `email-playground/` — a standalone React Email (TypeScript) project that is the visual source-of-truth for MediaPulse email templates (the newsletter is the first one). `npm run build:templates` renders each template to a tokenized `src/emails/templates/.html` that the Python side consumes (`emails/templates/newsletter.py` for the newsletter). Those HTML files are gitignored build artifacts, regenerated fresh in CI and the Docker build. Restyle emails there, not in Python. See `email-playground/README.md`. - `tests/` — pytest suite covering the deterministic logic (see Testing below). ## Setup @@ -57,7 +59,7 @@ pytest # whole suite conda run -n agentic-mediapulse pytest # if pytest is not on PATH ``` -The email-template tests read `src/templates/newsletter.html`, which is gitignored and generated by the playground. Build it once before running the suite locally (CI does this in the `tests` job): +The email-template tests read `src/emails/templates/newsletter.html`, which is gitignored and generated by the playground. Build it once before running the suite locally (CI does this in the `tests` job): ``` cd email-playground && npm install && npm run build:templates @@ -65,10 +67,10 @@ cd email-playground && npm install && npm run build:templates Config is in `pyproject.toml` under `[tool.pytest.ini_options]`: `pythonpath = ["src"]`, `testpaths = ["tests"]`, `asyncio_mode = "auto"` (async tests need no decorator). CI runs `pytest` in a separate `tests` job after installing both requirement files. -The suite covers the deterministic logic, not the LLM agents: orchestrator text/section helpers, the email template, Serper tools, the DB profile shaping, guardrail middleware, memory/ticker context providers, the mailer, the client model resolution, and the campaign delivery flow. Conventions for new tests: +The suite covers the deterministic logic, not the LLM agents: orchestrator text/section helpers, the email template, Serper tools, the MediaPulse profile shaping, the newsletter store and subject-memory upsert, guardrail middleware, the subject-memory and ticker-profile context providers, the mailer, the client model resolution, and the campaign delivery flow. Conventions for new tests: - `tests/conftest.py` sets dummy credentials before collection, because importing the agent modules constructs the whole agent graph at import time. -- Every external call (httpx, psycopg, redis, the LLM clients) is monkeypatched. The suite is fully offline. A test that reaches the network is a test bug. +- Every external call (httpx, psycopg, the LLM clients) is monkeypatched, and the SQLModel store is exercised against in-memory SQLite. The suite is fully offline. A test that reaches the network is a test bug. - Use `ACME` (and other clearly-fictional placeholders) for ticker symbols and company names, never real tickers. - The `test-runner` subagent runs the suite and reports failures without editing. @@ -81,9 +83,12 @@ The suite covers the deterministic logic, not the LLM agents: orchestrator text/ - Python 3.11, async throughout (`asyncio.gather` for parallel beats). - Keep prose free of em dashes and semicolons in generated output (`_humanize` enforces this in the pipeline). - Deterministic gates and agent feedback loops are bounded with explicit retry counts (`RETRIES`, `DISCUSSION_ROUNDS`); preserve those bounds when editing the orchestrator. +- Surround a multi-line block (`if`, `for`, `while`, `with`, `try`) with a blank line above and below whenever other code sits next to it in the same block. Also put a blank line above a `return` that has code before it. +- Keep docstrings and comments to a single concise line, and drop comments that only restate what the code shows. ## External services - `SERPER_API_KEY` — web search. -- `MEDIAPULSE_DATABASE_URL` — Postgres for subscriptions and ticker data; schema is defined in the upstream [MediaPulse](https://github.com/hyperjumptech/mediapulse) repo. +- `MEDIAPULSE_DATABASE_URL` — read-only Postgres for subscriptions and ticker data; schema is defined in the upstream [MediaPulse](https://github.com/hyperjumptech/mediapulse) repo. +- `DATABASE_URL` — the app's own read-write Postgres for archived newsletters and agent memory. Tables are auto-created on first use, separate from `MEDIAPULSE_DATABASE_URL`. - `SECRET_KEY` — API auth. diff --git a/Dockerfile b/Dockerfile index 6adeaae..e7fc91d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt COPY . . -COPY --from=template-builder /build/src/templates/ src/templates/ +COPY --from=template-builder /build/src/emails/templates/ src/emails/templates/ RUN useradd --no-create-home --shell /bin/false app \ && chown -R app:app /app diff --git a/README.md b/README.md index 772d789..cc3e2ed 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,8 @@ A simpler, agentic version of [MediaPulse](https://github.com/hyperjumptech/medi | `OPENAI_BASE_URL` | Leave blank for OpenAI, or point to an OpenAI-compatible gateway | Yes | | `OPENAI_MODEL` | Default model for every agent (e.g. `gpt-4.1-mini`) | Yes | | `SERPER_API_KEY` | Serper key (serper.dev) for news search and page scraping | Yes | -| `REDIS_URL` | Subject-memory store — default `redis://localhost:6379/0` | Yes | | `MEDIAPULSE_DATABASE_URL` | Postgres connection string (read-only) | Yes | +| `DATABASE_URL` | App Postgres (read-write): archived newsletters and agent memory | No | | `RESEND_API_KEY` | Resend key for email delivery | Yes | | `EMAIL_FROM` | Sender address, e.g. `MediaPulse ` | Yes | | `SECRET_KEY` | Required on every API request (`X-API-Key` header) | Yes | diff --git a/email-playground/README.md b/email-playground/README.md index e2b128e..696a909 100644 --- a/email-playground/README.md +++ b/email-playground/README.md @@ -28,7 +28,7 @@ npm run build:templates ``` For every `emails/*.tsx` that exports both a default component and a `templateProps` object, this -renders the component in template mode and writes `../src/templates/.html`: a tokenized HTML +renders the component in template mode and writes `../src/emails/templates/.html`: a tokenized HTML file with `{{placeholders}}` and `` blocks that the Python side fills per message. The output is a gitignored build artifact, regenerated in CI and the Docker build. **The Python tests and the running app require these files**, so run it once after `npm install`. @@ -41,7 +41,7 @@ per message. The output is a gitignored build artifact, regenerated in CI and th `` (which emits `[[#name]]`/`[[/name]]` markers) and leave content as `{{token}}` placeholders. 3. Export `templateProps`: the props that drive template-mode rendering. -4. Run `npm run build:templates`, then fill `src/templates/.html` from Python. +4. Run `npm run build:templates`, then fill `src/emails/templates/.html` from Python. Templates without `templateProps` are preview-only and are skipped by the build. diff --git a/email-playground/package.json b/email-playground/package.json index f5bbef9..d4ec299 100644 --- a/email-playground/package.json +++ b/email-playground/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "private": true, "type": "module", - "description": "React Email templates and preview playground for MediaPulse emails. Exports tokenized HTML templates into src/templates for the Python app to fill.", + "description": "React Email templates and preview playground for MediaPulse emails. Exports tokenized HTML templates into src/emails/templates for the Python app to fill.", "scripts": { "dev": "email dev", "export": "email export", diff --git a/email-playground/scripts/build-templates.mts b/email-playground/scripts/build-templates.mts index e1ff501..972349e 100644 --- a/email-playground/scripts/build-templates.mts +++ b/email-playground/scripts/build-templates.mts @@ -6,7 +6,7 @@ import * as React from "react"; const scriptDir = dirname(fileURLToPath(import.meta.url)); const emailsDir = resolve(scriptDir, "../emails"); -const outputDir = resolve(scriptDir, "../../src/templates"); +const outputDir = resolve(scriptDir, "../../src/emails/templates"); function toMarkers(html: string): string { return html.replace(/\[\[#(\w+)\]\]/g, "").replace(/\[\[\/(\w+)\]\]/g, ""); diff --git a/environment.yml b/environment.yml index e4a276d..e61c35f 100644 --- a/environment.yml +++ b/environment.yml @@ -8,7 +8,7 @@ dependencies: - agent-framework - python-dotenv - httpx - - redis - psycopg[binary] + - sqlmodel - fastapi - uvicorn diff --git a/requirements.txt b/requirements.txt index 13ff99d..7850382 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ agent-framework python-dotenv httpx -redis psycopg[binary] +sqlmodel fastapi uvicorn diff --git a/src/agents/analyst.py b/src/agents/analyst.py index 88a9147..e069958 100644 --- a/src/agents/analyst.py +++ b/src/agents/analyst.py @@ -1,10 +1,10 @@ from agent_framework import Agent +from agents.providers.memory import SubjectMemoryProvider +from agents.providers.ticker import TickerProfileProvider from agents.tools import search, web_fetch from utils.client import SKILLS, chat_client from utils.guardrails import SubjectGuardrail -from utils.memory import SubjectMemoryProvider -from utils.ticker import TickerProfileProvider analyst = Agent( name="analyst", diff --git a/src/agents/campaign.py b/src/agents/campaign.py index 506f9a9..c86ec0c 100644 --- a/src/agents/campaign.py +++ b/src/agents/campaign.py @@ -2,29 +2,25 @@ import time from agents.orchestrator import run_newsletter -from utils.db import fetch_subscriptions -from utils.email_template import render_newsletter_email -from utils.mailer import send_email +from db.mediapulse import fetch_subscriptions +from db.newsletters import save_newsletter +from emails.mailer import send_email +from emails.templates.newsletter import newsletter_sources, render_newsletter_email CONCURRENCY = 3 -SEND_INTERVAL = 1.0 # seconds between emails — keeps us under Resend's rate limit (1/sec) +SEND_INTERVAL = 1.0 # seconds between emails async def run_campaign(*, subscriptions: list[dict] | None = None, send: bool = False, log=print) -> dict: - """Generate one newsletter per followed ticker and deliver each the moment it is ready. - - Tickers are generated at most CONCURRENCY at a time. As soon as a ticker's newsletter is - done it is sent to that ticker's subscribers and then dropped, so the whole batch is never - held in memory at once. Sends are globally rate-limited to one email per SEND_INTERVAL - seconds. `send=False` is a dry run: it generates and renders but does not email. - Pass `subscriptions` to scope the run (e.g. one user); defaults to all active subscribers. - """ + """Generate one newsletter per followed ticker and deliver each the moment it is ready.""" if subscriptions is None: subscriptions = fetch_subscriptions() subscribers_by_ticker: dict[str, list[dict]] = {} + for subscription in subscriptions: - subscribers_by_ticker.setdefault(subscription["symbol"], []).append(subscription) + subscribers_by_ticker.setdefault(subscription["ticker"], []).append(subscription) + tickers = sorted(subscribers_by_ticker) log(f"{len(subscriptions)} subscriptions, {len(tickers)} unique tickers, concurrency {CONCURRENCY}") @@ -33,35 +29,44 @@ async def run_campaign(*, subscriptions: list[dict] | None = None, send: bool = last_send = 0.0 delivered: list[dict] = [] - async def deliver(symbol: str, recipients: list[dict], markdown: str) -> None: + async def deliver(ticker: str, recipients: list[dict], markdown: str) -> None: nonlocal last_send - email = render_newsletter_email(markdown, subject_symbol=symbol) + email = render_newsletter_email(markdown, ticker=ticker) + save_newsletter(ticker, markdown, {"ticker": ticker, "sources": newsletter_sources(markdown)}) + for subscription in recipients: if send: # Serialize sends so the 1/sec limit holds even across concurrently finished tickers. async with send_lock: wait = SEND_INTERVAL - (time.monotonic() - last_send) + if wait > 0: await asyncio.sleep(wait) + send_email(subscription["email"], email["subject"], email["html"], email["text"]) last_send = time.monotonic() - log(f"sent {symbol} -> {subscription['email']}") + + log(f"sent {ticker} -> {subscription['email']}") else: - log(f"[dry-run] would send {symbol} -> {subscription['email']}: {email['subject']}") - delivered.append({"email": subscription["email"], "symbol": symbol}) + log(f"[dry-run] would send {ticker} -> {subscription['email']}: {email['subject']}") - async def process(symbol: str, recipients: list[dict]) -> None: + delivered.append({"email": subscription["email"], "ticker": ticker}) + + async def process(ticker: str, recipients: list[dict]) -> None: async with generate_semaphore: - log(f"generating {symbol} ...") + log(f"generating {ticker} ...") + try: - markdown = await run_newsletter(symbol) + markdown = await run_newsletter(ticker) except Exception as error: - log(f"failed {symbol}: {error}") + log(f"failed {ticker}: {error}") + return - log(f"done {symbol}") - # Generation slot is freed above; send now, then `markdown` falls out of scope and is released. - await deliver(symbol, recipients, markdown) - await asyncio.gather(*(process(symbol, recipients) for symbol, recipients in subscribers_by_ticker.items())) + log(f"done {ticker}") + + await deliver(ticker, recipients, markdown) + + await asyncio.gather(*(process(ticker, recipients) for ticker, recipients in subscribers_by_ticker.items())) return {"subscriptions": len(subscriptions), "tickers": tickers, "delivered": delivered, "sent": send} diff --git a/src/agents/orchestrator.py b/src/agents/orchestrator.py index 037c32e..fe9a0f5 100644 --- a/src/agents/orchestrator.py +++ b/src/agents/orchestrator.py @@ -7,7 +7,7 @@ from agents.editor import editor from agents.managing_editor import managing_editor from agents.reviewer import reviewer -from utils.memory import remember_subject +from db.memory import remember_subject from utils.sections import SECTIONS _URL_RE = re.compile(r"https?://[^\s\)]+") @@ -41,13 +41,18 @@ def _is_article(url: str) -> bool: parsed = urllib.parse.urlparse(url) except ValueError: return False + if parsed.scheme not in ("http", "https") or not parsed.netloc: return False + segments = [segment for segment in parsed.path.split("/") if segment] + if not segments: return False + if parsed.query and {key.lower() for key in urllib.parse.parse_qs(parsed.query)} & _LISTING_PARAMS: return False + slug = re.sub(r"\.(html?|aspx?|php|jsp)$", "", segments[-1], flags=re.IGNORECASE) return slug.count("-") >= 2 or bool(re.fullmatch(r"\d{4,}", slug)) @@ -59,13 +64,16 @@ def _is_article(url: str) -> bool: def _section_problems(draft: str, allowed_urls: set[str]) -> list[str]: problems: list[str] = [] reads = _READ_RE.findall(draft) + if len(reads) < 2: problems.append( "Include at least 2 and at most 5 entries, each a single sentence" " ending with a '[Read: ](<url>)' line." ) + for raw in reads: url = raw.strip() + if not url.startswith("http"): problems.append(f"Entry has no real URL: {url or '(empty)'}") elif url not in allowed_urls: @@ -86,25 +94,32 @@ async def _run_beat(beat, brief: str) -> tuple[str, str]: # researcher finds sources; deterministic sufficiency feedback candidates, note = "", "" + for _ in range(RETRIES): candidates = (await beat.researcher.run(head + note)).text + if len({*_URL_RE.findall(candidates)}) >= 3: break + note = "\n\nFEEDBACK: too few sources with real URLs, widen recency to 'week' or try different queries." # writer drafts from candidates; deterministic citation/format feedback draft, feedback = "", "" + for _ in range(RETRIES + 1): draft = ( await beat.writer.run(f"BRIEF:\n{brief}\n\nCANDIDATES:\n{candidates}\n\nSECTION: {section.name}{feedback}") ).text problems = _section_problems(draft, beat.registry.urls) + if not problems: break + feedback = "\n\nFEEDBACK (fix these):\n- " + "\n- ".join(problems) # editor reviews the section (agent feedback -> one writer revision) verdict = (await editor.run(f"REVIEW the '{section.name}' section:\n\n{draft}")).text + if not _is_ok(verdict): draft = ( await beat.writer.run( @@ -125,11 +140,15 @@ async def _cover_beats(brief: str) -> dict[str, str]: def _gaps_from(verdict: str, sections: set[str]) -> list[tuple[str, str]]: """Parse the managing editor's 'Section :: missing angle' lines into known-section gaps.""" gaps: list[tuple[str, str]] = [] + for line in verdict.splitlines(): match = _GAP_RE.match(line) + if not match: continue + section, need = match.group(1).strip(), match.group(2).strip() + if section in sections and need: gaps.append((section, need)) @@ -158,15 +177,21 @@ async def _newsroom_discussion(brief: str, drafts: dict[str, str]) -> dict[str, once the editor says the edition is complete. New entries pass the same per-section guards. """ beats_by_name = {beat.section.name: beat for beat in BEATS} + for _ in range(DISCUSSION_ROUNDS): edition = "\n\n".join(f"## {section.name}\n{drafts[section.name]}" for section in SECTIONS) verdict = (await managing_editor.run(f"BRIEF:\n{brief}\n\nCURRENT EDITION:\n{edition}")).text + if verdict.strip().upper().startswith("COMPLETE"): break + gaps = _gaps_from(verdict, set(beats_by_name))[:MAX_GAPS_PER_ROUND] + if not gaps: break + additions = await asyncio.gather(*(_fill_gap(beats_by_name[section], brief, need) for section, need in gaps)) + for (section, _), addition in zip(gaps, additions): if addition.strip(): drafts[section] = f"{drafts[section]}\n\n{addition}" @@ -177,8 +202,10 @@ async def _newsroom_discussion(brief: str, drafts: dict[str, str]) -> dict[str, async def _masthead(brief: str, drafts: dict[str, str], notes: str = "") -> tuple[str, str]: body = "\n\n".join(f"## {section.name}\n{drafts[section.name]}" for section in SECTIONS) prompt = f"MASTHEAD\n\nBRIEF:\n{brief}\n\nSECTION DRAFTS:\n{body}" + if notes: prompt += f"\n\nREVIEWER NOTES (address these):\n{notes}" + out = (await editor.run(prompt)).text.strip() lines = out.splitlines() title = lines[0].lstrip("# ").strip() if lines else "Newsletter" @@ -226,14 +253,18 @@ def _subject_terms(subject: str, brief: str) -> tuple[str, "re.Pattern | None"]: match = _SUBJECT_LINE_RE.search(brief) raw_name = match.group(1).strip() if match else "" name = _short_name(raw_name) or raw_name or subject + if len(name) < 2: return subject, None # Company-name aliases match case-insensitively: they are distinctive proper nouns with no common-word risk. names: set[str] = {name} + if raw_name: names.update({raw_name, f"PT {name} Tbk", f"PT {name}", f"{name} Tbk"}) + words = name.split() + if len(words) >= 3: names.add(" ".join(words[:2])) # fold a shortened form back, but never down to one (often generic) word @@ -241,15 +272,19 @@ def _subject_terms(subject: str, brief: str) -> tuple[str, "re.Pattern | None"]: # so folding "baby" or "map" into the company name would wreck the prose; only the all-caps ticker is meant. tickers: set[str] = set() ticker_match = _TICKER_LINE_RE.search(brief) + for candidate in (subject, ticker_match.group(1) if ticker_match else ""): token = re.split(r"[\s,/(]+", candidate.strip(), maxsplit=1)[0].strip(" .") + if re.fullmatch(r"[A-Z]{3,6}\d{0,2}", token): # an uppercase exchange ticker like DSSA or GOTO tickers.add(token) alternatives = [re.escape(alias) for alias in sorted(names, key=len, reverse=True) if alias] alternatives += [f"(?-i:{re.escape(ticker)})" for ticker in sorted(tickers, key=len, reverse=True)] + if not alternatives: return name, None + pattern = re.compile(r"\b(?:" + "|".join(alternatives) + r")\b", re.IGNORECASE) return name, pattern @@ -263,6 +298,7 @@ def _canonicalize_subject(text: str, name: str, pattern: "re.Pattern | None") -> """ if pattern is None: return text + text = pattern.sub(name, text) text = re.sub(r"\b(" + re.escape(name) + r")(?:\s+\1\b)+", r"\1", text) # collapse an accidental "Name Name" @@ -275,6 +311,7 @@ def _canonical_url(url: str) -> str: parsed = urllib.parse.urlparse(url.strip().lower()) except ValueError: return url.strip().lower() + host = parsed.netloc.removeprefix("www.") return f"{host}{parsed.path.rstrip('/')}" @@ -289,6 +326,7 @@ def _too_similar(tokens: frozenset, seen: list, threshold: float = 0.55) -> bool """True if `tokens` overlaps any kept entry at or above `threshold` (Jaccard similarity).""" for prior in seen: union = tokens | prior + if union and len(tokens & prior) / len(union) >= threshold: return True @@ -303,9 +341,12 @@ def _first_sentences(text: str, limit: int = _MAX_SENTENCES) -> str: parts = _SENTENCE_SPLIT_RE.split(text) kept = text if len(parts) <= limit else " ".join(parts[:limit]) words = kept.split() + if len(words) <= _MAX_WORDS: return kept + clipped = " ".join(words[:_MAX_WORDS]) + if "," in clipped: clipped = clipped[: clipped.rfind(",")] # end on a clause, not mid-thought @@ -321,24 +362,36 @@ def _clean_section(draft: str, name: str = "", pattern: "re.Pattern | None" = No """ lines = draft.splitlines() entries: list[str] = [] + for index, line in enumerate(lines): link = _LINK_RE.search(line) + if not link: continue + cursor = index - 1 + while cursor >= 0 and not lines[cursor].strip(): cursor -= 1 + if cursor < 0: continue + body = lines[cursor].strip() + if body == "---" or body.startswith("#") or _LINK_RE.search(body): continue + body = _LEADING_MARKER_RE.sub("", body) # drop any leading bullet or number marker + if _MARKET_NOISE_RE.search(body) or _MARKET_NOISE_RE.search(link.group(0)): continue # drop stock-index / price / technical-analysis noise; readers want company news + body = _humanize(body) + if name: body = _canonicalize_subject(body, name, pattern) + body = _first_sentences(body) entries.append(f"{body}\n\n{link.group(0)}\n\n---") @@ -347,19 +400,25 @@ def _clean_section(draft: str, name: str = "", pattern: "re.Pattern | None" = No def _assemble( - symbol: str, title: str, summary: str, drafts: dict[str, str], name: str = "", pattern: "re.Pattern | None" = None + subject: str, title: str, summary: str, drafts: dict[str, str], name: str = "", pattern: "re.Pattern | None" = None ) -> str: - symbol = symbol.strip() + subject = subject.strip() + # The editor sometimes echoes a "<TICKER> Pulse:" prefix; strip any leading one so we never double it. while _PULSE_PREFIX_RE.match(title): title = _PULSE_PREFIX_RE.sub("", title, count=1).strip() + if name: summary = _canonicalize_subject(summary, name, pattern) - parts = [f"# {symbol} Pulse: {title}", "", summary, ""] + + parts = [f"# {subject} Pulse: {title}", "", summary, ""] + for section in SECTIONS: body = _clean_section(drafts.get(section.name, ""), name, pattern) + if not body.strip(): continue # skip a section with no usable entries rather than printing a bare heading + parts.append(f"## {section.name}") parts.append(body) parts.append("") @@ -380,18 +439,24 @@ def flush() -> bool: line for line in entry if line.strip() and line.strip() != "---" and not _LINK_RE.search(line) ) tokens = _content_key(summary) + if (urls and any(url in seen_urls for url in urls)) or _too_similar(tokens, seen_tokens): entry.clear() return False + seen_urls.update(urls) + if tokens: seen_tokens.append(tokens) + out.extend(entry) entry.clear() + return True for line in newsletter.splitlines(): stripped = line.strip() + if stripped == "---": if flush(): out.append(line) @@ -401,6 +466,7 @@ def flush() -> bool: out.append(line) else: entry.append(line) + out.extend(entry) return "\n".join(out) @@ -418,6 +484,7 @@ async def run_newsletter(subject: str) -> str: # reviewer critiques the whole edition (agent feedback -> one masthead revision) verdict = (await reviewer.run(newsletter)).text + if not _is_ok(verdict): title, summary = await _masthead(brief, drafts, notes=verdict) newsletter = _assemble(subject, title, summary, drafts, name, pattern) diff --git a/src/templates/.gitkeep b/src/agents/providers/__init__.py similarity index 100% rename from src/templates/.gitkeep rename to src/agents/providers/__init__.py diff --git a/src/agents/providers/memory.py b/src/agents/providers/memory.py new file mode 100644 index 0000000..ee2d366 --- /dev/null +++ b/src/agents/providers/memory.py @@ -0,0 +1,28 @@ +import asyncio +from typing import Any + +from agent_framework import AgentSession, ContextProvider, SessionContext + +from db.memory import recall_subject + + +class SubjectMemoryProvider(ContextProvider): + """Recall a previously resolved brief for the subject from Postgres.""" + + SOURCE_ID = "subject_memory" + + def __init__(self) -> None: + super().__init__(self.SOURCE_ID) + + async def before_run( + self, *, agent: Any, session: AgentSession | None, context: SessionContext, state: dict[str, Any] + ) -> None: + text = context.input_messages[-1].text if context.input_messages else "" + brief = await asyncio.to_thread(recall_subject, text) + + if brief: + context.extend_instructions( + self.source_id, + f"A previously resolved brief for '{text.strip()}' is below. Verify it is still current, " + f"then update only what changed:\n{brief}", + ) diff --git a/src/utils/ticker.py b/src/agents/providers/ticker.py similarity index 81% rename from src/utils/ticker.py rename to src/agents/providers/ticker.py index 7bd1d93..e9891aa 100644 --- a/src/utils/ticker.py +++ b/src/agents/providers/ticker.py @@ -3,7 +3,7 @@ from agent_framework import AgentSession, ContextProvider, SessionContext -from utils.db import fetch_ticker_profile +from db.mediapulse import fetch_ticker_profile class TickerProfileProvider(ContextProvider): @@ -17,20 +17,23 @@ def __init__(self) -> None: async def before_run( self, *, agent: Any, session: AgentSession | None, context: SessionContext, state: dict[str, Any] ) -> None: - symbol = (context.input_messages[-1].text if context.input_messages else "").strip() - if not symbol: + ticker = (context.input_messages[-1].text if context.input_messages else "").strip() + + if not ticker: return + try: - # Run the blocking DB lookup off the event loop; best-effort, never blocks generation. - profile = await asyncio.to_thread(fetch_ticker_profile, symbol) + profile = await asyncio.to_thread(fetch_ticker_profile, ticker) except Exception: profile = None + if not profile: return + details = "\n".join(f"- {label}: {value}" for label, value in profile.items()) context.extend_instructions( self.source_id, - f"The subject '{symbol}' is a listed stock ticker. Verified company details from the exchange " + f"The subject '{ticker}' is a listed stock ticker. Verified company details from the exchange " "listing database are below, treat them as ground truth. Anchor the brief on this exact company: " "use these for Subject, Ticker/Exchange, Sector, and Key players, and infer Market and Locale " "(gl/hl codes) from the headquarters location. Some values are in Indonesian, render them in " diff --git a/src/agents/tools/serper.py b/src/agents/tools/serper.py index 2111b1f..200569c 100644 --- a/src/agents/tools/serper.py +++ b/src/agents/tools/serper.py @@ -5,10 +5,13 @@ def serper(endpoint: str, query: str, gl: str, hl: str, tbs: str = "") -> list[dict]: body = {"q": query} + if gl: body["gl"] = gl + if hl: body["hl"] = hl + if tbs: body["tbs"] = tbs @@ -28,6 +31,7 @@ def format_results(results: list[dict]) -> str: return "No results found." blocks = [] + for item in results[:10]: title = item.get("title", "") link = item.get("link", "") diff --git a/src/api.py b/src/api.py index f81e500..295af45 100644 --- a/src/api.py +++ b/src/api.py @@ -9,7 +9,7 @@ from fastapi import BackgroundTasks, Depends, FastAPI, Header, HTTPException from agents.campaign import run_campaign -from utils.db import fetch_subscriptions +from db.mediapulse import fetch_subscriptions logging.basicConfig(level=logging.INFO) app = FastAPI(title="MediaPulse Newsletter API") @@ -19,8 +19,10 @@ def require_api_key(x_api_key: str = Header(default="")) -> None: """Reject requests whose `X-API-Key` header does not match the `SECRET_KEY` env var.""" expected = os.getenv("SECRET_KEY", "") + if not expected: raise HTTPException(status_code=503, detail="Server missing SECRET_KEY") + if not secrets.compare_digest(x_api_key, expected): raise HTTPException(status_code=401, detail="Invalid or missing API key") diff --git a/src/app.py b/src/app.py index 8e2bbe7..8b6b527 100644 --- a/src/app.py +++ b/src/app.py @@ -6,7 +6,7 @@ load_dotenv() from agents.campaign import run_campaign -from utils.db import fetch_subscriptions +from db.mediapulse import fetch_subscriptions def main() -> None: diff --git a/src/db/__init__.py b/src/db/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/db/engine.py b/src/db/engine.py new file mode 100644 index 0000000..1c84ec4 --- /dev/null +++ b/src/db/engine.py @@ -0,0 +1,37 @@ +import os + +from sqlmodel import SQLModel, create_engine + +_engine = None + + +def is_configured() -> bool: + """True when the app's own Postgres (`DATABASE_URL`) is configured.""" + return bool(os.getenv("DATABASE_URL")) + + +def _engine_url() -> str: + # Drop the Prisma-style `?schema=` query, then force the psycopg3 driver (the only one installed). + url = os.environ["DATABASE_URL"].split("?")[0] + + for prefix in ("postgresql://", "postgres://"): + if url.startswith(prefix): + return "postgresql+psycopg://" + url[len(prefix) :] + + return url + + +def get_engine(): + """Build the engine once and create the app's tables (idempotent, once per process).""" + global _engine + + if _engine is None: + # Import here so the models register on SQLModel.metadata before create_all, avoiding a cycle. + from db.memory import SubjectMemory + from db.newsletters import Newsletter + + engine = create_engine(_engine_url(), pool_pre_ping=True) + SQLModel.metadata.create_all(engine, tables=[Newsletter.__table__, SubjectMemory.__table__]) + _engine = engine + + return _engine diff --git a/src/utils/db.py b/src/db/mediapulse.py similarity index 74% rename from src/utils/db.py rename to src/db/mediapulse.py index f75b79e..b7dab72 100644 --- a/src/utils/db.py +++ b/src/db/mediapulse.py @@ -5,7 +5,7 @@ # Active (email, ticker) subscriptions. Read-only. _SUBSCRIPTIONS_SQL = """ - select u.email, u.name, t.symbol, t.name as ticker_name + select u.email, u.name, t.symbol as ticker, t.name as ticker_name from mediapulse.user_ticker ut join mediapulse.mediapulse_user u on u.id = ut.user_id join mediapulse.ticker t on t.id = ut.ticker_id @@ -16,27 +16,28 @@ def _conninfo() -> str: - # Drop the Prisma-style `?schema=` query; libpq does not accept it. + # Drop the Prisma-style `?schema=` query. libpq does not accept it. return os.environ["MEDIAPULSE_DATABASE_URL"].split("?")[0] -def fetch_subscriptions(symbol: str | None = None, email: str | None = None) -> list[dict]: - """Return active subscriptions as dicts: email, name, symbol, ticker_name. Read-only. - - Pass `symbol` to limit to one ticker's subscribers, or `email` to limit to one user. - """ +def fetch_subscriptions(ticker: str | None = None, email: str | None = None) -> list[dict]: + """Return active subscriptions as dicts: email, name, ticker, ticker_name. Read-only.""" sql = _SUBSCRIPTIONS_SQL params: list = [] - if symbol: + + if ticker: sql += " and t.symbol = %s" - params.append(symbol) + params.append(ticker) + if email: sql += " and u.email = %s" params.append(email) + sql += " order by u.email, t.symbol" with psycopg.connect(_conninfo(), connect_timeout=15) as conn: conn.read_only = True + with conn.cursor() as cur: cur.execute(sql, params) columns = [column.name for column in cur.description] @@ -73,21 +74,18 @@ def _clean(value: object) -> str: return re.sub(r"\s+", " ", str(value or "").replace("\r", " ").replace("\n", " ")).strip(" ,-") -def fetch_ticker_profile(symbol: str) -> dict[str, str] | None: - """Return a curated company profile for a ticker symbol, or None if unknown. Read-only. - - Reads the IDX listing metadata (sector taxonomy, main business, headquarters, website, - listing board and date) so the analyst has real context beyond the bare symbol. Returns - None when the symbol is blank, the database is not configured, or the symbol is not listed. - """ - if not (symbol and symbol.strip() and os.getenv("MEDIAPULSE_DATABASE_URL")): +def fetch_ticker_profile(ticker: str) -> dict[str, str] | None: + """Return a curated company profile for a ticker, or None if unknown. Read-only.""" + if not (ticker and ticker.strip() and os.getenv("MEDIAPULSE_DATABASE_URL")): return None with psycopg.connect(_conninfo(), connect_timeout=15) as conn: conn.read_only = True + with conn.cursor() as cur: - cur.execute(_TICKER_SQL, [symbol.strip()]) + cur.execute(_TICKER_SQL, [ticker.strip()]) row = cur.fetchone() + if not row: return None @@ -96,18 +94,23 @@ def fetch_ticker_profile(symbol: str) -> dict[str, str] | None: # The four taxonomy fields are often identical (e.g. Bank/Bank/Bank); keep each value once. seen: set[str] = set() + for key, label in _TAXONOMY: value = _clean(metadata.get(key)) + if value and value.lower() not in seen: profile[label] = value seen.add(value.lower()) for key, label in _DETAILS: value = _clean(metadata.get(key)) + if not value: continue + if label == "Listed since": value = value[:10] # keep the date, drop the T00:00:00 time part + profile[label] = value return profile diff --git a/src/db/memory.py b/src/db/memory.py new file mode 100644 index 0000000..5f42063 --- /dev/null +++ b/src/db/memory.py @@ -0,0 +1,64 @@ +import logging +from datetime import datetime + +from sqlalchemy import Column, DateTime, func +from sqlmodel import Field, Session, SQLModel + +from db.engine import get_engine, is_configured + +logger = logging.getLogger(__name__) + + +class SubjectMemory(SQLModel, table=True): + __tablename__ = "subject_memory" + + subject: str = Field(primary_key=True) # normalized key: stripped + lowercased + brief: str + updated_at: datetime | None = Field( + default=None, + sa_column=Column(DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now()), + ) + + +def _key(subject: str) -> str: + return (subject or "").strip().lower() + + +def remember_subject(subject: str, brief: str) -> None: + """Upsert a resolved brief for a subject (best-effort, no-op if unconfigured).""" + key = _key(subject) + + if not key or not is_configured(): + return + + try: + with Session(get_engine()) as session: + existing = session.get(SubjectMemory, key) + + if existing: + existing.brief = brief + else: + existing = SubjectMemory(subject=key, brief=brief) + + session.add(existing) + session.commit() + except Exception as error: + logger.warning("failed to store subject memory for %s: %s", subject, error) + + +def recall_subject(subject: str) -> str | None: + """Return the latest stored brief for a subject, or None (best-effort).""" + key = _key(subject) + + if not key or not is_configured(): + return None + + try: + with Session(get_engine()) as session: + row = session.get(SubjectMemory, key) + + return row.brief if row else None + except Exception as error: + logger.warning("failed to recall subject memory for %s: %s", subject, error) + + return None diff --git a/src/db/newsletters.py b/src/db/newsletters.py new file mode 100644 index 0000000..621cd79 --- /dev/null +++ b/src/db/newsletters.py @@ -0,0 +1,45 @@ +import logging +from datetime import datetime + +from sqlalchemy import JSON, Column, DateTime, func +from sqlalchemy.dialects.postgresql import JSONB +from sqlmodel import Field, Session, SQLModel + +from db.engine import get_engine, is_configured + +logger = logging.getLogger(__name__) + +# JSONB on Postgres, plain JSON elsewhere so the offline test suite can use SQLite. +_JSON = JSON().with_variant(JSONB(), "postgresql") + + +class Newsletter(SQLModel, table=True): + __tablename__ = "newsletters" + + id: int | None = Field(default=None, primary_key=True) + subject: str = Field(index=True) + content: str + meta: dict = Field(default_factory=dict, sa_column=Column("metadata", _JSON, nullable=False)) + created_at: datetime | None = Field( + default=None, sa_column=Column(DateTime(timezone=True), nullable=False, server_default=func.now()) + ) + + +def save_newsletter(subject: str, content: str, metadata: dict | None = None) -> int | None: + """Archive a generated newsletter and return its row id, or None if storage is unconfigured or the write fails.""" + if not is_configured(): + return None + + try: + newsletter = Newsletter(subject=subject, content=content, meta=metadata or {}) + + with Session(get_engine()) as session: + session.add(newsletter) + session.commit() + session.refresh(newsletter) + + return newsletter.id + except Exception as error: + logger.warning("failed to store newsletter for %s: %s", subject, error) + + return None diff --git a/src/emails/__init__.py b/src/emails/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/utils/mailer.py b/src/emails/mailer.py similarity index 99% rename from src/utils/mailer.py rename to src/emails/mailer.py index 4ce7d82..a39a96c 100644 --- a/src/utils/mailer.py +++ b/src/emails/mailer.py @@ -14,6 +14,7 @@ def send_email(to: str | list[str], subject: str, html: str, text: str | None = "subject": subject, "html": html, } + if text: payload["text"] = text diff --git a/src/emails/templates/__init__.py b/src/emails/templates/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/utils/email_template.py b/src/emails/templates/newsletter.py similarity index 89% rename from src/utils/email_template.py rename to src/emails/templates/newsletter.py index 2c472d0..474125c 100644 --- a/src/utils/email_template.py +++ b/src/emails/templates/newsletter.py @@ -1,7 +1,7 @@ """Render a newsletter (our markdown format) into a MediaPulse HTML email + plain text. The HTML comes from a tokenized template generated by the `email-playground` React Email -project (`npm run build:templates` writes `src/templates/newsletter.html`). This module only +project (`npm run build:templates` writes `src/emails/templates/newsletter.html`). This module only parses the markdown and fills the template's `{{tokens}}` and `<!--#region-->` blocks, so the visual design lives in React, not here. The template is a build artifact and is gitignored, so it must be generated before this module can render (CI and the Docker build do this; locally run @@ -13,7 +13,7 @@ from functools import lru_cache from pathlib import Path -_TEMPLATE_PATH = Path(__file__).resolve().parent.parent / "templates" / "newsletter.html" +_TEMPLATE_PATH = Path(__file__).resolve().parent / "newsletter.html" _QUICK_HITS = "Quick Hits" @@ -28,12 +28,14 @@ def _inline(text: str) -> str: """Escape text and turn inline `[label](https://…)` into anchors; newlines to <br>.""" out: list[str] = [] last = 0 + for match in _INLINE.finditer(text): out.append(_html.escape(text[last : match.start()])) label = _html.escape(match.group(1)) url = _html.escape(match.group(2), quote=True) out.append(f'<a href="{url}" style="{_LINK}">{label}</a>') last = match.end() + out.append(_html.escape(text[last:])) return "".join(out).replace("\n", "<br>") @@ -48,21 +50,39 @@ def _parse(markdown: str) -> tuple[str, str, list[tuple[str, list[dict]]]]: summary = parts[0].strip() sections: list[tuple[str, list[dict]]] = [] + for index in range(1, len(parts), 2): name = parts[index].strip() content = parts[index + 1] if index + 1 < len(parts) else "" items: list[dict] = [] + for block in re.split(r"(?m)^\s*---\s*$", content): link = _READ.search(block) + if not link: continue + text = re.sub(r"\n{2,}", "\n", _READ.sub("", block)).strip() items.append({"summary": text, "title": link.group(1).strip(), "url": link.group(2).strip()}) + sections.append((name, items)) return title, summary, sections +def newsletter_sources(markdown: str) -> list[str]: + """Return the ordered, de-duplicated source URLs cited in the newsletter (its `Read` links).""" + sources: list[str] = [] + + for _title, url in _READ.findall(markdown): + url = url.strip() + + if url and url not in sources: + sources.append(url) + + return sources + + @lru_cache(maxsize=1) def _template() -> str: """Load the tokenized HTML template generated by email-playground.""" @@ -99,6 +119,7 @@ def _render_item(item_inner: str, item: dict, *, is_first: bool) -> str: rendered = rendered.replace(itemsep_block, "" if is_first else itemsep_inner) readlink_block, readlink_inner = _region(rendered, "readlink") + if item["url"]: cta = _html.escape(item["title"] or "the source") url = _html.escape(item["url"], quote=True) @@ -127,7 +148,7 @@ def _render_html( summary: str, sections: list[tuple[str, list[dict]]], footer_note: str, - subject_symbol: str | None, + ticker: str | None, unsubscribe_url: str | None, ) -> str: template = _template() @@ -138,9 +159,11 @@ def _render_html( unsubscribe_block, unsubscribe_inner = _region(template, "unsubscribe") rendered_sections: list[str] = [] + for name, items in sections: if not items: continue + section_template = quickhits_template if name == _QUICK_HITS else standard_template rendered_sections.append(_render_section(section_template, name, items, is_first=not rendered_sections)) @@ -153,7 +176,7 @@ def _render_html( if unsubscribe_url: url = _html.escape(unsubscribe_url, quote=True) - label = _html.escape(subject_symbol or "these") + label = _html.escape(ticker or "these") filled = unsubscribe_inner.replace("{{unsubscribe_url}}", url).replace("{{symbol}}", label) html = html.replace(unsubscribe_block, filled) else: @@ -168,32 +191,39 @@ def _render_html( def render_newsletter_email( markdown: str, *, - subject_symbol: str | None = None, + ticker: str | None = None, unsubscribe_url: str | None = None, ) -> dict[str, str]: """Return ``{"subject", "html", "text"}`` for the newsletter markdown.""" title, summary, sections = _parse(markdown) - subject = (subject_symbol or "").strip() + ticker = (ticker or "").strip() footer_note = ( - f"You are receiving this because you subscribed to {subject} updates." - if subject + f"You are receiving this because you subscribed to {ticker} updates." + if ticker else "You are receiving this because you subscribed to updates." ) - html = _render_html(title, summary, sections, footer_note, subject_symbol, unsubscribe_url) + html = _render_html(title, summary, sections, footer_note, ticker, unsubscribe_url) text_lines = [title, "", summary, ""] if summary else [title, ""] + for name, items in sections: if not items: continue + text_lines.append(name) + for item in items: text_lines.append(item["summary"]) + if item["url"]: text_lines.append(f"Read {item['title']}: {item['url']}") + text_lines.append("---") + text_lines.append("") + text_lines.append(footer_note) return {"subject": title, "html": html, "text": "\n".join(text_lines).strip()} diff --git a/src/utils/guardrails.py b/src/utils/guardrails.py index 6a71dda..2d5479e 100644 --- a/src/utils/guardrails.py +++ b/src/utils/guardrails.py @@ -19,6 +19,7 @@ class SubjectGuardrail(AgentMiddleware): async def process(self, context: AgentContext, call_next: Callable[[], Awaitable[None]]) -> None: last = context.messages[-1] if context.messages else None + if not (last and last.text and last.text.strip()): context.result = AgentResponse( messages=[ @@ -43,8 +44,10 @@ def __init__(self) -> None: def _result_text(result: object) -> str: if result is None: return "" + if isinstance(result, str): return result + if isinstance(result, (list, tuple)): return " ".join(_result_text(item) for item in result) @@ -70,6 +73,7 @@ def __init__(self, registry: SourceRegistry) -> None: async def process(self, context: AgentContext, call_next: Callable[[], Awaitable[None]]) -> None: await call_next() + if context.stream or context.result is None or not self.registry.urls: return @@ -77,10 +81,15 @@ async def process(self, context: AgentContext, call_next: Callable[[], Awaitable for content in message.contents: if getattr(content, "type", None) != "text" or not content.text: continue + kept = [] + for line in content.text.splitlines(): urls = _URL.findall(line) + if urls and not all(url in self.registry.urls for url in urls): continue + kept.append(line) + content.text = "\n".join(kept) diff --git a/src/utils/memory.py b/src/utils/memory.py deleted file mode 100644 index b57e55a..0000000 --- a/src/utils/memory.py +++ /dev/null @@ -1,45 +0,0 @@ -import os -from typing import Any - -import redis -from agent_framework import AgentSession, ContextProvider, SessionContext - -_PREFIX = "mediapulse:subject:" -_TTL = int(os.getenv("SUBJECT_TTL", "604800")) # briefs expire after 7 days by default; 0 = never -_client = redis.from_url(os.getenv("REDIS_URL", "redis://localhost:6379/0"), decode_responses=True) - - -def _key(subject: str) -> str: - return _PREFIX + (subject or "").strip().lower() - - -def remember_subject(subject: str, brief: str) -> None: - """Persist a resolved brief for a subject to Redis (best-effort; no-op if Redis is down).""" - try: - _client.set(_key(subject), brief, ex=_TTL or None) - except redis.RedisError: - pass - - -class SubjectMemoryProvider(ContextProvider): - """Recall a previously resolved brief for the subject from Redis.""" - - SOURCE_ID = "subject_memory" - - def __init__(self) -> None: - super().__init__(self.SOURCE_ID) - - async def before_run( - self, *, agent: Any, session: AgentSession | None, context: SessionContext, state: dict[str, Any] - ) -> None: - text = context.input_messages[-1].text if context.input_messages else "" - try: - brief = _client.get(_key(text)) - except redis.RedisError: - brief = None - if brief: - context.extend_instructions( - self.source_id, - f"A previously resolved brief for '{text.strip()}' is below. Verify it is still current, " - f"then update only what changed:\n{brief}", - ) diff --git a/tests/conftest.py b/tests/conftest.py index 17cd4ef..636662b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,4 +12,3 @@ os.environ.setdefault("OPENAI_BASE_URL", "https://example.invalid/v1") os.environ.setdefault("SERPER_API_KEY", "test-serper-key") os.environ.setdefault("RESEND_API_KEY", "test-resend-key") -os.environ.setdefault("REDIS_URL", "redis://localhost:6379/0") diff --git a/tests/test_campaign.py b/tests/test_campaign.py index 943ec0b..b50f8e8 100644 --- a/tests/test_campaign.py +++ b/tests/test_campaign.py @@ -2,10 +2,11 @@ def _patch_pipeline(monkeypatch, *, fail_for=()): - async def fake_run_newsletter(symbol): - if symbol in fail_for: - raise RuntimeError(f"boom {symbol}") - return f"# {symbol} markdown" + async def fake_run_newsletter(ticker): + if ticker in fail_for: + raise RuntimeError(f"boom {ticker}") + + return f"# {ticker} markdown" sent = [] @@ -21,9 +22,9 @@ async def fake_run_newsletter(symbol): def _subs(): return [ - {"email": "a@x.com", "symbol": "ACME"}, - {"email": "b@x.com", "symbol": "ACME"}, - {"email": "c@x.com", "symbol": "GLOBEX"}, + {"email": "a@x.com", "ticker": "ACME"}, + {"email": "b@x.com", "ticker": "ACME"}, + {"email": "c@x.com", "ticker": "GLOBEX"}, ] @@ -52,15 +53,33 @@ async def test_failed_ticker_is_skipped_others_delivered(monkeypatch): _patch_pipeline(monkeypatch, fail_for={"ACME"}) result = await campaign.run_campaign(subscriptions=_subs(), send=False, log=lambda *a: None) - delivered_symbols = {entry["symbol"] for entry in result["delivered"]} - assert delivered_symbols == {"GLOBEX"} + delivered_tickers = {entry["ticker"] for entry in result["delivered"]} + assert delivered_tickers == {"GLOBEX"} async def test_run_campaign_fetches_subscriptions_when_unset(monkeypatch): _patch_pipeline(monkeypatch) - monkeypatch.setattr(campaign, "fetch_subscriptions", lambda: [{"email": "z@x.com", "symbol": "ACME"}]) + monkeypatch.setattr(campaign, "fetch_subscriptions", lambda: [{"email": "z@x.com", "ticker": "ACME"}]) result = await campaign.run_campaign(send=False, log=lambda *a: None) assert result["tickers"] == ["ACME"] - assert result["delivered"] == [{"email": "z@x.com", "symbol": "ACME"}] + assert result["delivered"] == [{"email": "z@x.com", "ticker": "ACME"}] + + +async def test_each_ticker_is_saved_once_with_ticker_and_sources(monkeypatch): + _patch_pipeline(monkeypatch) + saved = [] + + def fake_save(subject, markdown, metadata): + saved.append((subject, metadata)) + + monkeypatch.setattr(campaign, "save_newsletter", fake_save) + + await campaign.run_campaign(subscriptions=_subs(), send=False, log=lambda *a: None) + + assert sorted(subject for subject, _meta in saved) == ["ACME", "GLOBEX"] # once per unique ticker + + for _subject, metadata in saved: + assert metadata["ticker"] in {"ACME", "GLOBEX"} + assert isinstance(metadata["sources"], list) diff --git a/tests/test_email_template.py b/tests/test_email_template.py index 705c66c..8eabc07 100644 --- a/tests/test_email_template.py +++ b/tests/test_email_template.py @@ -1,4 +1,4 @@ -import utils.email_template as email_template +import emails.templates.newsletter as email_template _MARKDOWN = ( "# ACME Pulse: Big Week\n\nA strong week for ACME.\n\n" @@ -38,7 +38,7 @@ def test_parse_defaults_title_when_missing(): def test_render_returns_subject_html_and_text(): - out = email_template.render_newsletter_email(_MARKDOWN, subject_symbol="ACME") + out = email_template.render_newsletter_email(_MARKDOWN, ticker="ACME") assert out["subject"] == "ACME Pulse: Big Week" assert 'href="https://x.com/acme-funding-news"' in out["html"] assert "Read: Funding" in out["html"] @@ -52,9 +52,7 @@ def test_render_skips_sections_without_items(): def test_render_includes_unsubscribe_link_when_url_given(): - out = email_template.render_newsletter_email( - _MARKDOWN, subject_symbol="ACME", unsubscribe_url="https://x.com/unsub" - ) + out = email_template.render_newsletter_email(_MARKDOWN, ticker="ACME", unsubscribe_url="https://x.com/unsub") assert 'href="https://x.com/unsub"' in out["html"] assert "Unsubscribe from ACME updates" in out["html"] @@ -62,3 +60,21 @@ def test_render_includes_unsubscribe_link_when_url_given(): def test_render_uses_generic_footer_without_subject(): out = email_template.render_newsletter_email(_MARKDOWN) assert "subscribed to updates" in out["text"] + + +def test_newsletter_sources_extracts_cited_urls(): + assert email_template.newsletter_sources(_MARKDOWN) == ["https://x.com/acme-funding-news"] + + +def test_newsletter_sources_dedupes_and_preserves_order(): + markdown = ( + "# T\n\n## A\n" + "one [Read: a](https://x.com/1)\n\n---\n" + "two [Read: b](https://x.com/2)\n\n---\n" + "three [Read: c](https://x.com/1)\n" + ) + assert email_template.newsletter_sources(markdown) == ["https://x.com/1", "https://x.com/2"] + + +def test_newsletter_sources_empty_without_links(): + assert email_template.newsletter_sources("# Title\n\nNo links here.") == [] diff --git a/tests/test_mailer.py b/tests/test_mailer.py index c0d3a18..14e0c87 100644 --- a/tests/test_mailer.py +++ b/tests/test_mailer.py @@ -1,4 +1,4 @@ -import utils.mailer as mailer +import emails.mailer as mailer class FakeResponse: diff --git a/tests/test_db.py b/tests/test_mediapulse.py similarity index 69% rename from tests/test_db.py rename to tests/test_mediapulse.py index 0e3a0d5..f14442c 100644 --- a/tests/test_db.py +++ b/tests/test_mediapulse.py @@ -1,4 +1,4 @@ -import utils.db as db +import db.mediapulse as mediapulse class FakeCursor: @@ -36,28 +36,28 @@ def cursor(self): def test_conninfo_drops_prisma_schema_query(monkeypatch): monkeypatch.setenv("MEDIAPULSE_DATABASE_URL", "postgres://u:p@h/db?schema=mediapulse") - assert db._conninfo() == "postgres://u:p@h/db" + assert mediapulse._conninfo() == "postgres://u:p@h/db" def test_clean_collapses_whitespace_and_trims_punctuation(): - assert db._clean(" multi\nline\r value ,- ") == "multi line value" - assert db._clean(None) == "" + assert mediapulse._clean(" multi\nline\r value ,- ") == "multi line value" + assert mediapulse._clean(None) == "" def test_fetch_ticker_profile_none_without_env(monkeypatch): monkeypatch.delenv("MEDIAPULSE_DATABASE_URL", raising=False) - assert db.fetch_ticker_profile("ACME") is None + assert mediapulse.fetch_ticker_profile("ACME") is None -def test_fetch_ticker_profile_none_for_blank_symbol(monkeypatch): +def test_fetch_ticker_profile_none_for_blank_ticker(monkeypatch): monkeypatch.setenv("MEDIAPULSE_DATABASE_URL", "postgres://u:p@h/db") - assert db.fetch_ticker_profile(" ") is None + assert mediapulse.fetch_ticker_profile(" ") is None -def test_fetch_ticker_profile_none_when_symbol_not_listed(monkeypatch): +def test_fetch_ticker_profile_none_when_ticker_not_listed(monkeypatch): monkeypatch.setenv("MEDIAPULSE_DATABASE_URL", "postgres://u:p@h/db") - monkeypatch.setattr(db.psycopg, "connect", lambda *a, **k: FakeConn(None)) - assert db.fetch_ticker_profile("ACME") is None + monkeypatch.setattr(mediapulse.psycopg, "connect", lambda *a, **k: FakeConn(None)) + assert mediapulse.fetch_ticker_profile("ACME") is None def test_fetch_ticker_profile_shapes_metadata(monkeypatch): @@ -70,9 +70,9 @@ def test_fetch_ticker_profile_shapes_metadata(monkeypatch): "TanggalPencatatan": "2000-05-31T00:00:00.000Z", "Website": "https://example.com", } - monkeypatch.setattr(db.psycopg, "connect", lambda *a, **k: FakeConn(("Acme Sample Corp", metadata))) + monkeypatch.setattr(mediapulse.psycopg, "connect", lambda *a, **k: FakeConn(("Acme Sample Corp", metadata))) - profile = db.fetch_ticker_profile("acme") + profile = mediapulse.fetch_ticker_profile("acme") assert profile["Company"] == "Acme Sample Corp" assert profile["Sector"] == "Technology" assert "Sub-sector" not in profile # duplicate "Technology" value dropped @@ -84,6 +84,6 @@ def test_fetch_ticker_profile_shapes_metadata(monkeypatch): def test_fetch_ticker_profile_handles_missing_metadata(monkeypatch): monkeypatch.setenv("MEDIAPULSE_DATABASE_URL", "postgres://u:p@h/db") - monkeypatch.setattr(db.psycopg, "connect", lambda *a, **k: FakeConn(("Solo Name", None))) - profile = db.fetch_ticker_profile("SOLO") + monkeypatch.setattr(mediapulse.psycopg, "connect", lambda *a, **k: FakeConn(("Solo Name", None))) + profile = mediapulse.fetch_ticker_profile("SOLO") assert profile == {"Company": "Solo Name"} diff --git a/tests/test_memory.py b/tests/test_memory.py index 188af71..7c48cbd 100644 --- a/tests/test_memory.py +++ b/tests/test_memory.py @@ -1,77 +1,55 @@ -import utils.memory as memory +from sqlalchemy.pool import StaticPool +from sqlmodel import Session, SQLModel, create_engine, select +import db.engine as engine_module +import db.memory as memory -class Msg: - def __init__(self, text): - self.text = text +def _use_sqlite(monkeypatch): + engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool) + SQLModel.metadata.create_all(engine, tables=[memory.SubjectMemory.__table__]) + monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@h/db") + monkeypatch.setattr(engine_module, "_engine", engine) -class MemoryContext: - def __init__(self, text): - self.input_messages = [Msg(text)] - self.added = [] - - def extend_instructions(self, source_id, text): - self.added.append((source_id, text)) + return engine def test_key_normalizes_subject(): - assert memory._key(" HELLO ") == "mediapulse:subject:hello" - assert memory._key("") == "mediapulse:subject:" - - -def test_remember_subject_writes_with_ttl(monkeypatch): - captured = {} - - def fake_set(key, value, ex): - captured.update(key=key, value=value, ex=ex) - - monkeypatch.setattr(memory._client, "set", fake_set) - monkeypatch.setattr(memory, "_TTL", 1000) - memory.remember_subject("ACME", "brief text") - assert captured["key"] == "mediapulse:subject:acme" - assert captured["value"] == "brief text" - assert captured["ex"] == 1000 - + assert memory._key(" HELLO ") == "hello" + assert memory._key("") == "" -def test_remember_subject_swallows_redis_errors(monkeypatch): - def boom(*args, **kwargs): - raise memory.redis.RedisError("down") - monkeypatch.setattr(memory._client, "set", boom) - memory.remember_subject("ACME", "brief") # must not raise +def test_remember_subject_skips_without_env(monkeypatch): + monkeypatch.delenv("DATABASE_URL", raising=False) + memory.remember_subject("ACME", "brief") # no-op, must not raise -async def test_memory_provider_injects_recalled_brief(monkeypatch): - monkeypatch.setattr(memory._client, "get", lambda key: "previous brief") - provider = memory.SubjectMemoryProvider() - context = MemoryContext("ACME") +def test_recall_subject_skips_without_env(monkeypatch): + monkeypatch.delenv("DATABASE_URL", raising=False) + assert memory.recall_subject("ACME") is None - await provider.before_run(agent=None, session=None, context=context, state={}) - assert len(context.added) == 1 - source_id, text = context.added[0] - assert "previous brief" in text +def test_remember_and_recall_roundtrip(monkeypatch): + _use_sqlite(monkeypatch) + memory.remember_subject("ACME", "first brief") + assert memory.recall_subject("acme") == "first brief" # recall normalizes the key -async def test_memory_provider_noop_when_nothing_recalled(monkeypatch): - monkeypatch.setattr(memory._client, "get", lambda key: None) - provider = memory.SubjectMemoryProvider() - context = MemoryContext("ACME") - await provider.before_run(agent=None, session=None, context=context, state={}) +def test_remember_subject_upserts_one_row(monkeypatch): + engine = _use_sqlite(monkeypatch) + memory.remember_subject("ACME", "first") + memory.remember_subject("acme", "second") # same normalized key - assert context.added == [] + assert memory.recall_subject("ACME") == "second" + with Session(engine) as session: + rows = session.exec(select(memory.SubjectMemory)).all() -async def test_memory_provider_survives_redis_error(monkeypatch): - def boom(key): - raise memory.redis.RedisError("down") + assert len(rows) == 1 - monkeypatch.setattr(memory._client, "get", boom) - provider = memory.SubjectMemoryProvider() - context = MemoryContext("ACME") - await provider.before_run(agent=None, session=None, context=context, state={}) +def test_recall_unknown_subject_returns_none(monkeypatch): + _use_sqlite(monkeypatch) - assert context.added == [] + assert memory.recall_subject("NOPE") is None diff --git a/tests/test_newsletters.py b/tests/test_newsletters.py new file mode 100644 index 0000000..e14ee4b --- /dev/null +++ b/tests/test_newsletters.py @@ -0,0 +1,47 @@ +from sqlalchemy.pool import StaticPool +from sqlmodel import Session, SQLModel, create_engine, select + +import db.engine as engine_module +import db.newsletters as newsletters + + +def _use_sqlite(monkeypatch): + engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool) + SQLModel.metadata.create_all(engine, tables=[newsletters.Newsletter.__table__]) + monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@h/db") + monkeypatch.setattr(engine_module, "_engine", engine) + + return engine + + +def test_save_newsletter_skips_without_env(monkeypatch): + monkeypatch.delenv("DATABASE_URL", raising=False) + + assert newsletters.save_newsletter("ACME", "# ACME", {"ticker": "ACME"}) is None + + +def test_save_newsletter_inserts_and_roundtrips(monkeypatch): + engine = _use_sqlite(monkeypatch) + metadata = {"ticker": "ACME", "sources": ["https://x.com/a", "https://x.com/b"]} + row_id = newsletters.save_newsletter("ACME", "# ACME Pulse: Big Week", metadata) + + assert isinstance(row_id, int) + + with Session(engine) as session: + rows = session.exec(select(newsletters.Newsletter)).all() + + assert len(rows) == 1 + assert rows[0].subject == "ACME" + assert rows[0].content.startswith("# ACME Pulse") + assert rows[0].meta == metadata + + +def test_save_newsletter_swallows_errors(monkeypatch): + monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@h/db") + + def boom(): + raise RuntimeError("db down") + + monkeypatch.setattr(newsletters, "get_engine", boom) + + assert newsletters.save_newsletter("ACME", "content", {}) is None diff --git a/tests/test_providers.py b/tests/test_providers.py new file mode 100644 index 0000000..858248a --- /dev/null +++ b/tests/test_providers.py @@ -0,0 +1,88 @@ +import agents.providers.memory as memory_provider +import agents.providers.ticker as ticker_provider + + +class Msg: + def __init__(self, text): + self.text = text + + +class FakeContext: + def __init__(self, text): + self.input_messages = [Msg(text)] + self.added = [] + + def extend_instructions(self, source_id, text): + self.added.append((source_id, text)) + + +async def test_subject_memory_provider_injects_recalled_brief(monkeypatch): + monkeypatch.setattr(memory_provider, "recall_subject", lambda subject: "previous brief") + provider = memory_provider.SubjectMemoryProvider() + context = FakeContext("ACME") + + await provider.before_run(agent=None, session=None, context=context, state={}) + + assert len(context.added) == 1 + _source_id, text = context.added[0] + assert "previous brief" in text + + +async def test_subject_memory_provider_noop_when_nothing_recalled(monkeypatch): + monkeypatch.setattr(memory_provider, "recall_subject", lambda subject: None) + provider = memory_provider.SubjectMemoryProvider() + context = FakeContext("ACME") + + await provider.before_run(agent=None, session=None, context=context, state={}) + + assert context.added == [] + + +async def test_ticker_provider_injects_profile_details(monkeypatch): + monkeypatch.setattr( + ticker_provider, + "fetch_ticker_profile", + lambda ticker: {"Company": "Acme Sample Corp", "Sector": "Technology"}, + ) + provider = ticker_provider.TickerProfileProvider() + context = FakeContext("ACME") + + await provider.before_run(agent=None, session=None, context=context, state={}) + + assert len(context.added) == 1 + _source_id, text = context.added[0] + assert "Acme Sample Corp" in text + assert "- Sector: Technology" in text + + +async def test_ticker_provider_noop_for_blank_subject(monkeypatch): + monkeypatch.setattr(ticker_provider, "fetch_ticker_profile", lambda ticker: {"Company": "X"}) + provider = ticker_provider.TickerProfileProvider() + context = FakeContext(" ") + + await provider.before_run(agent=None, session=None, context=context, state={}) + + assert context.added == [] + + +async def test_ticker_provider_noop_when_profile_missing(monkeypatch): + monkeypatch.setattr(ticker_provider, "fetch_ticker_profile", lambda ticker: None) + provider = ticker_provider.TickerProfileProvider() + context = FakeContext("ACME") + + await provider.before_run(agent=None, session=None, context=context, state={}) + + assert context.added == [] + + +async def test_ticker_provider_swallows_lookup_errors(monkeypatch): + def boom(ticker): + raise RuntimeError("db down") + + monkeypatch.setattr(ticker_provider, "fetch_ticker_profile", boom) + provider = ticker_provider.TickerProfileProvider() + context = FakeContext("ACME") + + await provider.before_run(agent=None, session=None, context=context, state={}) + + assert context.added == [] diff --git a/tests/test_ticker.py b/tests/test_ticker.py deleted file mode 100644 index 0fbbbdf..0000000 --- a/tests/test_ticker.py +++ /dev/null @@ -1,63 +0,0 @@ -import utils.ticker as ticker - - -class Msg: - def __init__(self, text): - self.text = text - - -class TickerContext: - def __init__(self, text): - self.input_messages = [Msg(text)] - self.added = [] - - def extend_instructions(self, source_id, text): - self.added.append((source_id, text)) - - -async def test_provider_injects_profile_details(monkeypatch): - monkeypatch.setattr( - ticker, "fetch_ticker_profile", lambda symbol: {"Company": "Acme Sample Corp", "Sector": "Technology"} - ) - provider = ticker.TickerProfileProvider() - context = TickerContext("ACME") - - await provider.before_run(agent=None, session=None, context=context, state={}) - - assert len(context.added) == 1 - _, text = context.added[0] - assert "Acme Sample Corp" in text - assert "- Sector: Technology" in text - - -async def test_provider_noop_for_blank_subject(monkeypatch): - monkeypatch.setattr(ticker, "fetch_ticker_profile", lambda symbol: {"Company": "X"}) - provider = ticker.TickerProfileProvider() - context = TickerContext(" ") - - await provider.before_run(agent=None, session=None, context=context, state={}) - - assert context.added == [] - - -async def test_provider_noop_when_profile_missing(monkeypatch): - monkeypatch.setattr(ticker, "fetch_ticker_profile", lambda symbol: None) - provider = ticker.TickerProfileProvider() - context = TickerContext("ACME") - - await provider.before_run(agent=None, session=None, context=context, state={}) - - assert context.added == [] - - -async def test_provider_swallows_lookup_errors(monkeypatch): - def boom(symbol): - raise RuntimeError("db down") - - monkeypatch.setattr(ticker, "fetch_ticker_profile", boom) - provider = ticker.TickerProfileProvider() - context = TickerContext("ACME") - - await provider.before_run(agent=None, session=None, context=context, state={}) - - assert context.added == []