Restructure src into db, emails, and agents/providers packages

.env.example

@@ -1,29 +1,29 @@
-# Leave OPENAI_BASE_URL empty for OpenAI direct, or point it at any OpenAI-compatible gateway.
-# OpenRouter (cheap access to Gemini, DeepSeek, Qwen, ...): https://openrouter.ai/api/v1
+# LLM gateway base URL. Leave empty for OpenAI direct, or point at any OpenAI-compatible gateway.
 OPENAI_BASE_URL=https://openrouter.ai/api/v1
 # An OpenAI key for direct use, or an OpenRouter key (sk-or-...) for the gateway above.
 OPENAI_API_KEY=your-openrouter-or-openai-key-here
+# Default model for every agent (override per role at the bottom of this file).
 OPENAI_MODEL=google/gemini-3-flash-preview
-SERPER_API_KEY=your-serper-api-key-here
 
-# Subject memory (Redis). Defaults to a local server; recall is skipped if it's unreachable.
-REDIS_URL=redis://localhost:6379/0
-SUBJECT_TTL=604800
+# Web search via serper.dev. Required: the agents research every section through it.
+SERPER_API_KEY=your-serper-api-key-here
 
-# Email delivery (Resend). Only needed when sending: python main.py GOTO you@example.com
+# Email delivery via Resend. Only needed when actually sending (python src/app.py run --send).
 RESEND_API_KEY=
+# Sender shown on outgoing newsletters.
 EMAIL_FROM=MediaPulse <onboarding@resend.dev>
 
-# Subscriber + ticker database (Postgres/Supabase, read-only). Used by the campaign job and to
-# enrich the analyst with each ticker's listing profile (sector, business, HQ). Skipped if unset.
+# Upstream MediaPulse Postgres (read-only): subscribers and each ticker's listing profile.
 MEDIAPULSE_DATABASE_URL=
 
-# API auth. Callers must send this as the `X-API-Key` header. Generate: python -c "import secrets;print(secrets.token_urlsafe(32))"
+# The app's own Postgres (read-write): archived newsletters and agent memory.
+DATABASE_URL=
+
+# API auth. Callers send this as the `X-API-Key` header. Generate one with:
+# python -c "import secrets; print(secrets.token_urlsafe(32))"
 SECRET_KEY=
 
-# Optional per-agent model overrides (fall back to OPENAI_MODEL):
-# All roles default to one model. To cut cost, set a cheaper model (e.g. google/gemini-3.1-flash-lite)
-# for the high-volume researcher and writer roles.
+# Optional per-agent model overrides (each falls back to OPENAI_MODEL).
 ANALYST_MODEL=google/gemini-3-flash-preview
 RESEARCHER_MODEL=google/gemini-3-flash-preview
 WRITER_MODEL=google/gemini-3-flash-preview

.gitignore

@@ -27,7 +27,7 @@ env/
 .subject_cache.json
 
 # Generated email templates (built from email-playground, fresh each deploy/CI run)
-/src/templates/*.html
+/src/emails/templates/*.html
 
 # Node (email-playground)
 node_modules/

CLAUDE.md

@@ -8,15 +8,17 @@ Guidance for working in this repository.
 
 ## Layout
 
-All application code lives under `src/`. Packages keep their top-level names (`agents`, `utils`), so imports are `from agents...` / `from utils...`, with `src` on the path (pytest sets `pythonpath = ["src"]`; the Docker image runs uvicorn with `--app-dir src`). Run entrypoints from the repo root so `load_dotenv()` finds `.env`.
+All application code lives under `src/`. Packages keep their top-level names (`agents`, `db`, `utils`), so imports are `from agents...` / `from db...` / `from utils...`, with `src` on the path (pytest sets `pythonpath = ["src"]`; the Docker image runs uvicorn with `--app-dir src`). Run entrypoints from the repo root so `load_dotenv()` finds `.env`.
 
 - `src/api.py` — FastAPI service. `POST /run` (full campaign) and `POST /test` (one user). Both run in the background, return `202`, and default to dry-run. Auth via the `X-API-Key` header matched against `SECRET_KEY`.
 - `src/app.py` — local CLI mirroring the API (`run`, `test`), for testing without HTTP.
 - `src/agents/orchestrator.py` — the pipeline: analyst → 5 parallel beat desks (researcher → writer → editor) → managing-editor gap roundtable → masthead → reviewer → deterministic clean/assemble/dedupe. Most non-agent logic (citation gating, URL/article validation, dedupe, subject-name canonicalization, prose humanizing) lives here.
-- `src/agents/` — one module per agent (`analyst`, `researcher`, `writer`, `editor`, `managing_editor`, `reviewer`), plus `beats.py` (beat desks), `campaign.py` (top-level run over subscriptions), and `tools/` (Serper search, web fetch).
+- `src/agents/` — one module per agent (`analyst`, `researcher`, `writer`, `editor`, `managing_editor`, `reviewer`), plus `beats.py` (beat desks), `campaign.py` (top-level run over subscriptions), `providers/` (subject-memory and ticker-profile context providers), and `tools/` (Serper search, web fetch).
 - `src/agents/skills/` — `SKILL.md` files that control agent behavior (`subject-profile`, `section-research`, `newsletter-format`). Prefer editing these over code when changing how agents research or write.
-- `src/utils/` — `db.py` (subscriptions/tickers from the MediaPulse Postgres), `memory.py`, `client.py`, `guardrails.py`, `sections.py`, `mailer.py`, `email_template.py`, `ticker.py`. `email_template.py` no longer hand-builds HTML: it parses the newsletter markdown and fills the tokenized template at `src/templates/newsletter.html`.
-- `email-playground/` — a standalone React Email (TypeScript) project that is the visual source-of-truth for MediaPulse email templates (the newsletter is the first one). `npm run build:templates` renders each template to a tokenized `src/templates/<name>.html` that the Python side consumes (`email_template.py` for the newsletter). Those HTML files are gitignored build artifacts, regenerated fresh in CI and the Docker build. Restyle emails there, not in Python. See `email-playground/README.md`.
+- `src/db/` — all database access. `mediapulse.py` reads subscriptions and ticker profiles from the upstream MediaPulse Postgres (`MEDIAPULSE_DATABASE_URL`, read-only, raw psycopg). The app's own Postgres (`DATABASE_URL`, SQLModel) uses `engine.py` for the shared engine, with each table's model alongside its operations in `newsletters.py` (archives each generated newsletter as markdown plus JSONB metadata) and `memory.py` (subject-brief agent memory). Tables are auto-created on first write.
+- `src/emails/` — everything email-related. `mailer.py` sends via Resend. `templates/` pairs each template's renderer with its tokenized HTML: `templates/newsletter.py` parses the newsletter markdown and fills `templates/newsletter.html` (the gitignored build artifact from `email-playground`).
+- `src/utils/` — `client.py`, `guardrails.py`, `sections.py`.
+- `email-playground/` — a standalone React Email (TypeScript) project that is the visual source-of-truth for MediaPulse email templates (the newsletter is the first one). `npm run build:templates` renders each template to a tokenized `src/emails/templates/<name>.html` that the Python side consumes (`emails/templates/newsletter.py` for the newsletter). Those HTML files are gitignored build artifacts, regenerated fresh in CI and the Docker build. Restyle emails there, not in Python. See `email-playground/README.md`.
 - `tests/` — pytest suite covering the deterministic logic (see Testing below).
 
 ## Setup
@@ -57,18 +59,18 @@ pytest                                  # whole suite
 conda run -n agentic-mediapulse pytest  # if pytest is not on PATH
 ```
 
-The email-template tests read `src/templates/newsletter.html`, which is gitignored and generated by the playground. Build it once before running the suite locally (CI does this in the `tests` job):
+The email-template tests read `src/emails/templates/newsletter.html`, which is gitignored and generated by the playground. Build it once before running the suite locally (CI does this in the `tests` job):
 
 ```
 cd email-playground && npm install && npm run build:templates
 ```
 
 Config is in `pyproject.toml` under `[tool.pytest.ini_options]`: `pythonpath = ["src"]`, `testpaths = ["tests"]`, `asyncio_mode = "auto"` (async tests need no decorator). CI runs `pytest` in a separate `tests` job after installing both requirement files.
 
-The suite covers the deterministic logic, not the LLM agents: orchestrator text/section helpers, the email template, Serper tools, the DB profile shaping, guardrail middleware, memory/ticker context providers, the mailer, the client model resolution, and the campaign delivery flow. Conventions for new tests:
+The suite covers the deterministic logic, not the LLM agents: orchestrator text/section helpers, the email template, Serper tools, the MediaPulse profile shaping, the newsletter store and subject-memory upsert, guardrail middleware, the subject-memory and ticker-profile context providers, the mailer, the client model resolution, and the campaign delivery flow. Conventions for new tests:
 
 - `tests/conftest.py` sets dummy credentials before collection, because importing the agent modules constructs the whole agent graph at import time.
-- Every external call (httpx, psycopg, redis, the LLM clients) is monkeypatched. The suite is fully offline. A test that reaches the network is a test bug.
+- Every external call (httpx, psycopg, the LLM clients) is monkeypatched, and the SQLModel store is exercised against in-memory SQLite. The suite is fully offline. A test that reaches the network is a test bug.
 - Use `ACME` (and other clearly-fictional placeholders) for ticker symbols and company names, never real tickers.
 - The `test-runner` subagent runs the suite and reports failures without editing.
 
@@ -81,9 +83,12 @@ The suite covers the deterministic logic, not the LLM agents: orchestrator text/
 - Python 3.11, async throughout (`asyncio.gather` for parallel beats).
 - Keep prose free of em dashes and semicolons in generated output (`_humanize` enforces this in the pipeline).
 - Deterministic gates and agent feedback loops are bounded with explicit retry counts (`RETRIES`, `DISCUSSION_ROUNDS`); preserve those bounds when editing the orchestrator.
+- Surround a multi-line block (`if`, `for`, `while`, `with`, `try`) with a blank line above and below whenever other code sits next to it in the same block. Also put a blank line above a `return` that has code before it.
+- Keep docstrings and comments to a single concise line, and drop comments that only restate what the code shows.
 
 ## External services
 
 - `SERPER_API_KEY` — web search.
-- `MEDIAPULSE_DATABASE_URL` — Postgres for subscriptions and ticker data; schema is defined in the upstream [MediaPulse](https://github.com/hyperjumptech/mediapulse) repo.
+- `MEDIAPULSE_DATABASE_URL` — read-only Postgres for subscriptions and ticker data; schema is defined in the upstream [MediaPulse](https://github.com/hyperjumptech/mediapulse) repo.
+- `DATABASE_URL` — the app's own read-write Postgres for archived newsletters and agent memory. Tables are auto-created on first use, separate from `MEDIAPULSE_DATABASE_URL`.
 - `SECRET_KEY` — API auth.

Dockerfile

@@ -18,7 +18,7 @@ COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
 COPY . .
-COPY --from=template-builder /build/src/templates/ src/templates/
+COPY --from=template-builder /build/src/emails/templates/ src/emails/templates/
 
 RUN useradd --no-create-home --shell /bin/false app \
     && chown -R app:app /app

README.md

@@ -31,8 +31,8 @@ A simpler, agentic version of [MediaPulse](https://github.com/hyperjumptech/medi
 | `OPENAI_BASE_URL`         | Leave blank for OpenAI, or point to an OpenAI-compatible gateway      | Yes          |
 | `OPENAI_MODEL`            | Default model for every agent (e.g. `gpt-4.1-mini`)                   | Yes          |
 | `SERPER_API_KEY`          | Serper key (serper.dev) for news search and page scraping             | Yes          |
-| `REDIS_URL`               | Subject-memory store — default `redis://localhost:6379/0`             | Yes          |
 | `MEDIAPULSE_DATABASE_URL` | Postgres connection string (read-only)                                | Yes          |
+| `DATABASE_URL`            | App Postgres (read-write): archived newsletters and agent memory      | No           |
 | `RESEND_API_KEY`          | Resend key for email delivery                                         | Yes          |
 | `EMAIL_FROM`              | Sender address, e.g. `MediaPulse <hello@example.com>`                 | Yes          |
 | `SECRET_KEY`              | Required on every API request (`X-API-Key` header)                    | Yes          |

email-playground/README.md

@@ -28,7 +28,7 @@ npm run build:templates
 ```
 
 For every `emails/*.tsx` that exports both a default component and a `templateProps` object, this
-renders the component in template mode and writes `../src/templates/<name>.html`: a tokenized HTML
+renders the component in template mode and writes `../src/emails/templates/<name>.html`: a tokenized HTML
 file with `{{placeholders}}` and `<!--#region-->…<!--/region-->` blocks that the Python side fills
 per message. The output is a gitignored build artifact, regenerated in CI and the Docker build.
 **The Python tests and the running app require these files**, so run it once after `npm install`.
@@ -41,7 +41,7 @@ per message. The output is a gitignored build artifact, regenerated in CI and th
    `<Region name="...">` (which emits `[[#name]]`/`[[/name]]` markers) and leave content as
    `{{token}}` placeholders.
 3. Export `templateProps`: the props that drive template-mode rendering.
-4. Run `npm run build:templates`, then fill `src/templates/<name>.html` from Python.
+4. Run `npm run build:templates`, then fill `src/emails/templates/<name>.html` from Python.
 
 Templates without `templateProps` are preview-only and are skipped by the build.
 

email-playground/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "type": "module",
-  "description": "React Email templates and preview playground for MediaPulse emails. Exports tokenized HTML templates into src/templates for the Python app to fill.",
+  "description": "React Email templates and preview playground for MediaPulse emails. Exports tokenized HTML templates into src/emails/templates for the Python app to fill.",
   "scripts": {
     "dev": "email dev",
     "export": "email export",

email-playground/scripts/build-templates.mts

@@ -6,7 +6,7 @@ import * as React from "react";
 
 const scriptDir = dirname(fileURLToPath(import.meta.url));
 const emailsDir = resolve(scriptDir, "../emails");
-const outputDir = resolve(scriptDir, "../../src/templates");
+const outputDir = resolve(scriptDir, "../../src/emails/templates");
 
 function toMarkers(html: string): string {
   return html.replace(/\[\[#(\w+)\]\]/g, "<!--#$1-->").replace(/\[\[\/(\w+)\]\]/g, "<!--/$1-->");

environment.yml

@@ -8,7 +8,7 @@ dependencies:
       - agent-framework
       - python-dotenv
       - httpx
-      - redis
       - psycopg[binary]
+      - sqlmodel
       - fastapi
       - uvicorn

requirements.txt

@@ -1,7 +1,7 @@
 agent-framework
 python-dotenv
 httpx
-redis
 psycopg[binary]
+sqlmodel
 fastapi
 uvicorn

src/agents/analyst.py

@@ -1,10 +1,10 @@
 from agent_framework import Agent
 
+from agents.providers.memory import SubjectMemoryProvider
+from agents.providers.ticker import TickerProfileProvider
 from agents.tools import search, web_fetch
 from utils.client import SKILLS, chat_client
 from utils.guardrails import SubjectGuardrail
-from utils.memory import SubjectMemoryProvider
-from utils.ticker import TickerProfileProvider
 
 analyst = Agent(
     name="analyst",

src/agents/campaign.py

@@ -2,29 +2,25 @@
 import time
 
 from agents.orchestrator import run_newsletter
-from utils.db import fetch_subscriptions
-from utils.email_template import render_newsletter_email
-from utils.mailer import send_email
+from db.mediapulse import fetch_subscriptions
+from db.newsletters import save_newsletter
+from emails.mailer import send_email
+from emails.templates.newsletter import newsletter_sources, render_newsletter_email
 
 CONCURRENCY = 3
-SEND_INTERVAL = 1.0  # seconds between emails — keeps us under Resend's rate limit (1/sec)
+SEND_INTERVAL = 1.0  # seconds between emails
 
 
 async def run_campaign(*, subscriptions: list[dict] | None = None, send: bool = False, log=print) -> dict:
-    """Generate one newsletter per followed ticker and deliver each the moment it is ready.
-
-    Tickers are generated at most CONCURRENCY at a time. As soon as a ticker's newsletter is
-    done it is sent to that ticker's subscribers and then dropped, so the whole batch is never
-    held in memory at once. Sends are globally rate-limited to one email per SEND_INTERVAL
-    seconds. `send=False` is a dry run: it generates and renders but does not email.
-    Pass `subscriptions` to scope the run (e.g. one user); defaults to all active subscribers.
-    """
+    """Generate one newsletter per followed ticker and deliver each the moment it is ready."""
     if subscriptions is None:
         subscriptions = fetch_subscriptions()
 
     subscribers_by_ticker: dict[str, list[dict]] = {}
+
     for subscription in subscriptions:
-        subscribers_by_ticker.setdefault(subscription["symbol"], []).append(subscription)
+        subscribers_by_ticker.setdefault(subscription["ticker"], []).append(subscription)
+
     tickers = sorted(subscribers_by_ticker)
     log(f"{len(subscriptions)} subscriptions, {len(tickers)} unique tickers, concurrency {CONCURRENCY}")
 
@@ -33,35 +29,44 @@ async def run_campaign(*, subscriptions: list[dict] | None = None, send: bool =
     last_send = 0.0
     delivered: list[dict] = []
 
-    async def deliver(symbol: str, recipients: list[dict], markdown: str) -> None:
+    async def deliver(ticker: str, recipients: list[dict], markdown: str) -> None:
         nonlocal last_send
-        email = render_newsletter_email(markdown, subject_symbol=symbol)
+        email = render_newsletter_email(markdown, ticker=ticker)
+        save_newsletter(ticker, markdown, {"ticker": ticker, "sources": newsletter_sources(markdown)})
+
         for subscription in recipients:
             if send:
                 # Serialize sends so the 1/sec limit holds even across concurrently finished tickers.
                 async with send_lock:
                     wait = SEND_INTERVAL - (time.monotonic() - last_send)
+
                     if wait > 0:
                         await asyncio.sleep(wait)
+
                     send_email(subscription["email"], email["subject"], email["html"], email["text"])
                     last_send = time.monotonic()
-                log(f"sent {symbol} -> {subscription['email']}")
+
+                log(f"sent {ticker} -> {subscription['email']}")
             else:
-                log(f"[dry-run] would send {symbol} -> {subscription['email']}: {email['subject']}")
-            delivered.append({"email": subscription["email"], "symbol": symbol})
+                log(f"[dry-run] would send {ticker} -> {subscription['email']}: {email['subject']}")
 
-    async def process(symbol: str, recipients: list[dict]) -> None:
+            delivered.append({"email": subscription["email"], "ticker": ticker})
+
+    async def process(ticker: str, recipients: list[dict]) -> None:
         async with generate_semaphore:
-            log(f"generating {symbol} ...")
+            log(f"generating {ticker} ...")
+
             try:
-                markdown = await run_newsletter(symbol)
+                markdown = await run_newsletter(ticker)
             except Exception as error:
-                log(f"failed {symbol}: {error}")
+                log(f"failed {ticker}: {error}")
+
                 return
-            log(f"done {symbol}")
-        # Generation slot is freed above; send now, then `markdown` falls out of scope and is released.
-        await deliver(symbol, recipients, markdown)
 
-    await asyncio.gather(*(process(symbol, recipients) for symbol, recipients in subscribers_by_ticker.items()))
+            log(f"done {ticker}")
+
+        await deliver(ticker, recipients, markdown)
+
+    await asyncio.gather(*(process(ticker, recipients) for ticker, recipients in subscribers_by_ticker.items()))
 
     return {"subscriptions": len(subscriptions), "tickers": tickers, "delivered": delivered, "sent": send}