Skip to content

feat(export): consolidate HTTP and CLI bulk export behind shared engine #54

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
wpak-ai merged 5 commits into from
May 27, 2026
Merged

feat(export): consolidate HTTP and CLI bulk export behind shared engine #54

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2ce1e74
feat(export): consolidate HTTP and CLI bulk export behind shared engine
clean6378-max-it May 26, 2026
832f56e
fix(export): address PR review — parity asserts, stderr warnings, JSO…
clean6378-max-it May 27, 2026
9b7a9f0
fix(export): address CodeRabbit review (engine hardening + manifest p…
clean6378-max-it May 27, 2026
20cffe0
fix(export): restore incremental mtime diagnostic and batch parse res…
clean6378-max-it May 27, 2026
54ecd06
fix(export): ZipSink deferred writes, mode guards, UTC/POSIX paths
clean6378-max-it May 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
158 changes: 24 additions & 134 deletions api/export_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,31 +18,21 @@
export_state_lock,
load_export_state_from_disk,
)
from utils.session_path import (
get_claude_projects_dir,
list_projects,
list_sessions,
)
from utils.session_path import get_claude_projects_dir, list_projects
from utils.jsonl_parser import parse_session
from utils.exclusion_rules import is_session_excluded
from utils.session_stats import compute_stats
from utils.md_exporter import session_to_markdown
from utils.json_exporter import session_to_json
from utils.exclusion_rules import is_session_excluded
from utils.slugify import slugify
from utils.export_day_filter import collect_sessions_for_latest_activity_day
from utils.export_engine import EXPORT_ERRORS, ZipSink, run_bulk_export

export_bp = Blueprint("export", __name__)

# Tests monkeypatch this path; keep in sync with utils.export_state_store.
_STATE_FILE = EXPORT_STATE_FILE

_EXPORT_ERRORS = (
json.JSONDecodeError,
KeyError,
ValueError,
OSError,
FileNotFoundError,
)
_EXPORT_ERRORS = EXPORT_ERRORS


def _state_lock() -> Any:
Expand Down Expand Up @@ -120,128 +110,28 @@ def bulk_export() -> FlaskReturn:
)

buf = io.BytesIO()
count = 0
manifest: list[dict[str, Any]] = []
new_sessions_map: dict[str, float] = {}
latest_day = None

def _on_export_error(sid: str, exc: Exception) -> None:
current_app.logger.warning(
"Failed to export %s: %s", sid[:10], exc
)

with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
if since == "last":
d, rows, _n = collect_sessions_for_latest_activity_day(
projects,
list_sessions=list_sessions,
parse_session=parse_session,
is_session_excluded=is_session_excluded,
rules=rules,
)
latest_day = d
for project, sess_info, session, _st, _en in rows:
sid = sess_info["id"]
try:
stats = compute_stats(session)
md = session_to_markdown(session, stats)
title_slug = slugify(session["title"], default="session")
short_id = sid[:8]
proj_slug = slugify(project["name"], default="project")
ts = session["metadata"].get("first_timestamp", "")
ts_file = (
ts[:19].replace(":", "-")
if ts
else "0000-00-00T00-00-00"
)
rel_path = (
f"{proj_slug}/{ts_file}__{title_slug}__{short_id}.md"
)
zf.writestr(rel_path, md)
manifest.append(
{
"session_id": sid,
"title": session["title"],
"project": project["name"],
"tokens": session["metadata"]["total_input_tokens"]
+ session["metadata"]["total_output_tokens"],
"tool_calls": session["metadata"][
"total_tool_calls"
],
"cost_estimate_usd": stats.get(
"cost_estimate_usd"
),
}
)
new_sessions_map[sid] = sess_info.get("modified", 0)
count += 1
except _EXPORT_ERRORS as e:
current_app.logger.warning(
"Failed to export %s: %s", sid[:10], e
)
continue
else:
for project in projects:
sessions = list_sessions(project["path"])
for sess_info in sessions:
sid = sess_info["id"]
try:
if since == "incremental":
prev_mtime = last_export_sessions.get(sid, 0)
curr_mtime = sess_info.get("modified", 0)
if curr_mtime and curr_mtime <= prev_mtime:
continue

session = parse_session(sess_info["path"])
if session["title"] == "Untitled Session":
continue

if is_session_excluded(
rules,
session,
project.get("display_name") or project["name"],
):
continue

stats = compute_stats(session)
md = session_to_markdown(session, stats)
title_slug = slugify(
session["title"], default="session"
)
short_id = sid[:8]
proj_slug = slugify(project["name"], default="project")
ts = session["metadata"].get("first_timestamp", "")
ts_file = (
ts[:19].replace(":", "-")
if ts
else "0000-00-00T00-00-00"
)
rel_path = f"{proj_slug}/{ts_file}__{title_slug}__{short_id}.md"
zf.writestr(rel_path, md)
manifest.append(
{
"session_id": sid,
"title": session["title"],
"project": project["name"],
"tokens": session["metadata"][
"total_input_tokens"
]
+ session["metadata"]["total_output_tokens"],
"tool_calls": session["metadata"][
"total_tool_calls"
],
"cost_estimate_usd": stats.get(
"cost_estimate_usd"
),
}
)
new_sessions_map[sid] = sess_info.get("modified", 0)
count += 1
except _EXPORT_ERRORS as e:
current_app.logger.warning(
"Failed to export %s: %s", sid[:10], e
)
continue
if manifest:
manifest_str = "\n".join(
json.dumps(e, default=str) for e in manifest
)
zf.writestr("manifest.jsonl", manifest_str)
result = run_bulk_export(
projects=projects,
since=since,
rules=rules,
last_export_sessions=last_export_sessions,
sink=ZipSink(zf),
fmt="md",
path_layout="api",
manifest_style="api",
on_export_error=_on_export_error,
)

count = result.exported_session_count
new_sessions_map = result.new_sessions_map
latest_day = result.latest_day

if count == 0:
return error_response(
Expand Down
173 changes: 37 additions & 136 deletions scripts/export.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,9 @@
from utils.session_stats import compute_stats, _format_duration
from utils.md_exporter import session_to_markdown
from utils.json_exporter import session_to_json
from utils.exclusion_rules import (
resolve_exclusion_rules_path,
load_rules,
is_session_excluded,
)
from utils.exclusion_rules import resolve_exclusion_rules_path, load_rules
from utils.slugify import slugify
from utils.export_day_filter import collect_sessions_for_latest_activity_day
from utils.export_engine import ListSink, run_bulk_export
from utils.export_state_store import (
atomic_write_export_state,
export_state_lock,
Expand Down Expand Up @@ -389,66 +385,6 @@ def _aggregate_stats(base_dir: str, project_filter: str, fmt: str):
print(f" Est. cost: ~${totals['total_cost']:.2f} USD")


def _append_export_for_session(
project: dict,
sess_info: dict,
session: dict,
fmt: str,
all_exports: list,
manifest: list,
last_export: dict,
) -> None:
"""Append markdown/json entries and manifest row; update *last_export* mtime."""
sid = sess_info["id"]
stats = compute_stats(session)
meta = session["metadata"]
ts = meta.get("first_timestamp", "")
if not ts:
from datetime import datetime as _dt

ts = _dt.fromtimestamp(sess_info["modified"]).strftime(
"%Y-%m-%dT%H:%M:%S"
)
meta["first_timestamp"] = ts
date_str = ts[:10]
ts_file = ts[:19].replace(":", "-")
title_slug = slugify(session["title"], default="session")
short_id = sid[:8]
project_slug = slugify(project["name"], default="project")

if fmt in ("md", "both"):
md = session_to_markdown(session, stats)
rel_path = os.path.join(
date_str, project_slug, f"{ts_file}__{title_slug}__{short_id}.md"
)
all_exports.append((rel_path, md))

if fmt in ("json", "both"):
js = session_to_json(session, stats)
rel_path = os.path.join(
date_str, project_slug, f"{ts_file}__{title_slug}__{short_id}.json"
)
all_exports.append((rel_path, js))

manifest.append({
"session_id": sid,
"title": session["title"],
"project": project["name"],
"updated_at": meta.get("last_timestamp", ""),
"models": meta.get("models_used", []),
"tokens": meta["total_input_tokens"] + meta["total_output_tokens"],
"tool_calls": meta["total_tool_calls"],
"files_touched": stats.get("files_touched", {}).get(
"total_unique", 0
),
"commands_run": len(stats.get("commands_run", [])),
"cost_estimate_usd": stats.get("cost_estimate_usd"),
"wall_clock_seconds": meta.get("session_wall_time_seconds"),
})

last_export[sid] = sess_info["modified"]


def cmd_export(args):
"""The main export command. Writes md/json files, optionally zipped."""
base_dir = getattr(args, "base_dir", None) or get_claude_projects_dir()
Expand Down Expand Up @@ -488,86 +424,51 @@ def cmd_export(args):

print(f"Found {len(projects)} project(s) in {base_dir}")

all_exports = [] # list of (rel_path, content)
manifest = []
total_sessions = 0
skipped = 0
skipped_mtime_unchanged = 0
latest_day = None

def _on_export_error(sid: str, exc: Exception) -> None:
print(f" Warning: failed to export {sid}: {exc}")

collect_sink = ListSink()
export_result = run_bulk_export(
projects=projects,
since=since,
rules=rules,
last_export_sessions=last_export,
sink=collect_sink,
fmt=fmt,
path_layout="cli",
manifest_style="cli",
on_export_error=_on_export_error,
)

all_exports = export_result.exports
manifest = export_result.manifest
last_export.update(export_result.new_sessions_map)
total_sessions = export_result.total_candidates
skipped = export_result.skipped_count
latest_day = export_result.latest_day

if since == "last":
d, rows, total_sessions = collect_sessions_for_latest_activity_day(
projects,
list_sessions=list_sessions,
parse_session=parse_session,
is_session_excluded=is_session_excluded,
rules=rules,
)
if d is None:
if latest_day is None:
print("Nothing to export (no qualifying sessions in scope).")
return
latest_day = d
print(
f"Latest activity end-date (UTC): {d.isoformat()} — "
f"exporting sessions that overlap that calendar day."
f"Latest activity end-date (UTC): {latest_day.isoformat()} — "
"exporting sessions that overlap that calendar day."
)
if not rows:
if export_result.latest_day_match_count == 0:
print(
f"No sessions overlap {d.isoformat()} (UTC); nothing to export."
f"No sessions overlap {latest_day.isoformat()} (UTC); "
"nothing to export."
)
return
skipped = total_sessions - len(rows)
for project, sess_info, session, _st, _en in rows:
_append_export_for_session(
project,
sess_info,
session,
fmt,
all_exports,
manifest,
last_export,
)
else:
for project in projects:
sessions = list_sessions(project["path"])
for sess_info in sessions:
total_sessions += 1
sid = sess_info["id"]

if since == "incremental":
prev_mtime = last_export.get(sid, 0)
if sess_info["modified"] <= prev_mtime:
skipped += 1
skipped_mtime_unchanged += 1
continue

try:
session = parse_session(sess_info["path"])
except Exception as e:
print(f" Warning: failed to parse {sid}: {e}")
continue

if session["title"] == "Untitled Session":
skipped += 1
continue

if is_session_excluded(
rules,
session,
project.get("display_name") or project["name"],
):
skipped += 1
continue

_append_export_for_session(
project,
sess_info,
session,
fmt,
all_exports,
manifest,
last_export,
)
skipped = (
export_result.latest_day_scan_total
- export_result.latest_day_match_count
Comment thread
clean6378-max-it marked this conversation as resolved.
Outdated
)
elif since == "incremental":
skipped_mtime_unchanged = skipped
Comment thread
clean6378-max-it marked this conversation as resolved.
Outdated

exported = len(all_exports)
print(
Expand Down
Loading
Loading