3_Pipeline_Stats.py

"""Pipeline Statistics page. Reads JSONL logs directly (no API needed)."""
import json
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

import pandas as pd
import plotly.graph_objects as go
import streamlit as st

st.set_page_config(
    page_title="Pipeline Stats · OpenLens",
    page_icon=None,
    layout="wide",
)

with st.sidebar:
    st.page_link("app.py",                         label="Leaderboard")
    st.page_link("pages/1_Package_Detail.py",       label="Package Detail")
    st.page_link("pages/2_Compare.py",              label="Compare")
    st.page_link("pages/3_Pipeline_Stats.py",       label="Pipeline Stats")

LOGS_DIR = Path(__file__).resolve().parents[3] / "data" / "logs"

LOG_FILES = {
    "GitHub":         LOGS_DIR / "github_ingestor_structured.jsonl",
    "PyPI":           LOGS_DIR / "pypi_ingestor_structured.jsonl",
    "Stack Overflow": LOGS_DIR / "stackoverflow_ingestor_structured.jsonl",
}

SOURCE_COLORS = {
    "GitHub":         "#3498db",
    "PyPI":           "#e67e22",
    "Stack Overflow": "#9b59b6",
}

CHART_LAYOUT = dict(
    plot_bgcolor="rgba(0,0,0,0)",
    paper_bgcolor="rgba(0,0,0,0)",
    font=dict(color="#eee"),
    margin=dict(l=0, r=0, t=10, b=0),
    xaxis=dict(gridcolor="#333"),
    yaxis=dict(gridcolor="#333"),
)


@st.cache_data(ttl=300)
def load_logs() -> dict:
    result = {}
    for source, path in LOG_FILES.items():
        if not path.exists():
            result[source] = pd.DataFrame()
            continue
        records = []
        with path.open("r", encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if line:
                    try:
                        records.append(json.loads(line))
                    except json.JSONDecodeError:
                        continue
        df = pd.DataFrame(records)
        if not df.empty:
            df["timestamp"] = pd.to_datetime(df["timestamp"], utc=True)
            df["source"] = source
        result[source] = df
    return result


with st.spinner("Loading pipeline logs..."):
    logs = load_logs()

all_df = pd.concat([df for df in logs.values() if not df.empty], ignore_index=True)

st.title("Pipeline Statistics")
st.caption("Ingestion telemetry from structured JSONL logs.")
st.divider()

if all_df.empty:
    st.warning("No log data found. Run the ingestion scripts first to populate the logs.")
    st.stop()

run_completes = all_df[all_df["event"] == "run_complete"]
last_ts = all_df["timestamp"].max()
packages_tracked = all_df[all_df["event"] != "run_complete"]["package"].nunique()
error_count = int(run_completes["error"].sum()) if "error" in run_completes.columns else 0

c1, c2, c3, c4 = st.columns(4)
c1.metric("Total Ingestion Runs", len(run_completes))
c2.metric("Data Sources", len([s for s, df in logs.items() if not df.empty]))
c3.metric("Packages Tracked", packages_tracked)
c4.metric("Last Run", last_ts.strftime("%Y-%m-%d %H:%M UTC") if pd.notna(last_ts) else "—")

st.divider()

st.subheader("Ingestion run history")

if not run_completes.empty:
    fig_tl = go.Figure()
    for source, color in SOURCE_COLORS.items():
        src_rc = run_completes[run_completes["source"] == source]
        if src_rc.empty:
            continue
        hover_text = [
            f"{row['success']} packages OK  ·  {row['error']} errors"
            for _, row in src_rc.iterrows()
        ]
        fig_tl.add_trace(go.Scatter(
            x=src_rc["timestamp"],
            y=[source] * len(src_rc),
            mode="markers",
            marker=dict(size=16, color=color, symbol="circle"),
            name=source,
            text=hover_text,
            hovertemplate="%{x|%Y-%m-%d %H:%M}<br>%{text}<extra>%{fullData.name}</extra>",
        ))
    fig_tl.update_layout(
        height=180,
        **{k: v for k, v in CHART_LAYOUT.items() if k not in ("xaxis", "yaxis")},
        xaxis=dict(gridcolor="#333", type="date"),
        yaxis=dict(gridcolor="#333"),
        legend=dict(orientation="h", yanchor="bottom", y=1.1),
    )
    st.plotly_chart(fig_tl, width="stretch")

st.divider()

col_l, col_r = st.columns(2)

with col_l:
    st.subheader("GitHub — stars per package")
    gh_df = logs.get("GitHub", pd.DataFrame())
    if not gh_df.empty:
        # Keep most recent repo_metadata event per package
        repo_ev = (
            gh_df[gh_df["event"] == "repo_metadata"]
            .sort_values("timestamp", ascending=False)
            .drop_duplicates("package")
            .sort_values("stars", ascending=False)
        )
        if not repo_ev.empty:
            fig_gh = go.Figure(go.Bar(
                x=repo_ev["package"],
                y=repo_ev["stars"],
                marker_color=SOURCE_COLORS["GitHub"],
                text=repo_ev["stars"].apply(lambda v: f"{v:,.0f}"),
                textposition="outside",
            ))
            fig_gh.update_layout(height=300, yaxis_title="Stars", **CHART_LAYOUT)
            st.plotly_chart(fig_gh, width="stretch")
    else:
        st.info("No GitHub log data available.")

with col_r:
    st.subheader("PyPI — last-month downloads")
    pypi_df = logs.get("PyPI", pd.DataFrame())
    if not pypi_df.empty:
        dl_ev = (
            pypi_df[pypi_df["event"] == "downloads_recent"]
            .sort_values("timestamp", ascending=False)
            .drop_duplicates("package")
            .sort_values("last_month", ascending=False)
        )
        if not dl_ev.empty:
            fig_pypi = go.Figure(go.Bar(
                x=dl_ev["package"],
                y=dl_ev["last_month"],
                marker_color=SOURCE_COLORS["PyPI"],
                text=dl_ev["last_month"].apply(lambda v: f"{v / 1e6:.1f}M"),
                textposition="outside",
            ))
            fig_pypi.update_layout(
                height=300, yaxis_title="Downloads (last month)", **CHART_LAYOUT
            )
            st.plotly_chart(fig_pypi, width="stretch")
    else:
        st.info("No PyPI log data available.")

st.divider()

# ── Stack Overflow total questions ────────────────────────────────────────────
st.subheader("Stack Overflow — total questions per tag")
so_df = logs.get("Stack Overflow", pd.DataFrame())
if not so_df.empty:
    tag_ev = (
        so_df[so_df["event"] == "tag_info"]
        .sort_values("timestamp", ascending=False)
        .drop_duplicates("package")
        .sort_values("total_questions", ascending=False)
    )
    if not tag_ev.empty:
        fig_so = go.Figure(go.Bar(
            x=tag_ev["package"],
            y=tag_ev["total_questions"],
            marker_color=SOURCE_COLORS["Stack Overflow"],
            text=tag_ev["total_questions"].apply(lambda v: f"{v:,.0f}"),
            textposition="outside",
        ))
        fig_so.update_layout(
            height=300, yaxis_title="Total SO questions", **CHART_LAYOUT
        )
        st.plotly_chart(fig_so, width="stretch")
else:
    st.info("No Stack Overflow log data available.")

st.divider()

st.subheader("Run log")
if not run_completes.empty:
    run_table = run_completes[["timestamp", "source", "success", "error"]].copy()
    run_table["timestamp"] = run_table["timestamp"].dt.strftime("%Y-%m-%d %H:%M UTC")
    run_table = run_table.sort_values("timestamp", ascending=False).reset_index(drop=True)
    run_table.columns = ["Timestamp", "Source", "Packages OK", "Errors"]
    st.dataframe(run_table, width="stretch", hide_index=True)
else:
    st.info("No run_complete events in logs.")