rename: compute.ts → transform.ts

Author: Hweinstock

.github/dashboard/data.py

@@ -0,0 +1,256 @@
+"""Fetch and process GitHub issues + PR data for aws/agentcore-cli."""
+import json
+import subprocess
+import sys
+from collections import Counter, defaultdict
+from datetime import datetime, timedelta
+
+REPO = "aws/agentcore-cli"
+
+
+def _gh(endpoint):
+    r = subprocess.run(["gh", "api", "--paginate", f"/repos/{REPO}/{endpoint}"],
+                       capture_output=True, text=True, check=True)
+    return json.loads(r.stdout)
+
+
+def _gh_single(endpoint):
+    r = subprocess.run(["gh", "api", f"/repos/{REPO}/{endpoint}"],
+                       capture_output=True, text=True, check=True)
+    return json.loads(r.stdout)
+
+
+def _dt(s):
+    return datetime.fromisoformat(s.replace("Z", "+00:00")) if s else None
+
+
+def _fmt_h(h):
+    if h < 1: return f"{h*60:.0f}m"
+    if h < 24: return f"{h:.1f}h"
+    return f"{h/24:.1f}d"
+
+
+def _percentiles(vals):
+    if not vals: return {"med": 0, "avg": 0, "p90": 0}
+    s = sorted(vals)
+    return {"med": round(s[len(s)//2], 1), "avg": round(sum(s)/len(s), 1), "p90": round(s[int(len(s)*0.9)], 1)}
+
+
+# ── Issues ──────────────────────────────────────────────────────────
+
+def fetch_issues():
+    print("Fetching issues...")
+    raw = _gh("issues?state=all&per_page=100")
+    issues = [i for i in raw if "pull_request" not in i]
+    print(f"  {len(issues)} issues")
+    return issues
+
+
+def compute_issues(raw):
+    issues = [{
+        "number": i["number"], "title": i["title"], "state": i["state"],
+        "created": _dt(i["created_at"]), "closed": _dt(i["closed_at"]),
+        "labels": [l["name"] for l in i["labels"]],
+        "assignees": [a["login"] for a in i["assignees"]],
+        "comments": i["comments"], "reactions": i["reactions"]["total_count"],
+        "state_reason": i.get("state_reason"),
+        "author": i["user"]["login"], "author_type": i["author_association"],
+    } for i in raw]
+    issues.sort(key=lambda i: i["created"])
+    now = datetime.now(issues[0]["created"].tzinfo)
+    total = len(issues)
+    opened = sum(1 for i in issues if i["state"] == "open")
+    labeled = sum(1 for i in issues if i["labels"])
+    assigned = sum(1 for i in issues if i["assignees"])
+    dates = [i["created"] for i in issues]
+    span = (max(dates) - min(dates)).days
+    res = sorted([(i["closed"] - i["created"]).total_seconds() / 3600 for i in issues if i["closed"]])
+    p = _percentiles(res)
+
+    # Weekly timeline
+    close_dates = [i["closed"] for i in issues if i["closed"]]
+    start = min(dates).replace(hour=0, minute=0, second=0, microsecond=0) - timedelta(days=min(dates).weekday())
+    weeks, cum = [], 0
+    cur = start
+    while cur <= max(dates):
+        nxt = cur + timedelta(days=7)
+        wo = sum(1 for d in dates if cur <= d < nxt)
+        wc = sum(1 for d in close_dates if cur <= d < nxt)
+        cum += wo - wc
+        weeks.append({"week": cur.strftime("%b %d"), "opened": wo, "closed": wc, "cum": cum})
+        cur = nxt
+
+    # Labels
+    lc = Counter()
+    for i in issues:
+        for l in i["labels"]: lc[l] += 1
+    lc["(unlabeled)"] = total - labeled
+    label_stats = []
+    for label, count in lc.most_common():
+        sub = [i for i in issues if label in i["labels"]] if label != "(unlabeled)" else [i for i in issues if not i["labels"]]
+        o = sum(1 for i in sub if i["state"] == "open")
+        label_stats.append({"label": label, "count": count, "open": o, "closed": len(sub)-o, "pct": round((len(sub)-o)*100/len(sub)) if sub else 0})
+
+    # Resolution by label
+    lt = defaultdict(list)
+    for i in issues:
+        if not i["closed"]: continue
+        h = (i["closed"] - i["created"]).total_seconds() / 3600
+        for l in (i["labels"] or ["(unlabeled)"]): lt[l].append(h)
+    res_by_label = [{"label": l, "n": len(t), **_percentiles(t)} for l, t in sorted(lt.items(), key=lambda x: -len(x[1]))]
+
+    # Age buckets
+    ab = {"< 1 day": 0, "1-3 days": 0, "3-7 days": 0, "1-2 weeks": 0, "2-4 weeks": 0, "1-2 months": 0, "> 2 months": 0}
+    for i in issues:
+        if i["state"] != "open": continue
+        h = (now - i["created"]).total_seconds() / 3600
+        if h < 24: ab["< 1 day"] += 1
+        elif h < 72: ab["1-3 days"] += 1
+        elif h < 168: ab["3-7 days"] += 1
+        elif h < 336: ab["1-2 weeks"] += 1
+        elif h < 672: ab["2-4 weeks"] += 1
+        elif h < 1440: ab["1-2 months"] += 1
+        else: ab["> 2 months"] += 1
+
+    # Authors
+    ac = Counter(i["author"] for i in issues)
+    authors = [{"author": a, "count": c, "type": next((i["author_type"] for i in issues if i["author"] == a), "")} for a, c in ac.most_common(15)]
+
+    # Engagement
+    top_eng = [{"number": i["number"], "title": i["title"][:60], "comments": i["comments"], "reactions": i["reactions"], "state": i["state"]} for i in sorted(issues, key=lambda i: i["comments"], reverse=True)[:10]]
+
+    # Stale
+    stale = sorted([{"number": i["number"], "title": i["title"][:55], "age": (now - i["created"]).days, "labels": i["labels"]} for i in issues if i["state"] == "open" and (now - i["created"]).days > 14 and i["comments"] == 0], key=lambda x: -x["age"])
+
+    # Auto-extract common terms from unlabeled issue titles
+    ul = [i for i in issues if not i["labels"]]
+    stop = {"the","a","an","in","on","of","to","for","is","and","or","not","with","from","by","at","it","as","be","was","are","this","that","but","if","no","do","does","can","has","have","had","i","my","we","our","you","your","its","all","any","new","after","when","should","would","could","into","than","then","also","just","about","up","out","so","how","what","why","which","where","who","been","being","will","more","some","only","other","each","both","few","most","very","too","here","there","these","those","such","same","own","between","through","during","before","while","since","until","against","above","below","over","under","again","further","once","already","still","now","get","set","use","add","run","try","see","let","make","take","give","go","come","find","keep","put","show","tell","say","ask","work","seem","feel","leave","call","need","may","must","shall","might","done","got","went","came","made","took","gave","said","told","used","found","left","called","started","tried","ran","saw","let","known","given","taken","shown","become","gone","kept","put","brought","thought","told","sent","received","held","read","written","spoken","broken","chosen","driven","eaten","fallen","forgotten","gotten","hidden","ridden","risen","shaken","stolen","sworn","thrown","worn","woken","wound","wrung","built","burnt","dealt","dreamt","felt","hung","knelt","leant","leapt","learnt","meant","met","paid","sold","shot","slid","slung","slit","smelt","spelt","spent","spilt","spun","split","spoilt","spread","sprung","stood","stuck","stung","stunk","struck","strung","swept","swum","swung","taught","torn","trod","understood","wept","won","wound","woven","wrung"}
+    word_counts = Counter()
+    for i in ul:
+        words = set(w.lower().strip("[]():#'\",.!?") for w in i["title"].split() if len(w) > 2)
+        word_counts.update(words - stop)
+    # Group by frequency, take top clusters
+    common_terms = {term: count for term, count in word_counts.most_common(20) if count >= 3}
+
+    # Existing labels that are defined but unused on issues
+    all_labels_on_issues = set()
+    for i in issues:
+        all_labels_on_issues.update(i["labels"])
+    repo_labels = {"bug", "enhancement", "question", "documentation", "good first issue", "help wanted", "invalid", "duplicate", "wontfix"}
+    unused_labels = sorted(repo_labels - all_labels_on_issues)
+
+    return {
+        "gen": now.strftime("%Y-%m-%d %H:%M UTC"),
+        "ov": {"total": total, "open": opened, "closed": total-opened, "labeled": labeled, "unlabeled": total-labeled, "assigned": assigned, "unassigned": total-assigned,
+               "completed": sum(1 for i in issues if i["state_reason"] == "completed"), "not_planned": sum(1 for i in issues if i["state_reason"] == "not_planned"), "dupes": sum(1 for i in issues if i["state_reason"] == "duplicate"),
+               "start": min(dates).strftime("%b %d, %Y"), "end": max(dates).strftime("%b %d, %Y"), "span": span, "rate": round(total/max(span,1)*7, 1),
+               "med": _fmt_h(p["med"]), "avg": _fmt_h(p["avg"]), "p90": _fmt_h(p["p90"]),
+               "member": sum(1 for i in issues if i["author_type"] == "MEMBER"), "community": sum(1 for i in issues if i["author_type"] != "MEMBER")},
+        "timeline": weeks, "labels": label_stats, "res_by_label": res_by_label,
+        "age": ab, "authors": authors, "engagement": top_eng,
+        "zero_eng": sum(1 for i in issues if i["comments"] == 0 and i["reactions"] == 0 and i["state"] == "open"),
+        "stale": stale, "common_terms": common_terms, "unused_labels": unused_labels,
+    }
+
+
+# ── Pull Requests (GraphQL for speed) ──────────────────────────────
+
+def fetch_prs():
+    print("Fetching PRs with reviews (GraphQL)...")
+    prs = []
+    cursor = None
+    while True:
+        after = f', after: "{cursor}"' if cursor else ""
+        query = '{repository(owner:"aws",name:"agentcore-cli"){pullRequests(first:100,states:[MERGED,CLOSED,OPEN],orderBy:{field:CREATED_AT,direction:DESC}%s){pageInfo{hasNextPage endCursor}nodes{number title state createdAt mergedAt closedAt isDraft author{login}labels(first:10){nodes{name}}reviews(first:20){nodes{author{login}state submittedAt}}}}}}' % after
+        r = subprocess.run(["gh", "api", "graphql", "-f", f"query={query}"],
+                           capture_output=True, text=True, check=True)
+        data = json.loads(r.stdout)["data"]["repository"]["pullRequests"]
+        prs.extend(data["nodes"])
+        print(f"  ...{len(prs)} PRs")
+        if not data["pageInfo"]["hasNextPage"]:
+            break
+        cursor = data["pageInfo"]["endCursor"]
+    print(f"  Done: {len(prs)} PRs with inline reviews")
+    return prs
+
+
+def compute_prs(gql_prs):
+    prs = []
+    for p in gql_prs:
+        author = p["author"]["login"] if p["author"] else "ghost"
+        if author == "github-actions[bot]":
+            continue
+        created = _dt(p["createdAt"])
+        merged = _dt(p["mergedAt"])
+        first_review = None
+        for rv in p["reviews"]["nodes"]:
+            t = _dt(rv.get("submittedAt"))
+            if t and (first_review is None or t < first_review):
+                first_review = t
+        prs.append({
+            "number": p["number"], "title": p["title"],
+            "state": "open" if p["state"] == "OPEN" else "closed",
+            "created": created, "merged": merged,
+            "draft": p["isDraft"], "author": author,
+            "labels": [l["name"] for l in p["labels"]["nodes"]],
+            "ttfr_h": round((first_review - created).total_seconds() / 3600, 1) if first_review else None,
+            "ttm_h": round((merged - created).total_seconds() / 3600, 1) if merged else None,
+        })
+
+    prs.sort(key=lambda p: p["created"])
+    now = datetime.now(prs[0]["created"].tzinfo)
+    total = len(prs)
+    n_merged = sum(1 for p in prs if p["merged"])
+    closed_no_merge = sum(1 for p in prs if p["state"] == "closed" and not p["merged"])
+    open_prs = sum(1 for p in prs if p["state"] == "open")
+    drafts = sum(1 for p in prs if p["draft"] and p["state"] == "open")
+
+    ttfr = [p["ttfr_h"] for p in prs if p["ttfr_h"] is not None]
+    ttm = [p["ttm_h"] for p in prs if p["ttm_h"] is not None]
+    no_review = sum(1 for p in prs if p["ttfr_h"] is None and p["merged"])
+
+    # Weekly timeline
+    dates = [p["created"] for p in prs]
+    merge_dates = [p["merged"] for p in prs if p["merged"]]
+    start = min(dates).replace(hour=0, minute=0, second=0, microsecond=0) - timedelta(days=min(dates).weekday())
+    weeks, cum = [], 0
+    cur = start
+    while cur <= max(dates):
+        nxt = cur + timedelta(days=7)
+        wo = sum(1 for d in dates if cur <= d < nxt)
+        wm = sum(1 for d in merge_dates if cur <= d < nxt)
+        cum += wo - wm
+        weeks.append({"week": cur.strftime("%b %d"), "opened": wo, "merged": wm, "cum": cum})
+        cur = nxt
+
+    # Size distribution
+    size_counts = Counter()
+    size_ttm = defaultdict(list)
+    for p in prs:
+        sz = next((l for l in p["labels"] if l.startswith("size/")), "(no size label)")
+        size_counts[sz] += 1
+        if p["ttm_h"] is not None:
+            size_ttm[sz].append(p["ttm_h"])
+    size_stats = [{"size": s, "count": c, **_percentiles(size_ttm.get(s, []))} for s, c in size_counts.most_common()]
+
+    # Authors
+    ac = Counter(p["author"] for p in prs)
+    top_authors = [{"author": a, "count": c} for a, c in ac.most_common(15)]
+
+    # Stale
+    stale = sorted([{"number": p["number"], "title": p["title"][:55], "age": (now - p["created"]).days, "author": p["author"], "draft": p["draft"]} for p in prs if p["state"] == "open" and (now - p["created"]).days > 7], key=lambda x: -x["age"])
+
+    tp = _percentiles(ttfr)
+    mp = _percentiles(ttm)
+    return {
+        "gen": now.strftime("%Y-%m-%d %H:%M UTC"),
+        "ov": {"total": total, "merged": n_merged, "closed_no_merge": closed_no_merge, "open": open_prs, "drafts": drafts,
+               "merge_rate": round(n_merged*100/(n_merged+closed_no_merge)) if (n_merged+closed_no_merge) else 0,
+               "ttfr_med": _fmt_h(tp["med"]), "ttfr_avg": _fmt_h(tp["avg"]), "ttfr_p90": _fmt_h(tp["p90"]),
+               "ttm_med": _fmt_h(mp["med"]), "ttm_avg": _fmt_h(mp["avg"]), "ttm_p90": _fmt_h(mp["p90"]),
+               "no_review_merged": no_review,
+               "start": min(dates).strftime("%b %d, %Y"), "end": max(dates).strftime("%b %d, %Y"),
+               "span": (max(dates) - min(dates)).days},
+        "timeline": weeks, "ttfr_raw": ttfr, "ttm_raw": ttm,
+        "size_stats": size_stats, "top_authors": top_authors, "stale": stale,
+    }

.github/dashboard/generate.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+"""Generate static dashboard HTML files for GitHub Pages."""
+import sys
+from pathlib import Path
+
+# Allow imports from this directory
+sys.path.insert(0, str(Path(__file__).parent))
+
+from data import fetch_issues, compute_issues, fetch_prs, compute_prs
+from html_issues import build_issues_html
+from html_prs import build_prs_html
+
+OUT = Path(__file__).parent / "site"
+
+
+def main():
+    OUT.mkdir(exist_ok=True)
+
+    # Issues
+    raw_issues = fetch_issues()
+    issue_data = compute_issues(raw_issues)
+    (OUT / "issues.html").write_text(build_issues_html(issue_data))
+    print(f"  → {OUT / 'issues.html'}")
+
+    # PRs
+    raw_prs = fetch_prs()
+    pr_data = compute_prs(raw_prs)
+    (OUT / "prs.html").write_text(build_prs_html(pr_data))
+    print(f"  → {OUT / 'prs.html'}")
+
+    # Index redirect
+    (OUT / "index.html").write_text(
+        '<!DOCTYPE html><html><head><meta http-equiv="refresh" content="0;url=issues.html"></head></html>'
+    )
+    print(f"  → {OUT / 'index.html'}")
+    print("Done!")
+
+
+if __name__ == "__main__":
+    main()