refactor: prune historical data, consolidate reporting scripts, and improve CI/CD infrastructure

Author: kanywst

.github/workflows/deploy-pages.yml

@@ -29,7 +29,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
-          node-version: "20"
+          node-version: "22"
           cache: "npm"
           cache-dependency-path: web/package-lock.json
       - run: npm ci

.github/workflows/pr-checks.yml

@@ -0,0 +1,49 @@
+name: PR checks
+
+on:
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  web-build:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: web
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+          cache: "npm"
+          cache-dependency-path: web/package-lock.json
+      - run: npm ci
+      - run: npm run build
+
+  python-syntax:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Compile all track scripts
+        run: python -m compileall -q tracks/*/scripts
+
+  shellcheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run shellcheck
+        run: |
+          shopt -s nullglob globstar
+          files=(scripts/*.sh tracks/*/scripts/*.sh)
+          if [ ${#files[@]} -eq 0 ]; then
+            echo "No shell scripts found"
+            exit 0
+          fi
+          shellcheck "${files[@]}"

scripts/prune.sh

@@ -1,27 +1,71 @@
 #!/usr/bin/env bash
-# Keep only the latest entry under each track artifact directory.
+# Trim track artifacts to the canonical retention policy:
+#   - reports/{daily,weekly}/ : keep latest 1 file
+#   - data/snapshots/, data/raw/ : keep latest 1 entry
+#   - data/normalized/<prefix>-<YYYY-MM-DD>.json : keep latest 7 per prefix
+#     (weekly reports may need recent dated history; non-dated files are left alone)
 # Usage: bash scripts/prune.sh <track-dir>
 set -euo pipefail
+shopt -s nullglob
 
 TRACK_DIR="${1:?track directory required (e.g. tracks/ietf-wimse)}"
 
 prune_keep_latest() {
   local dir="$1"
   local keep="${2:-1}"
   [ -d "$dir" ] || return 0
-  local total
-  total=$(ls -1 "$dir" 2>/dev/null | wc -l | tr -d ' ')
+  local entries=()
+  local entry
+  for entry in "$dir"/*; do
+    entries+=("$(basename "$entry")")
+  done
+  local total=${#entries[@]}
   if [ "$total" -le "$keep" ]; then
     return 0
   fi
   local drop=$((total - keep))
-  ls -1 "$dir" | sort | head -n "$drop" | while read -r entry; do
-    rm -rf "$dir/$entry"
+  IFS=$'\n' read -r -d '' -a sorted < <(printf '%s\n' "${entries[@]}" | sort && printf '\0')
+  local i
+  for ((i = 0; i < drop; i++)); do
+    rm -rf "${dir:?}/${sorted[i]}"
   done
   echo "pruned $drop from $dir"
 }
 
+# For files like <prefix>-<YYYY-MM-DD>.json under a directory, keep the latest N per prefix.
+prune_dated_normalized() {
+  local dir="$1"
+  local keep="${2:-7}"
+  [ -d "$dir" ] || return 0
+  local file base prefix
+  local raw_prefixes=()
+  for file in "$dir"/*-[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9].json; do
+    base=$(basename "$file")
+    prefix=${base%-[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9].json}
+    raw_prefixes+=("$prefix")
+  done
+  [ ${#raw_prefixes[@]} -eq 0 ] && return 0
+  local prefixes=()
+  IFS=$'\n' read -r -d '' -a prefixes < <(printf '%s\n' "${raw_prefixes[@]}" | sort -u && printf '\0')
+  for prefix in "${prefixes[@]}"; do
+    local matches=()
+    for file in "$dir"/"$prefix"-[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9].json; do
+      matches+=("$(basename "$file")")
+    done
+    local total=${#matches[@]}
+    if [ "$total" -le "$keep" ]; then continue; fi
+    local drop=$((total - keep))
+    IFS=$'\n' read -r -d '' -a sorted < <(printf '%s\n' "${matches[@]}" | sort && printf '\0')
+    local i
+    for ((i = 0; i < drop; i++)); do
+      rm -f "${dir:?}/${sorted[i]}"
+    done
+    echo "pruned $drop ${prefix}-* files from $dir"
+  done
+}
+
 prune_keep_latest "$TRACK_DIR/reports/daily" 1
 prune_keep_latest "$TRACK_DIR/reports/weekly" 1
 prune_keep_latest "$TRACK_DIR/data/snapshots" 1
 prune_keep_latest "$TRACK_DIR/data/raw" 1
+prune_dated_normalized "$TRACK_DIR/data/normalized" 7

templates/new-track/README.md

@@ -2,41 +2,47 @@
 
 Automated monitoring and deep-dive workspace for **<Track Name>**.
 
-## Goals
+## Purpose
 
-- Continuously collect the latest signals from primary sources.
-- Detect meaningful changes (draft revisions, new specs, repo activity, mailing-list trends).
+- Continuously collect signals from primary sources.
+- Detect meaningful changes (draft revisions, repo activity, mailing-list trends).
 - Prioritize deep-dive candidates with reproducible scoring.
-- Maintain a hands-on backlog tied to real working-group movement.
+- Generate daily and weekly Markdown reports.
 
-## Data Sources
+## Sources
 
-- <!-- Add your sources here -->
+Configured in `config/sources.yaml`. Add the upstream URLs and tracked repositories you want to watch.
 
-All source endpoints and tracked repositories are configured in `config/sources.yaml`.
+## Layout
 
-## Repository Layout
-
-- `config/`: Source and scoring configuration.
-- `data/raw/<YYYY-MM-DD>/<source>.json`: Raw fetch artifacts.
-- `data/normalized/`: Merged state and scoring output.
-- `reports/daily/<YYYY-MM-DD>.md` and `reports/weekly/<YYYY>-W<NN>.md`: Generated reports.
-- `backlog/candidate-queue.md`: Scored deep-dive candidate queue.
-- `deep-dives/`: Investigation notes (scaffold from `../../templates/deep-dive/`).
-- `scripts/`: `_common.py` helpers + `collect.py` / `normalize.py` / `score.py` / `report.py`.
+```text
+config/                      sources.yaml + scoring.yaml
+data/raw/<YYYY-MM-DD>/       Raw fetch artifacts (latest only — pruned)
+data/normalized/             Merged state + scoring output
+reports/daily/<YYYY-MM-DD>.md     Latest daily report (older pruned)
+reports/weekly/<YYYY>-W<NN>.md    Latest weekly digest (older pruned)
+deep-dives/_backlog.md       Scored deep-dive candidate queue
+deep-dives/<topic>/          Investigation notes (scaffold from templates/deep-dive/)
+scripts/                     _common.py + collect.py / normalize.py / score.py / report.py
+```
 
-## Local Usage
+## Usage
 
 ```bash
 python3 -m venv .venv
 source .venv/bin/activate
 make install
-make update     # collect + normalize + score + report-daily
-make weekly     # collect + normalize + score + report-weekly
+make update     # collect + normalize + score + report-daily + prune
+make weekly     # collect + normalize + score + report-weekly + prune
 ```
 
-Individual stages: `make collect`, `make normalize`, `make score`, `make report-daily`, `make report-weekly`.
+Individual stages: `make collect`, `make normalize`, `make score`, `make report-daily`, `make report-weekly`, `make prune`.
 
 ## Automation
 
-The repo-root `.github/workflows/{daily-update,weekly-digest}.yml` files use a `matrix.track` list. Add this track's directory name to that list to wire it into CI.
+The repo-root `.github/workflows/{daily-update,weekly-digest}.yml` files run `make update` / `make weekly` per track via a `matrix.track` list. Add this track's directory name to that list to wire it into CI.
+
+## Notes
+
+- Retention policy: `make prune` (auto-invoked by `update` / `weekly`) keeps only the latest report and snapshot per category. Use git history for older state.
+- Set `GITHUB_TOKEN` (or `GH_TOKEN`) to avoid rate limits on GitHub fetches.

templates/new-track/scripts/score.py

@@ -54,7 +54,7 @@ def main() -> None:
     else:
         for i, c in enumerate(ranked, start=1):
             lines.append(f"| {i} | {c['score']} | {c['title']} |")
-    write_text(ROOT / "backlog" / "candidate-queue.md", "\n".join(lines) + "\n")
+    write_text(ROOT / "deep-dives" / "_backlog.md", "\n".join(lines) + "\n")
     print(f"score: candidates={len(ranked)} threshold={threshold}")
 
 

tracks/ietf-wimse/Makefile

@@ -17,10 +17,10 @@ score:
 	$(PYTHON) scripts/score.py
 
 report-daily:
-	$(PYTHON) scripts/render_reports.py --mode daily
+	$(PYTHON) scripts/report.py --mode daily
 
 report-weekly:
-	$(PYTHON) scripts/render_reports.py --mode weekly
+	$(PYTHON) scripts/report.py --mode weekly
 
 prune:
 	bash ../../scripts/prune.sh .

tracks/ietf-wimse/README.md

@@ -1,64 +1,53 @@
 # IETF WIMSE Deep Dive
 
-Automated monitoring and deep-dive workspace for the IETF WIMSE working group.
+Continuous monitoring and deep-dive workspace for the IETF WIMSE working group (Workload Identity in Multi-System Environments).
 
-## Goals
+## Purpose
 
-- Continuously collect the latest WIMSE signals from primary sources.
-- Detect meaningful changes (draft revisions, new related drafts, meeting artifacts, mailing-list trends).
-- Prioritize deep-dive candidates with reproducible scoring.
-- Maintain a hands-on lab backlog tied to real WG movement.
+- Track WIMSE drafts, related specifications, and working-group activity.
+- Surface deep-dive candidates ranked by lifecycle state, recency, and mailing-list signal.
+- Generate daily and weekly Markdown reports for review.
 
-## Data Sources
+## Sources
 
-- Datatracker WG pages: documents, meetings, history.
-- IETF mail archive (`wimse` list).
-- GitHub repositories under `ietf-wg-wimse`.
+Configured in `config/sources.yaml`:
 
-All source endpoints and tracked repositories are configured in `config/sources.yaml`.
+- IETF Datatracker (WG documents, meetings, history) — HTML scrape
+- IETF Mail Archive (`mailarchive.ietf.org/arch/browse/wimse/`) — HTML scrape
+- GitHub repos under `ietf-wg-wimse/` — REST API
 
-## Repository Layout
+## Layout
 
-- `config/`: source and scoring configuration.
-- `data/raw/`: raw fetch artifacts by date.
-- `data/normalized/`: latest normalized source outputs and merged state.
-- `data/snapshots/`: daily state snapshots.
-- `reports/daily/`: generated daily markdown reports.
-- `reports/weekly/`: generated weekly markdown reports.
-- `backlog/candidate-queue.md`: scored deep-dive candidate queue.
-- `deep-dives/`: manually curated deep-dive notes (scaffold from `../../templates/deep-dive/`).
-- `scripts/`: data collection, normalization, scoring, and report generation.
+```text
+config/                      sources.yaml + scoring.yaml
+data/raw/<YYYY-MM-DD>/       Per-source audit JSON (latest only — pruned)
+data/normalized/             datatracker.json / mailarchive.json / github.json /
+                             wimse_state.json / candidates.json
+data/snapshots/<YYYY-MM-DD>/ Snapshot of normalized state
+reports/daily/<YYYY-MM-DD>.md     Latest daily report
+reports/weekly/<YYYY>-W<NN>.md    Latest weekly digest
+deep-dives/_backlog.md       Scored deep-dive candidate queue
+deep-dives/<topic>/          Investigation notes
+scripts/                     _common.py + collect_*.py / normalize.py / score.py / report.py
+```
 
-## Local Usage
+## Usage
 
 ```bash
 python3 -m venv .venv
 source .venv/bin/activate
 make install
-make update
+make update     # collect + normalize + score + report-daily + prune
+make weekly     # collect + normalize + score + report-weekly + prune
 ```
 
-`make update` runs:
-
-1. Source collectors
-2. State normalization
-3. Candidate scoring
-4. Daily report rendering
-
-Useful targets:
-
-- `make collect`
-- `make normalize`
-- `make score`
-- `make report-daily`
-- `make report-weekly`
-- `make weekly`
+Set `GITHUB_TOKEN` (or `GH_TOKEN`) to avoid GitHub API rate limits.
 
 ## Automation
 
-Driven from the repo-root `.github/workflows/{daily-update,weekly-digest}.yml`. Both workflows commit generated outputs when changed.
+Driven by repo-root `.github/workflows/{daily-update,weekly-digest}.yml` (matrix over all tracks).
 
 ## Notes
 
-- Generated files are intentionally committed for historical traceability.
-- Scoring logic is intentionally simple and transparent; tune weights in `config/scoring.yaml`.
+- Scoring weights live in `config/scoring.yaml` (`recency_days`, `weights`, `priority_keywords`).
+- Retention: `make prune` keeps only the latest report and snapshot. For older state use git history.

tracks/ietf-wimse/backlog/candidate-queue.md

@@ -2,9 +2,11 @@
 
 import datetime as dt
 import json
+import os
 import re
 from pathlib import Path
 from typing import Any
+from urllib.parse import urlparse
 
 import requests
 import yaml
@@ -51,15 +53,25 @@ def write_text(path: Path, text: str) -> None:
     path.write_text(text, encoding="utf-8")
 
 
+def _build_headers(url: str) -> dict[str, str]:
+    headers = dict(DEFAULT_HEADERS)
+    host = urlparse(url).hostname or ""
+    if host.endswith("api.github.com"):
+        token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN")
+        if token:
+            headers["Authorization"] = f"Bearer {token}"
+    return headers
+
+
 def fetch_text(url: str, timeout: int = 30) -> str:
-    resp = requests.get(url, headers=DEFAULT_HEADERS, timeout=timeout)
+    resp = requests.get(url, headers=_build_headers(url), timeout=timeout)
     resp.raise_for_status()
     resp.encoding = resp.encoding or "utf-8"
     return resp.text
 
 
 def fetch_json(url: str, timeout: int = 30) -> Any:
-    resp = requests.get(url, headers=DEFAULT_HEADERS, timeout=timeout)
+    resp = requests.get(url, headers=_build_headers(url), timeout=timeout)
     resp.raise_for_status()
     return resp.json()