|
| 1 | +#!/usr/bin/env bash |
| 2 | +# Regression test for collect-feedback.sh telemetry staleness (GH #266). |
| 3 | +# |
| 4 | +# The per-tool-call telemetry writer was removed with the Experience Engine |
| 5 | +# (GH #200, commit 3beb8e52, 2026-06-01), but collect-feedback.sh kept reading |
| 6 | +# the orphaned ~/.claude/rn-agent/telemetry/*.jsonl files and presented |
| 7 | +# weeks-old events as "recent" in filed feedback issues. The collector must |
| 8 | +# cross-check the newest event's age against now and report honestly: |
| 9 | +# - fresh (<24h, legacy writer still active) → telemetry_status "ok" + events |
| 10 | +# - stale (>=24h) → "stale (...)" + NO events |
| 11 | +# - absent (no dir / no files) → "none" + NO events |
| 12 | +# |
| 13 | +# End-to-end: fake $HOME with planted telemetry files, run the real script, |
| 14 | +# assert on the emitted JSON. |
| 15 | +# |
| 16 | +# Run: bash scripts/test/telemetry-staleness.test.sh |
| 17 | + |
| 18 | +set -uo pipefail |
| 19 | + |
| 20 | +SCRIPT_DIR="$(cd "$(dirname "$0")/.." && pwd)" |
| 21 | +COLLECT="$SCRIPT_DIR/collect-feedback.sh" |
| 22 | + |
| 23 | +fail=0 |
| 24 | + |
| 25 | +# jq-free JSON assertions via python3 (already a hard dependency of the script). |
| 26 | +get_field() { # $1=json $2=python expr over `d` |
| 27 | + printf '%s' "$1" | python3 -c "import json,sys; d=json.load(sys.stdin); print($2)" 2>/dev/null |
| 28 | +} |
| 29 | + |
| 30 | +run_collect() { # $1=fake home |
| 31 | + HOME="$1" CLAUDE_PLUGIN_DATA="$1/plugin-data" bash "$COLLECT" 2>/dev/null |
| 32 | +} |
| 33 | + |
| 34 | +# make_home runs inside $(...) subshells, so cleanup state can't be |
| 35 | +# accumulated there — the trap references the parent-scope home vars instead |
| 36 | +# (":-" keeps `set -u` happy for homes not yet created at failure time). |
| 37 | +trap 'rm -rf "${home1:-}" "${home2:-}" "${home3:-}" "${home4:-}" "${home5:-}"' EXIT |
| 38 | + |
| 39 | +make_home() { |
| 40 | + local h |
| 41 | + h="$(mktemp -d)" |
| 42 | + mkdir -p "$h/plugin-data" |
| 43 | + printf '%s' "$h" |
| 44 | +} |
| 45 | + |
| 46 | +check() { # $1=label $2=actual $3=expected-prefix |
| 47 | + case "$2" in |
| 48 | + "$3"*) echo "ok: $1 ($2)";; |
| 49 | + *) echo "FAIL: $1 — expected prefix '$3', got '$2'"; fail=1;; |
| 50 | + esac |
| 51 | +} |
| 52 | + |
| 53 | +# ── Case 1: stale telemetry (the GH #266 repro) ───────────────────── |
| 54 | +home1="$(make_home)" |
| 55 | +tdir1="$home1/.claude/rn-agent/telemetry" |
| 56 | +mkdir -p "$tdir1" |
| 57 | +cat > "$tdir1/2026-05-31-session-1234.jsonl" <<'EOF' |
| 58 | +{"ts":"2026-05-31T18:44:32.560Z","event":"tool_call","tool":"cdp_status","result":"PASS","latency_ms":1,"phase":"tool"} |
| 59 | +EOF |
| 60 | +# Backdate mtime far past the 24h threshold (portable across BSD/GNU touch). |
| 61 | +touch -t 202605311844 "$tdir1/2026-05-31-session-1234.jsonl" |
| 62 | + |
| 63 | +out1="$(run_collect "$home1")" |
| 64 | +status1="$(get_field "$out1" "d.get('telemetry_status','<missing>')")" |
| 65 | +count1="$(get_field "$out1" "len(d.get('recent_telemetry_lines',['sentinel']))")" |
| 66 | +check "stale: telemetry_status flags staleness" "$status1" "stale" |
| 67 | +check "stale: status names the age in days" "$status1" "stale (last event" |
| 68 | +check "stale: old events are NOT shipped as recent" "$count1" "0" |
| 69 | + |
| 70 | +# ── Case 2: no telemetry at all (fresh install post-#200) ─────────── |
| 71 | +home2="$(make_home)" |
| 72 | +out2="$(run_collect "$home2")" |
| 73 | +status2="$(get_field "$out2" "d.get('telemetry_status','<missing>')")" |
| 74 | +count2="$(get_field "$out2" "len(d.get('recent_telemetry_lines',['sentinel']))")" |
| 75 | +check "none: telemetry_status reports none" "$status2" "none" |
| 76 | +check "none: no events shipped" "$count2" "0" |
| 77 | + |
| 78 | +# ── Case 3: fresh telemetry (legacy writer still active) ──────────── |
| 79 | +home3="$(make_home)" |
| 80 | +tdir3="$home3/.claude/rn-agent/telemetry" |
| 81 | +mkdir -p "$tdir3" |
| 82 | +cat > "$tdir3/current-session.jsonl" <<'EOF' |
| 83 | +{"ts":"2026-06-12T12:00:00.000Z","event":"tool_call","tool":"cdp_status","result":"PASS","latency_ms":2,"phase":"tool"} |
| 84 | +EOF |
| 85 | +# mtime = now (just written) → fresh. |
| 86 | + |
| 87 | +out3="$(run_collect "$home3")" |
| 88 | +status3="$(get_field "$out3" "d.get('telemetry_status','<missing>')")" |
| 89 | +count3="$(get_field "$out3" "len(d.get('recent_telemetry_lines',[]))")" |
| 90 | +check "fresh: telemetry_status ok" "$status3" "ok" |
| 91 | +check "fresh: events ARE shipped" "$count3" "1" |
| 92 | + |
| 93 | +# ── Case 4: telemetry dir exists but is EMPTY (manual cleanup) ────── |
| 94 | +# Pre-existing hazard surfaced in the #266 review: the unmatched glob made |
| 95 | +# `ls` fail and, under `set -euo pipefail`, killed the WHOLE collector — |
| 96 | +# /send-feedback got zero JSON. Must degrade to status "none" instead. |
| 97 | +home4="$(make_home)" |
| 98 | +mkdir -p "$home4/.claude/rn-agent/telemetry" |
| 99 | +out4="$(run_collect "$home4")" |
| 100 | +status4="$(get_field "$out4" "d.get('telemetry_status','<missing>')")" |
| 101 | +check "empty dir: collector survives and reports none" "$status4" "none" |
| 102 | + |
| 103 | +# ── Case 5: future mtime (clock skew / filesystem restore) ────────── |
| 104 | +# A negative age must never count as fresh (it would ship stale events). |
| 105 | +home5="$(make_home)" |
| 106 | +tdir5="$home5/.claude/rn-agent/telemetry" |
| 107 | +mkdir -p "$tdir5" |
| 108 | +cat > "$tdir5/skewed-session.jsonl" <<'EOF' |
| 109 | +{"ts":"2031-01-01T00:00:00.000Z","event":"tool_call","tool":"cdp_status","result":"PASS","latency_ms":3,"phase":"tool"} |
| 110 | +EOF |
| 111 | +touch -t 203101010000 "$tdir5/skewed-session.jsonl" |
| 112 | +out5="$(run_collect "$home5")" |
| 113 | +status5="$(get_field "$out5" "d.get('telemetry_status','<missing>')")" |
| 114 | +count5="$(get_field "$out5" "len(d.get('recent_telemetry_lines',['sentinel']))")" |
| 115 | +check "future mtime: not treated as fresh" "$status5" "stale" |
| 116 | +check "future mtime: no events shipped" "$count5" "0" |
| 117 | + |
| 118 | +if [ "$fail" -ne 0 ]; then |
| 119 | + echo "telemetry-staleness.test.sh: FAILURES" |
| 120 | + exit 1 |
| 121 | +fi |
| 122 | +echo "telemetry-staleness.test.sh: all assertions passed" |
0 commit comments