diff --git a/README.md b/README.md index 6205776..a0f85bf 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,15 @@ The plugin also ships 30+ task-specific skills that your AI client loads on dema > Show me the top 10 pages by pageviews ``` +## Plugin usage telemetry + +The plugin collects anonymous telemetry. To turn it off, set either of these (in your shell or the `env` block of `~/.claude/settings.json`): + +```bash +export DO_NOT_TRACK=1 # disables all telemetry (widely honored) +export POSTHOG_PLUGIN_TELEMETRY_DISABLED=1 # disables just this plugin's telemetry +``` + ## Self-hosted For self-hosted PostHog instances, set the `POSTHOG_MCP_URL` environment variable to point to your instance: diff --git a/hooks/hooks.json b/hooks/hooks.json index fa517ba..f11ac20 100644 --- a/hooks/hooks.json +++ b/hooks/hooks.json @@ -1,6 +1,18 @@ { - "description": "PostHog LLM Analytics + permission gating for write commands via mcp__posthog__exec", + "description": "PostHog LLM Analytics + permission gating for write commands via mcp__posthog__exec + anonymous skill-usage telemetry", "hooks": { + "PostToolUse": [ + { + "matcher": "Skill", + "hooks": [ + { + "type": "command", + "command": "${CLAUDE_PLUGIN_ROOT:-$PLUGIN_ROOT}/hooks/skill-invoked.sh", + "timeout": 5 + } + ] + } + ], "SessionEnd": [ { "hooks": [ diff --git a/hooks/skill-invoked.sh b/hooks/skill-invoked.sh new file mode 100755 index 0000000..68defbc --- /dev/null +++ b/hooks/skill-invoked.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +# PostToolUse telemetry for the `Skill` tool — anonymous plugin usage analytics. +# +# Fires after a skill is invoked and, *only for this plugin's own skills*, +# sends a single "plugin skill invoked" event to a PostHog-owned project so the +# team can see which skills get used in the field. Captures the skill name plus +# safe metadata only — never the skill arguments or any free-text content. +# +# Privacy & consent: +# - Anonymous: distinct_id is a random per-install UUID (no email, no PII), +# and events set `$process_person_profile: false` so no person is created. +# - Opt-out (on by default). Disable with either: +# export DO_NOT_TRACK=1 +# export POSTHOG_PLUGIN_TELEMETRY_DISABLED=1 +# - Endpoint/key are overridable for testing via POSTHOG_PLUGIN_TELEMETRY_KEY +# and POSTHOG_PLUGIN_TELEMETRY_HOST. +# +# Pure bash + curl; no jq or other third-party tools. Reads the hook JSON on +# stdin, sends in the background, and always exits 0 — losing a telemetry event +# is fine, interfering with the user's session is not. +# +# Scope: Claude Code only in practice. Codex injects skills as context (no Skill +# tool → no PostToolUse event), and Cursor/Gemini have no hooks, so the matcher +# simply never fires there. See https://developers.openai.com/codex/hooks + +set -u + +# PostHog project (public, write-only ingestion key — safe to ship). +API_KEY="${POSTHOG_PLUGIN_TELEMETRY_KEY:-sTMFPsFhdP1Ssg}" +HOST="${POSTHOG_PLUGIN_TELEMETRY_HOST:-https://us.i.posthog.com}" + +# --- consent guards (opt-out) -------------------------------------------- +# DO_NOT_TRACK: any value other than empty/0/false means "do not track". +case "${DO_NOT_TRACK:-}" in + ""|0|false|FALSE|False) ;; + *) exit 0 ;; +esac +case "${POSTHOG_PLUGIN_TELEMETRY_DISABLED:-}" in + ""|0|false|FALSE|False) ;; + *) exit 0 ;; +esac +# No key configured (e.g. placeholder build) → no-op. +[[ -n "$API_KEY" && "$API_KEY" != "phc_REPLACE_ME" ]] || exit 0 + +# curl is required to send; if absent, no-op silently. +command -v curl >/dev/null 2>&1 || exit 0 + +# Plugin root: Claude Code sets CLAUDE_PLUGIN_ROOT, Codex sets PLUGIN_ROOT (plus +# a CLAUDE_PLUGIN_ROOT alias). Fall back to the script's parent dir. +ROOT="${CLAUDE_PLUGIN_ROOT:-${PLUGIN_ROOT:-}}" +if [[ -z "$ROOT" ]]; then + ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." 2>/dev/null && pwd)" +fi + +# Client label — forward-compatible; today only Claude Code fires this hook. +if [[ -n "${PLUGIN_ROOT:-}" ]]; then + client="codex" +else + client="claude-code" +fi + +input="$(cat)" + +# Only act on the Skill tool (the matcher should already guarantee this). +tool_name="" +if [[ "$input" =~ \"tool_name\"[[:space:]]*:[[:space:]]*\"([^\"]+)\" ]]; then + tool_name="${BASH_REMATCH[1]}" +fi +[[ "$tool_name" == "Skill" ]] || exit 0 + +# Skill identifier lives in tool_input.skill, e.g. "posthog:creating-experiments" +# (verified against real transcripts). Names are [a-zA-Z0-9:_-]+, so a narrow +# regex on the raw payload is safe and matches the first (tool_input) occurrence, +# never anything inside the trailing tool_response. +skill="" +if [[ "$input" =~ \"skill\"[[:space:]]*:[[:space:]]*\"([a-zA-Z0-9:_-]+)\" ]]; then + skill="${BASH_REMATCH[1]}" +fi +[[ -n "$skill" ]] || exit 0 + +# Split "namespace:name" → namespace + base name. +if [[ "$skill" == *:* ]]; then + skill_namespace="${skill%%:*}" + skill_name="${skill##*:}" +else + skill_namespace="" + skill_name="$skill" +fi + +# Capture only THIS plugin's skills: the `posthog:` namespace, or a bare name +# that matches a bundled skill directory. Everything else is ignored. +if [[ "$skill_namespace" != "posthog" && ! -d "$ROOT/skills/$skill_name" ]]; then + exit 0 +fi + +# Whether arguments were passed — presence only, never the content. +args_present=false +if [[ "$input" =~ \"args\"[[:space:]]*:[[:space:]]*\"[^\"]+\" ]]; then + args_present=true +fi + +# permission_mode is a short enum (default/acceptEdits/plan/...). +permission_mode="" +if [[ "$input" =~ \"permission_mode\"[[:space:]]*:[[:space:]]*\"([a-zA-Z]+)\" ]]; then + permission_mode="${BASH_REMATCH[1]}" +fi + +# Plugin version from the manifest. +plugin_version="unknown" +if [[ -f "$ROOT/.claude-plugin/plugin.json" ]]; then + pj="$(cat "$ROOT/.claude-plugin/plugin.json" 2>/dev/null || true)" + if [[ "$pj" =~ \"version\"[[:space:]]*:[[:space:]]*\"([^\"]+)\" ]]; then + plugin_version="${BASH_REMATCH[1]}" + fi +fi + +os="$(uname -s 2>/dev/null || echo unknown)" + +# Stable anonymous per-install id (UUID). Persisted under the plugin's writable +# data dir (Codex: PLUGIN_DATA) or ~/.claude. No PII — deliberately not git email. +data_dir="${CLAUDE_PLUGIN_DATA:-${PLUGIN_DATA:-$HOME/.claude}}" +id_file="$data_dir/posthog-plugin-telemetry-id" +distinct_id="" +[[ -f "$id_file" ]] && distinct_id="$(tr -d '[:space:]' < "$id_file" 2>/dev/null || true)" +if [[ -z "$distinct_id" ]]; then + if command -v uuidgen >/dev/null 2>&1; then + distinct_id="$(uuidgen 2>/dev/null || true)" + elif [[ -r /proc/sys/kernel/random/uuid ]]; then + distinct_id="$(cat /proc/sys/kernel/random/uuid 2>/dev/null || true)" + fi + [[ -z "$distinct_id" ]] && distinct_id="anon-${RANDOM}${RANDOM}-$$" + distinct_id="$(printf '%s' "$distinct_id" | tr -d '[:space:]')" + mkdir -p "$data_dir" 2>/dev/null && printf '%s\n' "$distinct_id" > "$id_file" 2>/dev/null || true +fi +[[ -n "$distinct_id" ]] || exit 0 + +# All interpolated values are constrained to safe charsets (slugs, enums, a +# UUID, a semver), so no JSON/shell escaping is required — same reasoning as +# gate-exec-write.sh. +payload="$(printf '{"api_key":"%s","event":"plugin skill invoked","distinct_id":"%s","properties":{"skill":"%s","skill_name":"%s","skill_namespace":"%s","args_present":%s,"plugin_version":"%s","client":"%s","permission_mode":"%s","os":"%s","$lib":"posthog-ai-plugin","$lib_version":"%s","$process_person_profile":false}}' \ + "$API_KEY" "$distinct_id" "$skill" "$skill_name" "$skill_namespace" "$args_present" "$plugin_version" "$client" "$permission_mode" "$os" "$plugin_version")" + +# Fire-and-forget: background, detached fds, 3s cap → ~0 user-facing latency. +curl -sf -m 3 -X POST "$HOST/i/v0/e/" \ + -H 'Content-Type: application/json' \ + -d "$payload" >/dev/null 2>&1 /dev/null || true + +exit 0 diff --git a/tests/test_skill_invoked.sh b/tests/test_skill_invoked.sh new file mode 100755 index 0000000..f9312dd --- /dev/null +++ b/tests/test_skill_invoked.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# Tests for hooks/skill-invoked.sh. +# +# Plain bash, no dependencies. Run from anywhere: +# +# ./tests/test_skill_invoked.sh +# +# Each case feeds a PostToolUse JSON payload (the shape Claude Code passes on +# stdin) to the hook with a controlled environment, then asserts whether the +# hook sent an event and what the payload contained. The network call is +# intercepted by a fake `curl` on PATH that records the request to a file, so +# nothing leaves the machine. + +set -u + +HERE="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd -- "$HERE/.." && pwd)" +HOOK="$REPO_ROOT/hooks/skill-invoked.sh" + +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT + +# --- fake curl: records URL + -d payload, sends nothing ------------------ +FAKEBIN="$TMP/bin" +mkdir -p "$FAKEBIN" +cat > "$FAKEBIN/curl" <<'EOF' +#!/usr/bin/env bash +url="" ; payload="" +while (( $# )); do + case "$1" in + -d) payload="$2"; shift 2;; + -X|-H|-m) shift 2;; + http://*|https://*) url="$1"; shift;; + *) shift;; + esac +done +{ printf 'URL %s\n' "$url"; printf 'BODY %s\n' "$payload"; } >> "$CURL_CAPTURE" +EOF +chmod +x "$FAKEBIN/curl" + +pass=0 +fail=0 + +# Run the hook with an isolated env. Globals set per call: +# CAP — capture file for this run (reset) +# Args: +# $1 payload $2 fresh-home? (1 to wipe the telemetry-id) $3+ env overrides +CAP="" +run_hook() { + local payload="$1" fresh="$2"; shift 2 + CAP="$TMP/capture.$RANDOM" + : > "$CAP" + local home="$TMP/home" + (( fresh )) && rm -rf "$home" + mkdir -p "$home" + env -i PATH="$FAKEBIN:$PATH" HOME="$home" \ + CLAUDE_PLUGIN_ROOT="$REPO_ROOT" \ + CURL_CAPTURE="$CAP" \ + POSTHOG_PLUGIN_TELEMETRY_HOST="http://localhost:0" \ + "$@" bash "$HOOK" <<<"$payload" +} + +# Wait (bounded) for the backgrounded fake curl to record a request. +wait_sent() { local i; for i in $(seq 1 60); do [[ -s "$CAP" ]] && return 0; sleep 0.05; done; return 1; } +# Give a no-op case a moment, then confirm nothing was recorded. +assert_silent() { sleep 0.3; [[ ! -s "$CAP" ]]; } + +ok() { pass=$((pass + 1)); printf " ok %s\n" "$1"; } +bad() { fail=$((fail + 1)); printf " FAIL %s\n %s\n" "$1" "${2:-}"; } + +body() { sed -n 's/^BODY //p' "$CAP"; } + +# PostToolUse payload for a Skill invocation. +skill_payload() { + local skill="$1" args="${2:-}" + if [[ -n "$args" ]]; then + printf '{"hook_event_name":"PostToolUse","permission_mode":"default","tool_name":"Skill","tool_input":{"skill":"%s","args":"%s"},"tool_response":{"ok":true}}' "$skill" "$args" + else + printf '{"hook_event_name":"PostToolUse","permission_mode":"default","tool_name":"Skill","tool_input":{"skill":"%s"},"tool_response":{"ok":true}}' "$skill" + fi +} + +echo "Running skill-invoked.sh tests..." + +# --- captured: this plugin's skills -------------------------------------- + +run_hook "$(skill_payload "posthog:querying-posthog-data")" 1 +if wait_sent && [[ "$(body)" == *'"skill":"posthog:querying-posthog-data"'* \ + && "$(body)" == *'"skill_name":"querying-posthog-data"'* \ + && "$(body)" == *'"skill_namespace":"posthog"'* \ + && "$(body)" == *'"event":"plugin skill invoked"'* \ + && "$(body)" == *'"$process_person_profile":false'* \ + && "$(body)" == *'"args_present":false'* ]]; then + ok "posthog: namespaced skill is captured" +else + bad "posthog: namespaced skill is captured" "body=$(body)" +fi + +# A bundled skill invoked by bare name (matches a dir under skills/). +BARE_SKILL="" +for d in "$REPO_ROOT"/skills/*/; do BARE_SKILL="$(basename "$d")"; break; done +run_hook "$(skill_payload "$BARE_SKILL")" 1 +if wait_sent && [[ "$(body)" == *"\"skill\":\"$BARE_SKILL\""* \ + && "$(body)" == *'"skill_namespace":""'* ]]; then + ok "bundled bare-name skill ($BARE_SKILL) is captured" +else + bad "bundled bare-name skill is captured" "body=$(body)" +fi + +# args present → boolean true, but the content must never appear. +run_hook "$(skill_payload "posthog:exploring-llm-traces" "SECRET_ARG_PAYLOAD path/to/file")" 1 +if wait_sent && [[ "$(body)" == *'"args_present":true'* && "$(body)" != *"SECRET_ARG_PAYLOAD"* ]]; then + ok "args presence captured as boolean, content never sent" +else + bad "args presence captured as boolean, content never sent" "body=$(body)" +fi + +# --- ignored: not this plugin's skills ----------------------------------- + +run_hook "$(skill_payload "some-other-plugin:do-thing")" 1 +if assert_silent; then ok "other-plugin skill is ignored"; else bad "other-plugin skill is ignored" "body=$(body)"; fi + +run_hook "$(skill_payload "definitely-not-a-bundled-skill")" 1 +if assert_silent; then ok "unknown project skill is ignored"; else bad "unknown project skill is ignored" "body=$(body)"; fi + +run_hook '{"tool_name":"Bash","tool_input":{"command":"ls"}}' 1 +if assert_silent; then ok "non-Skill tool is ignored"; else bad "non-Skill tool is ignored" "body=$(body)"; fi + +# --- consent guards ------------------------------------------------------ + +run_hook "$(skill_payload "posthog:querying-posthog-data")" 1 DO_NOT_TRACK=1 +if assert_silent; then ok "DO_NOT_TRACK=1 disables capture"; else bad "DO_NOT_TRACK=1 disables capture" "body=$(body)"; fi + +run_hook "$(skill_payload "posthog:querying-posthog-data")" 1 POSTHOG_PLUGIN_TELEMETRY_DISABLED=true +if assert_silent; then ok "POSTHOG_PLUGIN_TELEMETRY_DISABLED=true disables capture"; else bad "telemetry disabled flag" "body=$(body)"; fi + +run_hook "$(skill_payload "posthog:querying-posthog-data")" 1 DO_NOT_TRACK=0 +if wait_sent; then ok "DO_NOT_TRACK=0 does not disable capture"; else bad "DO_NOT_TRACK=0 does not disable capture" "no send"; fi + +# --- anonymous id is stable across runs ---------------------------------- + +run_hook "$(skill_payload "posthog:querying-posthog-data")" 1 # fresh home → new id +wait_sent; id1="$(body | sed -n 's/.*"distinct_id":"\([^"]*\)".*/\1/p')" +run_hook "$(skill_payload "posthog:querying-posthog-data")" 0 # reuse home → same id +wait_sent; id2="$(body | sed -n 's/.*"distinct_id":"\([^"]*\)".*/\1/p')" +if [[ -n "$id1" && "$id1" == "$id2" ]]; then + ok "anonymous distinct_id persists across runs ($id1)" +else + bad "anonymous distinct_id persists across runs" "id1=$id1 id2=$id2" +fi + +# --- summary ------------------------------------------------------------- + +echo +echo "Passed: $pass Failed: $fail" +(( fail == 0 ))