Skip to content

Commit 5d11123

Browse files
feat(telemetry): add skill usage telemetry
1 parent 7153352 commit 5d11123

8 files changed

Lines changed: 433 additions & 5 deletions

File tree

README.md

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ An [opencode](https://opencode.ai) plugin that exports telemetry via OpenTelemet
4747
| `opencode.session.cost.total` | Histogram | Total cost per session in USD, recorded on idle |
4848
| `opencode.model.usage` | Counter | Messages per model and provider |
4949
| `opencode.retry.count` | Counter | API retries observed via `session.status` events |
50+
| `opencode.skill.count` | Counter | Skill command invocations when opencode exposes command `source=skill` metadata |
5051

5152
### Log events
5253

@@ -60,6 +61,7 @@ An [opencode](https://opencode.ai) plugin that exports telemetry via OpenTelemet
6061
| `api_error` | Failed assistant message (error summary, duration) |
6162
| `tool_result` | Tool completed or errored (duration, success, output size) |
6263
| `tool_decision` | Permission prompt answered (accept/reject) |
64+
| `skill_invoked` | Skill command invoked (includes skill name, command name, optional agent/subtask metadata, and argument length) |
6365
| `commit` | Git commit detected |
6466

6567
## Installation
@@ -165,7 +167,7 @@ Disabling a metric only stops the counter/histogram from being incremented — t
165167
export OPENCODE_DISABLE_METRICS="retry.count"
166168

167169
# Disable multiple metrics
168-
export OPENCODE_DISABLE_METRICS="cache.count,session.duration,session.token.total,session.cost.total,model.usage,retry.count,message.count"
170+
export OPENCODE_DISABLE_METRICS="cache.count,session.duration,session.token.total,session.cost.total,model.usage,retry.count,message.count,skill.count"
169171

170172
# Disable the new per-session cumulative gauge while keeping the delta counter
171173
export OPENCODE_DISABLE_METRICS="lines_of_code.total"
@@ -176,7 +178,7 @@ export OPENCODE_DISABLE_METRICS="lines_of_code.total"
176178
The following metrics are specific to opencode and have no equivalent in Claude Code's built-in monitoring. If you are using a Claude Code dashboard and want to avoid cluttering it with opencode-only metrics, you can disable them:
177179

178180
```bash
179-
export OPENCODE_DISABLE_METRICS="cache.count,session.duration,session.token.total,session.cost.total,model.usage,retry.count,message.count"
181+
export OPENCODE_DISABLE_METRICS="cache.count,session.duration,session.token.total,session.cost.total,model.usage,retry.count,message.count,skill.count"
180182
```
181183

182184
| Metric suffix | Why it's opencode-only |
@@ -188,6 +190,7 @@ export OPENCODE_DISABLE_METRICS="cache.count,session.duration,session.token.tota
188190
| `model.usage` | Per-model message counter — not emitted by Claude Code |
189191
| `retry.count` | API retry counter — not emitted by Claude Code |
190192
| `message.count` | Completed message counter — not emitted by Claude Code |
193+
| `skill.count` | OpenCode skill command counter — not emitted by Claude Code |
191194

192195
### Disabling OTLP logs
193196

src/handlers/skill.ts

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
import { SeverityNumber } from "@opentelemetry/api-logs"
2+
import { isMetricEnabled } from "../util.ts"
3+
import type { HandlerContext, PluginLogger } from "../types.ts"
4+
5+
type SkillCommandInfo = {
6+
name: string
7+
description?: string
8+
agent?: string
9+
subtask?: boolean
10+
}
11+
12+
type RawCommand = {
13+
name?: unknown
14+
source?: unknown
15+
description?: unknown
16+
agent?: unknown
17+
subtask?: unknown
18+
}
19+
20+
type CommandListClient = {
21+
command?: {
22+
list(options?: { query?: { directory?: string; workspace?: string } }): Promise<{ data?: unknown; error?: unknown } | unknown>
23+
}
24+
}
25+
26+
type CommandExecuteInput = {
27+
command: string
28+
sessionID: string
29+
arguments: string
30+
}
31+
32+
type SkillCommandLookup = {
33+
ok: boolean
34+
commands: SkillCommandInfo[]
35+
}
36+
37+
function rawSkillCommand(command: RawCommand): SkillCommandInfo | undefined {
38+
if (command.source !== "skill" || typeof command.name !== "string") return undefined
39+
return {
40+
name: command.name,
41+
...(typeof command.description === "string" ? { description: command.description } : {}),
42+
...(typeof command.agent === "string" ? { agent: command.agent } : {}),
43+
...(typeof command.subtask === "boolean" ? { subtask: command.subtask } : {}),
44+
}
45+
}
46+
47+
function commandsFromPayload(payload: unknown): SkillCommandInfo[] {
48+
const data = payload && typeof payload === "object" && "data" in payload
49+
? (payload as { data?: unknown }).data
50+
: payload
51+
const commands = data && typeof data === "object" && "data" in data
52+
? (data as { data?: unknown }).data
53+
: data
54+
if (!Array.isArray(commands)) return []
55+
return commands
56+
.map((command) => rawSkillCommand(command as RawCommand))
57+
.filter((command): command is SkillCommandInfo => !!command)
58+
}
59+
60+
async function clientSkillCommands(client: CommandListClient, directory: string | undefined): Promise<SkillCommandLookup> {
61+
if (!client.command?.list) return { ok: false, commands: [] }
62+
const response = await client.command.list(directory ? { query: { directory } } : undefined)
63+
if (response && typeof response === "object" && "error" in response && (response as { error?: unknown }).error !== undefined) {
64+
return { ok: false, commands: [] }
65+
}
66+
return { ok: true, commands: commandsFromPayload(response) }
67+
}
68+
69+
async function rawSkillCommands(serverUrl: URL, directory: string | undefined, path: "/command" | "/api/command"): Promise<SkillCommandLookup> {
70+
const url = new URL(path, serverUrl)
71+
if (directory) {
72+
if (path === "/api/command") {
73+
url.searchParams.set("location[directory]", directory)
74+
} else {
75+
url.searchParams.set("directory", directory)
76+
}
77+
}
78+
const controller = new AbortController()
79+
const timeout = setTimeout(() => controller.abort(), 1_000)
80+
try {
81+
const response = await fetch(url, { signal: controller.signal })
82+
if (!response.ok) return { ok: false, commands: [] }
83+
return { ok: true, commands: commandsFromPayload(await response.json()) }
84+
} finally {
85+
clearTimeout(timeout)
86+
}
87+
}
88+
89+
export function createSkillCommandResolver(input: {
90+
client: CommandListClient
91+
serverUrl: URL
92+
directory?: string
93+
log: PluginLogger
94+
}) {
95+
let lastRefresh = 0
96+
let refreshPromise: Promise<void> | undefined
97+
const skillCommands = new Map<string, SkillCommandInfo>()
98+
99+
const refresh = async (force = false) => {
100+
if (!force && Date.now() - lastRefresh < 30_000) return
101+
if (refreshPromise) return refreshPromise
102+
refreshPromise = (async () => {
103+
const next = new Map<string, SkillCommandInfo>()
104+
let catalogLoaded = false
105+
try {
106+
const lookup = await clientSkillCommands(input.client, input.directory)
107+
catalogLoaded = catalogLoaded || lookup.ok
108+
for (const command of lookup.commands) {
109+
next.set(command.name, command)
110+
}
111+
} catch (err) {
112+
await input.log("debug", "otel: command catalog lookup failed", {
113+
error: err instanceof Error ? err.message : String(err),
114+
})
115+
}
116+
if (next.size === 0) {
117+
try {
118+
const lookup = await rawSkillCommands(input.serverUrl, input.directory, "/command")
119+
catalogLoaded = catalogLoaded || lookup.ok
120+
for (const command of lookup.commands) {
121+
next.set(command.name, command)
122+
}
123+
} catch (err) {
124+
await input.log("debug", "otel: raw command catalog lookup failed", {
125+
error: err instanceof Error ? err.message : String(err),
126+
})
127+
}
128+
}
129+
if (next.size === 0) {
130+
try {
131+
const lookup = await rawSkillCommands(input.serverUrl, input.directory, "/api/command")
132+
catalogLoaded = catalogLoaded || lookup.ok
133+
for (const command of lookup.commands) {
134+
next.set(command.name, command)
135+
}
136+
} catch (err) {
137+
await input.log("debug", "otel: v2 command catalog lookup failed", {
138+
error: err instanceof Error ? err.message : String(err),
139+
})
140+
}
141+
}
142+
if (!catalogLoaded) {
143+
await input.log("debug", "otel: skill command catalog refresh skipped")
144+
return
145+
}
146+
skillCommands.clear()
147+
for (const [name, command] of next) skillCommands.set(name, command)
148+
lastRefresh = Date.now()
149+
await input.log("debug", "otel: skill command catalog refreshed", { count: skillCommands.size })
150+
})().finally(() => {
151+
refreshPromise = undefined
152+
})
153+
return refreshPromise
154+
}
155+
156+
return {
157+
refresh,
158+
resolve: async (command: string) => {
159+
let skill = skillCommands.get(command)
160+
if (skill) return skill
161+
await refresh(false)
162+
skill = skillCommands.get(command)
163+
return skill
164+
},
165+
}
166+
}
167+
168+
export async function handleCommandExecuteBefore(
169+
input: CommandExecuteInput,
170+
ctx: HandlerContext,
171+
resolveSkillCommand: (command: string) => Promise<SkillCommandInfo | undefined>,
172+
) {
173+
const skill = await resolveSkillCommand(input.command)
174+
if (!skill) return
175+
176+
const attrs = {
177+
...ctx.commonAttrs,
178+
"session.id": input.sessionID,
179+
skill_name: skill.name,
180+
command_name: input.command,
181+
...(skill.agent ? { agent: skill.agent } : {}),
182+
...(skill.subtask !== undefined ? { subtask: skill.subtask } : {}),
183+
}
184+
185+
if (isMetricEnabled("skill.count", ctx)) {
186+
ctx.instruments.skillCounter.add(1, attrs)
187+
}
188+
189+
ctx.emitLog({
190+
severityNumber: SeverityNumber.INFO,
191+
severityText: "INFO",
192+
timestamp: Date.now(),
193+
observedTimestamp: Date.now(),
194+
body: "skill_invoked",
195+
attributes: {
196+
"event.name": "skill_invoked",
197+
...attrs,
198+
arguments_length: input.arguments.length,
199+
...(skill.description ? { description: skill.description } : {}),
200+
},
201+
})
202+
203+
return ctx.log("info", "otel: skill_invoked", {
204+
sessionID: input.sessionID,
205+
skill_name: skill.name,
206+
command_name: input.command,
207+
})
208+
}

src/index.ts

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import { handleSessionCreated, handleSessionIdle, handleSessionError, handleSess
2525
import { handleMessageUpdated, handleMessagePartUpdated, startMessageSpan } from "./handlers/message.ts"
2626
import { handlePermissionUpdated, handlePermissionReplied } from "./handlers/permission.ts"
2727
import { handleSessionDiff, handleCommandExecuted } from "./handlers/activity.ts"
28+
import { createSkillCommandResolver, handleCommandExecuteBefore } from "./handlers/skill.ts"
2829

2930
const PLUGIN_VERSION: string = (pkg as { version?: string }).version ?? "unknown"
3031

@@ -33,7 +34,7 @@ const PLUGIN_VERSION: string = (pkg as { version?: string }).version ?? "unknown
3334
* Instruments metrics (sessions, tokens, cost, lines of code, commits, tool durations)
3435
* and structured log events. All instrumentation is gated on `OPENCODE_ENABLE_TELEMETRY`.
3536
*/
36-
export const OtelPlugin: Plugin = async ({ project, client, directory, worktree }) => {
37+
export const OtelPlugin: Plugin = async ({ project, client, directory, worktree, serverUrl }) => {
3738
const config = loadConfig()
3839
const otlpHeadersHelper = resolveHelperPath(config.otlpHeadersHelper, directory, worktree)
3940
let minLevel: Level = "info"
@@ -139,6 +140,8 @@ export const OtelPlugin: Plugin = async ({ project, client, directory, worktree
139140
sessionInputs,
140141
messageOutputs,
141142
}
143+
const skillCommands = createSkillCommandResolver({ client, serverUrl, directory: worktree ?? directory, log })
144+
await skillCommands.refresh(true)
142145

143146
async function shutdown() {
144147
await Promise.allSettled([meterProvider.shutdown(), loggerProvider.shutdown(), tracerProvider.shutdown()])
@@ -217,6 +220,10 @@ export const OtelPlugin: Plugin = async ({ project, client, directory, worktree
217220
})
218221
}),
219222

223+
"command.execute.before": safe("command.execute.before", async (input) => {
224+
await handleCommandExecuteBefore(input, ctx, skillCommands.resolve)
225+
}),
226+
220227
event: safe("event", async ({ event }) => {
221228
switch (event.type) {
222229
case "session.created":

src/otel.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,5 +209,9 @@ export function createInstruments(prefix: string): Instruments {
209209
unit: "{subtask}",
210210
description: "Number of sub-agent invocations observed via subtask message parts",
211211
}),
212+
skillCounter: meter.createCounter(`${prefix}skill.count`, {
213+
unit: "{skill}",
214+
description: "Number of skill command invocations observed via command execution hooks",
215+
}),
212216
}
213217
}

src/types.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ export type Instruments = {
5252
modelUsageCounter: Counter
5353
retryCounter: Counter
5454
subtaskCounter: Counter
55+
skillCounter: Counter
5556
}
5657

5758
/** Accumulated per-session totals used for gauge snapshots on session.idle. */

tests/handlers/disabled-metrics.test.ts

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { describe, test, expect } from "bun:test"
22
import { handleSessionCreated, handleSessionIdle, handleSessionStatus } from "../../src/handlers/session.ts"
33
import { handleMessageUpdated, handleMessagePartUpdated } from "../../src/handlers/message.ts"
44
import { handleSessionDiff, handleCommandExecuted } from "../../src/handlers/activity.ts"
5+
import { handleCommandExecuteBefore } from "../../src/handlers/skill.ts"
56
import { makeCtx } from "../helpers.ts"
67
import type { EventSessionCreated, EventSessionIdle, EventSessionStatus, EventMessageUpdated, EventMessagePartUpdated, EventSessionDiff, EventCommandExecuted } from "@opencode-ai/sdk"
78

@@ -226,13 +227,35 @@ describe("OPENCODE_DISABLE_METRICS", () => {
226227
})
227228
})
228229

230+
describe("skill.count disabled", () => {
231+
test("does not increment skill counter", async () => {
232+
const { ctx, counters } = makeCtx("proj_test", ["skill.count"])
233+
await handleCommandExecuteBefore(
234+
{ command: "review", sessionID: "ses_1", arguments: "args" },
235+
ctx,
236+
async () => ({ name: "review" }),
237+
)
238+
expect(counters.skill.calls).toHaveLength(0)
239+
})
240+
241+
test("still emits skill_invoked log record", async () => {
242+
const { ctx, logger } = makeCtx("proj_test", ["skill.count"])
243+
await handleCommandExecuteBefore(
244+
{ command: "review", sessionID: "ses_1", arguments: "args" },
245+
ctx,
246+
async () => ({ name: "review" }),
247+
)
248+
expect(logger.records.at(0)!.body).toBe("skill_invoked")
249+
})
250+
})
251+
229252
describe("multiple disabled at once", () => {
230253
test("disabling all metrics stops all counter/histogram calls", async () => {
231254
const all = [
232255
"session.count", "token.usage", "cost.usage", "lines_of_code.count",
233256
"commit.count", "tool.duration", "cache.count", "session.duration",
234257
"message.count", "session.token.total", "session.cost.total",
235-
"model.usage", "retry.count", "subtask.count",
258+
"model.usage", "retry.count", "subtask.count", "skill.count",
236259
]
237260
const { ctx, counters, histograms, gauges } = makeCtx("proj_test", all)
238261
const subtaskEvent = {
@@ -251,6 +274,11 @@ describe("OPENCODE_DISABLE_METRICS", () => {
251274
await handleMessagePartUpdated(makeToolPart("running"), ctx)
252275
await handleMessagePartUpdated(makeToolPart("completed"), ctx)
253276
await handleMessagePartUpdated(subtaskEvent, ctx)
277+
await handleCommandExecuteBefore(
278+
{ command: "review", sessionID: "ses_1", arguments: "args" },
279+
ctx,
280+
async () => ({ name: "review" }),
281+
)
254282

255283
expect(counters.session.calls).toHaveLength(0)
256284
expect(counters.token.calls).toHaveLength(0)
@@ -262,6 +290,7 @@ describe("OPENCODE_DISABLE_METRICS", () => {
262290
expect(counters.lines.calls).toHaveLength(0)
263291
expect(counters.commit.calls).toHaveLength(0)
264292
expect(counters.subtask.calls).toHaveLength(0)
293+
expect(counters.skill.calls).toHaveLength(0)
265294
expect(histograms.tool.calls).toHaveLength(0)
266295
expect(histograms.sessionDuration.calls).toHaveLength(0)
267296
expect(gauges.sessionToken.calls).toHaveLength(0)

0 commit comments

Comments
 (0)