Skip to content

Commit 43fbe35

Browse files
chubes4chubes4
authored
Project tool observability into reviewer transcripts (#1809)
1 parent b8c4cfe commit 43fbe35

6 files changed

Lines changed: 249 additions & 3 deletions

File tree

‎.github/scripts/run-agent-task/execute-native-agent-task.mjs‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,56 @@ function record(value) {
139139
return value && typeof value === "object" && !Array.isArray(value) ? value : {}
140140
}
141141

142+
function canonicalToolObservability(metadata) {
143+
const source = record(record(record(metadata).agents_api).tool_observability)
144+
if (source.version !== 1 || !Array.isArray(source.calls) || source.calls.length > 64) return undefined
145+
const calls = source.calls.map(projectCanonicalToolCall).filter(Boolean)
146+
return calls.length ? { version: 1, calls } : undefined
147+
}
148+
149+
function projectCanonicalToolCall(value) {
150+
const call = record(value)
151+
const argumentsSummary = record(call.arguments)
152+
const keys = Array.isArray(argumentsSummary.keys) ? argumentsSummary.keys : []
153+
if (!Number.isSafeInteger(call.sequence) || call.sequence < 1 || !Number.isSafeInteger(call.turn) || call.turn < 1
154+
|| !safeToolIdentifier(call.tool_call_id) || !safeToolIdentifier(call.tool_name)
155+
|| !["succeeded", "failed", "rejected", "pending"].includes(call.status)
156+
|| argumentsSummary.redacted !== true || !Number.isSafeInteger(argumentsSummary.count) || argumentsSummary.count < 0
157+
|| argumentsSummary.count !== keys.length || keys.length > 32 || !keys.every(safeToolIdentifier)) return undefined
158+
const resultSummary = projectCanonicalToolResult(call.result)
159+
if (call.result !== undefined && !resultSummary) return undefined
160+
return Object.fromEntries(Object.entries({
161+
sequence: call.sequence, turn: call.turn, tool_call_id: call.tool_call_id, tool_name: call.tool_name, status: call.status,
162+
arguments: { keys, count: argumentsSummary.count, redacted: true }, result: resultSummary,
163+
error: call.status === "failed" ? { code: "tool_call_failed", message: "Tool call failed." }
164+
: call.status === "rejected" ? { code: "tool_call_rejected", message: "Tool call was rejected." } : undefined,
165+
}).filter(([, item]) => item !== undefined))
166+
}
167+
168+
function projectCanonicalToolResult(value) {
169+
const result = record(value)
170+
if (Object.keys(result).length === 0) return undefined
171+
if (["array", "object"].includes(result.type)) return Number.isSafeInteger(result.count) && result.count >= 0 ? { type: result.type, count: result.count } : undefined
172+
if (result.type === "string") return Number.isSafeInteger(result.size) && result.size >= 0 ? { type: result.type, size: result.size } : undefined
173+
return ["integer", "double", "boolean", "null"].includes(result.type) ? { type: result.type } : undefined
174+
}
175+
176+
function safeToolIdentifier(value) {
177+
return typeof value === "string" && value.length <= 256 && /^[A-Za-z0-9][A-Za-z0-9._:/-]*$/.test(value)
178+
}
179+
180+
function removeRawToolObservability(metadata) {
181+
const visit = (value) => {
182+
if (Array.isArray(value)) return value.forEach(visit)
183+
const entry = record(value)
184+
if (!Object.keys(entry).length) return
185+
const agentsApi = record(record(entry.metadata).agents_api)
186+
delete agentsApi.tool_observability
187+
Object.values(entry).forEach(visit)
188+
}
189+
visit(metadata)
190+
}
191+
142192
function string(value) {
143193
return typeof value === "string" ? value.trim() : ""
144194
}
@@ -519,7 +569,17 @@ const nativeRuntimeResult = request.run_agent && !request.dry_run
519569
: {}
520570
await rm(nativeResultPath, { force: true })
521571
reviewerEvidence = await canonicalReviewerTranscript(nativeRuntimeResult, artifactsPath)
572+
const toolObservability = canonicalToolObservability(record(nativeRuntimeResult).metadata)
573+
?? canonicalToolObservability(record(record(nativeRuntimeResult).agent_task_run_result).metadata)
522574
let runtimeResult = sanitizeRuntimeSourceValue(nativeRuntimeResult, privateRuntimeSourceRootForSanitization)
575+
removeRawToolObservability(runtimeResult)
576+
const normalizedAgentTaskResult = record(record(runtimeResult).agent_task_run_result)
577+
if (toolObservability) {
578+
normalizedAgentTaskResult.metadata = {
579+
...record(normalizedAgentTaskResult.metadata),
580+
tool_observability: toolObservability,
581+
}
582+
}
523583
assertNoRuntimeSourcePaths(runtimeResult, privateRuntimeSourceRootForSanitization)
524584
let workspaceApply = { status: "no-op", changedFiles: [] }
525585
let runnerWorkspaceCore = null

‎.github/scripts/run-agent-task/prepare-agent-task-upload.mjs‎

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import { assertNoRuntimeSourcePaths, sanitizeRuntimeSourceJson } from "./runtime
88
const MAX_UPLOAD_FILE_BYTES = 4 * 1024 * 1024
99
const MAX_TRANSCRIPT_EXECUTIONS = 64
1010
const MAX_REVIEW_TEXT_BYTES = 32 * 1024
11+
const MAX_TOOL_ARGUMENT_KEYS = 32
1112
const workspace = resolve(process.env.AGENT_TASK_WORKSPACE || process.cwd())
1213
const uploadPath = resolve(process.env.AGENT_TASK_UPLOAD_PATH || join(workspace, ".codebox", "agent-task-upload"))
1314
const requestPath = resolve(process.env.AGENT_TASK_REQUEST_PATH || join(workspace, ".codebox", "agent-task-request.json"))
@@ -208,15 +209,64 @@ function projectParsed(value) {
208209
const results = Array.isArray(parsed.tool_results) ? parsed.tool_results : Array.isArray(parsed.toolResults) ? parsed.toolResults : []
209210
const errors = Array.isArray(parsed.errors) ? parsed.errors : parsed.error ? [parsed.error] : []
210211
const agent = record(parsed.agent ?? parsed.agent_metadata)
212+
const toolObservability = canonicalToolObservability(parsed.metadata)
213+
?? canonicalToolObservability(record(parsed.agent_runtime).result?.metadata)
211214
return Object.fromEntries(Object.entries({
212-
agent: Object.fromEntries(["id", "name", "model", "provider", "status"].flatMap((key) => boundedText(agent[key]) ? [[key, boundedText(agent[key])]] : [])),
215+
agent: Object.fromEntries(["id", "name", "status"].flatMap((key) => boundedText(agent[key]) ? [[key, boundedText(agent[key])]] : [])),
213216
model_messages: messages.slice(0, MAX_TRANSCRIPT_EXECUTIONS).flatMap((message) => boundedText(record(message).content ?? record(message).text ?? message) ? [{ role: boundedText(record(message).role), content: boundedText(record(message).content ?? record(message).text ?? message) }] : []),
214217
tool_calls: tools.slice(0, MAX_TRANSCRIPT_EXECUTIONS).map(projectToolCall),
215218
tool_results: results.slice(0, MAX_TRANSCRIPT_EXECUTIONS).map(projectToolCall),
216219
errors: errors.slice(0, MAX_TRANSCRIPT_EXECUTIONS).flatMap((error) => boundedText(record(error).message ?? error) ? [boundedText(record(error).message ?? error)] : []),
220+
tool_observability: toolObservability,
217221
}).filter(([, item]) => Array.isArray(item) ? item.length > 0 : Object.keys(item).length > 0))
218222
}
219223

224+
// This is deliberately limited to the public Agents API summary. Tool payloads
225+
// and provider-specific tool records are not inputs to reviewer artifacts.
226+
function canonicalToolObservability(metadata) {
227+
const source = record(record(record(metadata).agents_api).tool_observability)
228+
if (source.version !== 1 || !Array.isArray(source.calls) || source.calls.length > MAX_TRANSCRIPT_EXECUTIONS) return undefined
229+
const calls = source.calls.map(projectCanonicalToolCall).filter(Boolean)
230+
return calls.length ? { version: 1, calls } : undefined
231+
}
232+
233+
function projectCanonicalToolCall(value) {
234+
const call = record(value)
235+
const argumentsSummary = record(call.arguments)
236+
const keys = Array.isArray(argumentsSummary.keys) ? argumentsSummary.keys : []
237+
if (!Number.isSafeInteger(call.sequence) || call.sequence < 1 || !Number.isSafeInteger(call.turn) || call.turn < 1
238+
|| !safeToolIdentifier(call.tool_call_id) || !safeToolIdentifier(call.tool_name)
239+
|| !["succeeded", "failed", "rejected", "pending"].includes(call.status)
240+
|| argumentsSummary.redacted !== true || !Number.isSafeInteger(argumentsSummary.count) || argumentsSummary.count < 0
241+
|| argumentsSummary.count !== keys.length || keys.length > MAX_TOOL_ARGUMENT_KEYS || !keys.every(safeToolIdentifier)) return undefined
242+
const resultSummary = projectCanonicalToolResult(call.result)
243+
if (call.result !== undefined && !resultSummary) return undefined
244+
return Object.fromEntries(Object.entries({
245+
sequence: call.sequence,
246+
turn: call.turn,
247+
tool_call_id: call.tool_call_id,
248+
tool_name: call.tool_name,
249+
status: call.status,
250+
arguments: { keys, count: argumentsSummary.count, redacted: true },
251+
result: resultSummary,
252+
error: call.status === "failed" ? { code: "tool_call_failed", message: "Tool call failed." }
253+
: call.status === "rejected" ? { code: "tool_call_rejected", message: "Tool call was rejected." }
254+
: undefined,
255+
}).filter(([, item]) => item !== undefined))
256+
}
257+
258+
function projectCanonicalToolResult(value) {
259+
const result = record(value)
260+
if (Object.keys(result).length === 0) return undefined
261+
if (["array", "object"].includes(result.type)) return Number.isSafeInteger(result.count) && result.count >= 0 ? { type: result.type, count: result.count } : undefined
262+
if (result.type === "string") return Number.isSafeInteger(result.size) && result.size >= 0 ? { type: result.type, size: result.size } : undefined
263+
return ["integer", "double", "boolean", "null"].includes(result.type) ? { type: result.type } : undefined
264+
}
265+
266+
function safeToolIdentifier(value) {
267+
return typeof value === "string" && value.length <= 256 && /^[A-Za-z0-9][A-Za-z0-9._:/-]*$/.test(value)
268+
}
269+
220270
async function trustedTranscriptFile(path) {
221271
const root = await realpath(artifactsPath).catch(() => "")
222272
if (!root) return { unavailable: "artifact-root-missing" }

‎packages/runtime-core/src/agent-task-run-result.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { isPlainObject, numberValue, objectValue, stringValue, stripUndefined }
22
import { normalizeAgentTerminalResult, type AgentTerminalResult } from "./agent-terminal-result.js"
33
import { RUNTIME_ACCESS_SCHEMA, normalizeRuntimeAccess, type RuntimeAccess } from "./runtime-boundary-contracts.js"
44
import { normalizeAgentTaskStatus } from "./status-taxonomy.js"
5+
import { normalizeToolObservability } from "./tool-observability.js"
56

67
export const AGENT_TASK_RUN_RESULT_SCHEMA = "wp-codebox/agent-task-run-result/v1" as const
78

@@ -137,6 +138,7 @@ export function normalizeAgentTaskRunResult(raw: unknown, options: AgentTaskRunR
137138
provider_error: objectValue(result.provider_error),
138139
timeout: result.timeout === true ? true : undefined,
139140
failure_evidence: objectValue(result.failure_evidence),
141+
tool_observability: normalizeToolObservability(result.metadata) ?? normalizeToolObservability(agentResult.metadata),
140142
}),
141143
terminal_result: terminalResult,
142144
runtime_access: agentTaskRuntimeAccess(result),

‎packages/runtime-core/src/tool-observability.ts‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
import { isPlainObject, objectValue, stripUndefined } from "./object-utils.js"
2+
3+
export const TOOL_OBSERVABILITY_VERSION = 1 as const
4+
const MAX_TOOL_CALLS = 64
5+
const MAX_IDENTIFIER_LENGTH = 256
6+
const MAX_ARGUMENT_KEYS = 32
7+
const RESULT_TYPES = ["array", "object", "string", "integer", "double", "boolean", "null"] as const
8+
type ResultType = typeof RESULT_TYPES[number]
9+
10+
export interface ToolObservabilityCall {
11+
sequence: number
12+
turn: number
13+
tool_call_id: string
14+
tool_name: string
15+
status: "succeeded" | "failed" | "rejected" | "pending"
16+
arguments: { keys: string[], count: number, redacted: true }
17+
result?: { type: ResultType, count?: number, size?: number }
18+
error?: { code: string, message: string }
19+
}
20+
21+
export interface ToolObservability {
22+
version: typeof TOOL_OBSERVABILITY_VERSION
23+
calls: ToolObservabilityCall[]
24+
}
25+
26+
/** Projects the public Agents API lifecycle without retaining tool payloads. */
27+
export function normalizeToolObservability(metadata: unknown): ToolObservability | undefined {
28+
const observability = objectValue(objectValue(metadata).agents_api).tool_observability
29+
const source = objectValue(observability)
30+
if (source.version !== TOOL_OBSERVABILITY_VERSION || !Array.isArray(source.calls) || source.calls.length > MAX_TOOL_CALLS) return undefined
31+
32+
const calls = source.calls.map(projectCall).filter((call): call is ToolObservabilityCall => call !== undefined)
33+
return calls.length > 0 ? { version: TOOL_OBSERVABILITY_VERSION, calls } : undefined
34+
}
35+
36+
function projectCall(value: unknown): ToolObservabilityCall | undefined {
37+
if (!isPlainObject(value)) return undefined
38+
const call = objectValue(value)
39+
const status = call.status
40+
const argumentsSummary = objectValue(call.arguments)
41+
const keys = Array.isArray(argumentsSummary.keys) ? argumentsSummary.keys : []
42+
const count = argumentsSummary.count
43+
if (!isPositiveInteger(call.sequence) || !isPositiveInteger(call.turn)
44+
|| !safeIdentifier(call.tool_call_id) || !safeIdentifier(call.tool_name)
45+
|| !["succeeded", "failed", "rejected", "pending"].includes(String(status))
46+
|| argumentsSummary.redacted !== true || !isNonNegativeInteger(count) || count !== keys.length || keys.length > MAX_ARGUMENT_KEYS
47+
|| !keys.every(safeIdentifier)) return undefined
48+
49+
const result = projectResult(call.result)
50+
if (call.result !== undefined && !result) return undefined
51+
return stripUndefined({
52+
sequence: call.sequence,
53+
turn: call.turn,
54+
tool_call_id: call.tool_call_id,
55+
tool_name: call.tool_name,
56+
status: status as ToolObservabilityCall["status"],
57+
arguments: { keys, count, redacted: true },
58+
result,
59+
error: status === "failed" ? { code: "tool_call_failed", message: "Tool call failed." }
60+
: status === "rejected" ? { code: "tool_call_rejected", message: "Tool call was rejected." }
61+
: undefined,
62+
}) as ToolObservabilityCall
63+
}
64+
65+
function projectResult(value: unknown): ToolObservabilityCall["result"] | undefined {
66+
const result = objectValue(value)
67+
if (Object.keys(result).length === 0) return undefined
68+
if (!isResultType(result.type)) return undefined
69+
if (result.type === "array" || result.type === "object") {
70+
return isNonNegativeInteger(result.count) ? { type: result.type, count: result.count } : undefined
71+
}
72+
if (result.type === "string") {
73+
return isNonNegativeInteger(result.size) ? { type: result.type, size: result.size } : undefined
74+
}
75+
return { type: result.type }
76+
}
77+
78+
function safeIdentifier(value: unknown): value is string {
79+
return typeof value === "string" && value.length <= MAX_IDENTIFIER_LENGTH && /^[A-Za-z0-9][A-Za-z0-9._:/-]*$/.test(value)
80+
}
81+
82+
function isResultType(value: unknown): value is ResultType {
83+
return typeof value === "string" && RESULT_TYPES.includes(value as ResultType)
84+
}
85+
86+
function isPositiveInteger(value: unknown): value is number {
87+
return typeof value === "number" && Number.isSafeInteger(value) && value > 0
88+
}
89+
90+
function isNonNegativeInteger(value: unknown): value is number {
91+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0
92+
}

‎tests/agent-task-contracts.test.ts‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,41 @@ assert.equal(succeeded.schema, AGENT_TASK_RUN_RESULT_SCHEMA)
1818
assert.equal(succeeded.status, "succeeded")
1919
assert.equal(agentTaskRunExitCode({ success: true, agent_task_run_result: succeeded }), 0)
2020

21+
const toolObservability = normalizeAgentTaskRunResult({
22+
success: true,
23+
metadata: {
24+
agents_api: {
25+
tool_observability: {
26+
version: 1,
27+
calls: [
28+
{ sequence: 1, turn: 1, tool_call_id: "call-success", tool_name: "workspace.read", status: "succeeded", arguments: { keys: ["path"], count: 1, redacted: true }, result: { type: "object", count: 2 }, result_body: "secret-sentinel" },
29+
{ sequence: 2, turn: 1, tool_call_id: "call-failure", tool_name: "workspace.write", status: "failed", arguments: { keys: [], count: 0, redacted: true }, error: { code: "raw-secret", message: "secret-sentinel" } },
30+
{ sequence: 3, turn: 2, tool_call_id: "call-rejected", tool_name: "workspace.delete", status: "rejected", arguments: { keys: ["path"], count: 1, redacted: true } },
31+
{ sequence: 4, turn: 2, tool_call_id: "call-pending", tool_name: "workspace.grep", status: "pending", arguments: { keys: ["query"], count: 1, redacted: true } },
32+
{ sequence: 5, turn: 2, tool_call_id: "malformed", tool_name: "workspace.read", status: "succeeded", arguments: { keys: ["path"], count: 2, redacted: true } },
33+
{ sequence: 6, turn: 2, tool_call_id: "untrusted secret sentinel", tool_name: "workspace.read", status: "pending", arguments: { keys: [], count: 0, redacted: true } },
34+
{ sequence: 7, turn: 2, tool_call_id: "call-secret-name", tool_name: "untrusted secret sentinel", status: "pending", arguments: { keys: [], count: 0, redacted: true } },
35+
{ sequence: 8, turn: 2, tool_call_id: "call-secret-key", tool_name: "workspace.read", status: "pending", arguments: { keys: ["untrusted secret sentinel"], count: 1, redacted: true } },
36+
{ sequence: 9, turn: 2, tool_call_id: "call-secret-type", tool_name: "workspace.read", status: "succeeded", arguments: { keys: [], count: 0, redacted: true }, result: { type: "untrusted secret sentinel", count: 1 } },
37+
],
38+
},
39+
},
40+
},
41+
})
42+
assert.deepEqual((toolObservability.metadata.tool_observability as any)?.calls.map((call: any) => call.status), ["succeeded", "failed", "rejected", "pending"])
43+
assert.deepEqual((toolObservability.metadata.tool_observability as any)?.calls[0].result, { type: "object", count: 2 })
44+
assert.deepEqual((toolObservability.metadata.tool_observability as any)?.calls[1].error, { code: "tool_call_failed", message: "Tool call failed." })
45+
assert.equal(JSON.stringify(toolObservability).includes("secret-sentinel"), false, "normalized tool observability never retains payloads or raw errors")
46+
assert.equal(JSON.stringify(toolObservability).includes("untrusted secret sentinel"), false, "normalized tool observability rejects non-identifier summary strings")
47+
for (const resultShape of [{ type: "string", size: 12 }, { type: "integer" }]) {
48+
const normalized = normalizeAgentTaskRunResult({ metadata: { agents_api: { tool_observability: { version: 1, calls: [{ sequence: 1, turn: 1, tool_call_id: "call-shape", tool_name: "workspace.read", status: "succeeded", arguments: { keys: [], count: 0, redacted: true }, result: resultShape }] } } } })
49+
assert.deepEqual((normalized.metadata.tool_observability as any)?.calls[0].result, resultShape)
50+
}
51+
for (const unsupported of [0, 2, "1"]) {
52+
const result = normalizeAgentTaskRunResult({ metadata: { agents_api: { tool_observability: { version: unsupported, calls: [{ sequence: 1, turn: 1, tool_call_id: "call", tool_name: "tool", status: "pending", arguments: { keys: [], count: 0, redacted: true } }] } } } })
53+
assert.equal("tool_observability" in result.metadata, false, "only canonical version 1 tool observability is consumed")
54+
}
55+
2156
const resultFileDirectory = mkdtempSync(join(tmpdir(), "wp-codebox-agent-task-result-file-"))
2257
const resultFilePath = join(resultFileDirectory, "result.json")
2358
const atomicResult = { schema: "wp-codebox/agent-task-run/v1", success: true, status: "succeeded", padding: "x".repeat(40 * 1024) }

0 commit comments

Comments
 (0)