Skip to content

Commit 78e2dd4

Browse files
chubes4chubes4
authored
Expose compact tool observability in chat metadata (#428)
1 parent fbd5641 commit 78e2dd4

9 files changed

Lines changed: 314 additions & 7 deletions

‎composer.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@
8383
"php tests/provider-turn-adapter-smoke.php",
8484
"php tests/default-provider-turn-adapter-smoke.php",
8585
"php tests/default-agents-chat-handler-smoke.php",
86+
"php tests/tool-observability-smoke.php",
8687
"php tests/ability-tool-executor-smoke.php",
8788
"php tests/ability-tool-ceiling-smoke.php",
8889
"php tests/ability-tool-capability-denied-event-smoke.php",

‎src/Channels/register-agents-chat-ability.php‎

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -565,7 +565,53 @@ function agents_chat_output_schema(): array {
565565
),
566566
'metadata' => array(
567567
'type' => 'object',
568-
'description' => 'Runtime-specific metadata (token usage, model, latency, tool calls). Opaque to the dispatcher.',
568+
'description' => 'Runtime metadata. The default handler exposes `agents_api.tool_observability`, a content-redacted v1 tool lifecycle contract.',
569+
'properties' => array(
570+
'agents_api' => array(
571+
'type' => 'object',
572+
'properties' => array(
573+
'tool_observability' => array(
574+
'type' => 'object',
575+
'description' => 'Additive content-redacted tool observability. Calls are globally ordered and expose no argument values, result content, hashes, raw errors, paths, or arbitrary runtime metadata.',
576+
'required' => array( 'version', 'calls' ),
577+
'properties' => array(
578+
'version' => array( 'type' => 'integer', 'enum' => array( 1 ) ),
579+
'calls' => array(
580+
'type' => 'array',
581+
'items' => array(
582+
'type' => 'object',
583+
'required' => array( 'sequence', 'turn', 'tool_call_id', 'tool_name', 'status', 'arguments' ),
584+
'properties' => array(
585+
'sequence' => array( 'type' => 'integer' ),
586+
'turn' => array( 'type' => 'integer' ),
587+
'tool_call_id' => array( 'type' => 'string' ),
588+
'tool_name' => array( 'type' => 'string' ),
589+
'status' => array( 'type' => 'string', 'enum' => array( 'pending', 'succeeded', 'failed', 'rejected' ) ),
590+
'arguments' => array(
591+
'type' => 'object',
592+
'required' => array( 'keys', 'count', 'redacted' ),
593+
'properties' => array(
594+
'keys' => array( 'type' => 'array', 'items' => array( 'type' => 'string' ) ),
595+
'count' => array( 'type' => 'integer' ),
596+
'redacted' => array( 'type' => 'boolean', 'enum' => array( true ) ),
597+
),
598+
),
599+
'result' => array(
600+
'type' => 'object',
601+
'description' => 'Optional result shape only: type plus count for containers or byte size for strings.',
602+
),
603+
'error' => array(
604+
'type' => 'object',
605+
'description' => 'Agents API-owned normalized code and fixed safe message; never the executor or provider error.',
606+
),
607+
),
608+
),
609+
),
610+
),
611+
),
612+
),
613+
),
614+
),
569615
),
570616
),
571617
);

‎src/Channels/register-default-agents-chat-handler.php‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -566,10 +566,11 @@ private static function session_metadata( array $result ): array {
566566
private static function to_canonical_output( string $session_id, array $result ): array {
567567
$metadata = array_filter(
568568
array(
569-
'status' => is_string( $result['status'] ?? null ) ? $result['status'] : null,
570-
'turn_count' => isset( $result['turn_count'] ) ? self::int_value( $result['turn_count'] ) : null,
571-
'usage' => is_array( $result['usage'] ?? null ) ? $result['usage'] : null,
572-
'run_outcome' => is_array( $result['run_outcome'] ?? null ) ? $result['run_outcome'] : null,
569+
'status' => is_string( $result['status'] ?? null ) ? $result['status'] : null,
570+
'turn_count' => isset( $result['turn_count'] ) ? self::int_value( $result['turn_count'] ) : null,
571+
'usage' => is_array( $result['usage'] ?? null ) ? $result['usage'] : null,
572+
'run_outcome' => is_array( $result['run_outcome'] ?? null ) ? $result['run_outcome'] : null,
573+
'tool_observability' => is_array( $result['tool_observability'] ?? null ) ? $result['tool_observability'] : null,
573574
),
574575
static fn( $value ): bool => null !== $value
575576
);

‎src/Runtime/class-wp-agent-conversation-loop.php‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -880,8 +880,9 @@ public static function mediate_tool_calls(
880880
$tool_call_id,
881881
$turn,
882882
array(
883-
'status' => ! empty( $exec_result['success'] ) ? 'success' : 'error',
884-
'success' => (bool) ( $exec_result['success'] ?? false ),
883+
'status' => ! empty( $exec_result['success'] ) ? 'success' : 'error',
884+
'success' => (bool) ( $exec_result['success'] ?? false ),
885+
'rejected' => 'reject' === $mediator_decision['action'],
885886
)
886887
);
887888

‎src/Runtime/class-wp-agent-conversation-result.php‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ class WP_Agent_Conversation_Result {
3030
public const OUTCOME_STOP_NATURAL = WP_Agent_Run_Outcome::STOP_NATURAL;
3131
public const OUTCOME_STOP_MAX_TURNS = WP_Agent_Run_Outcome::STOP_MAX_TURNS;
3232
public const OUTCOME_STOP_PROVIDER_ERROR = WP_Agent_Run_Outcome::STOP_PROVIDER_ERROR;
33+
public const TOOL_OBSERVABILITY_VERSION = 1;
3334

3435
/**
3536
* Validate and normalize a loop result.
@@ -252,11 +253,141 @@ public static function normalize( array $result ): array {
252253
throw self::invalid( 'runtime_tool_pending', 'must be an array when present' );
253254
}
254255

256+
$result['tool_observability'] = self::tool_observability( $result['tool_events'], $result['tool_execution_results'] );
255257
$result['run_outcome'] = WP_Agent_Run_Outcome::normalize( $result['run_outcome'] ?? null, $result );
256258

257259
return $result;
258260
}
259261

262+
/**
263+
* Project internal tool lifecycle data to a compact content-redacted contract.
264+
*
265+
* @param array<mixed> $events Ordered tool lifecycle events.
266+
* @param array<mixed> $results Tool execution results.
267+
* @return array{version:int,calls:array<int,array<string,mixed>>}
268+
*/
269+
private static function tool_observability( array $events, array $results ): array {
270+
$calls = array();
271+
$pending = array();
272+
$result_uses = array();
273+
274+
foreach ( $events as $event ) {
275+
if ( ! is_array( $event ) ) {
276+
continue;
277+
}
278+
279+
$type = $event['type'] ?? '';
280+
$tool_call_id = $event['tool_call_id'] ?? '';
281+
if ( ! is_string( $type ) || ! is_string( $tool_call_id ) || '' === $tool_call_id ) {
282+
continue;
283+
}
284+
285+
if ( 'tool_call' === $type ) {
286+
$metadata = is_array( $event['metadata'] ?? null ) ? $event['metadata'] : array();
287+
$parameters = is_array( $metadata['parameters'] ?? null ) ? $metadata['parameters'] : array();
288+
$keys = array_map( 'strval', array_keys( $parameters ) );
289+
$calls[] = array(
290+
'sequence' => count( $calls ) + 1,
291+
'turn' => is_int( $event['turn_count'] ?? null ) ? $event['turn_count'] : 0,
292+
'tool_call_id' => $tool_call_id,
293+
'tool_name' => is_string( $event['tool_name'] ?? null ) ? $event['tool_name'] : '',
294+
'status' => 'pending',
295+
'arguments' => array(
296+
'keys' => $keys,
297+
'count' => count( $keys ),
298+
'redacted' => true,
299+
),
300+
);
301+
$pending[ $tool_call_id ][] = count( $calls ) - 1;
302+
continue;
303+
}
304+
305+
if ( ! in_array( $type, array( 'tool_result', 'pending' ), true ) || empty( $pending[ $tool_call_id ] ) ) {
306+
continue;
307+
}
308+
309+
if ( 'pending' === $type ) {
310+
continue;
311+
}
312+
$call_index = (int) array_shift( $pending[ $tool_call_id ] );
313+
314+
$metadata = is_array( $event['metadata'] ?? null ) ? $event['metadata'] : array();
315+
$rejected = true === ( $metadata['rejected'] ?? false );
316+
$succeeded = true === ( $metadata['success'] ?? false );
317+
$calls[ $call_index ]['status'] = $rejected ? 'rejected' : ( $succeeded ? 'succeeded' : 'failed' );
318+
319+
$result = self::matching_tool_result( $results, $tool_call_id, $result_uses );
320+
if ( is_array( $result ) ) {
321+
$execution_result = is_array( $result['result'] ?? null ) ? $result['result'] : array();
322+
$canonical_name = $execution_result['tool_name'] ?? null;
323+
if ( is_string( $canonical_name ) && '' !== $canonical_name ) {
324+
$calls[ $call_index ]['tool_name'] = $canonical_name;
325+
}
326+
327+
if ( $succeeded && array_key_exists( 'result', $execution_result ) ) {
328+
$calls[ $call_index ]['result'] = self::result_shape( $execution_result['result'] );
329+
}
330+
}
331+
332+
if ( ! $succeeded ) {
333+
$calls[ $call_index ]['error'] = $rejected
334+
? array( 'code' => 'agents_api_tool_call_rejected', 'message' => 'Tool call was rejected.' )
335+
: array( 'code' => 'agents_api_tool_execution_failed', 'message' => 'Tool execution failed.' );
336+
}
337+
}
338+
339+
return array(
340+
'version' => self::TOOL_OBSERVABILITY_VERSION,
341+
'calls' => $calls,
342+
);
343+
}
344+
345+
/**
346+
* Find the next execution result for a tool-call id.
347+
*
348+
* @param array<mixed> $results Tool results.
349+
* @param string $tool_call_id Tool call id.
350+
* @param array<string, int> $uses Per-id result offsets.
351+
* @return array<mixed>|null
352+
*/
353+
private static function matching_tool_result( array $results, string $tool_call_id, array &$uses ): ?array {
354+
$offset = $uses[ $tool_call_id ] ?? 0;
355+
$match = 0;
356+
foreach ( $results as $result ) {
357+
if ( ! is_array( $result ) || $tool_call_id !== ( $result['tool_call_id'] ?? null ) ) {
358+
continue;
359+
}
360+
if ( $match === $offset ) {
361+
$uses[ $tool_call_id ] = $offset + 1;
362+
return $result;
363+
}
364+
++$match;
365+
}
366+
367+
return null;
368+
}
369+
370+
/**
371+
* Describe a result without exposing its content.
372+
*
373+
* @param mixed $value Result value.
374+
* @return array<string,int|string>
375+
*/
376+
private static function result_shape( $value ): array {
377+
if ( is_array( $value ) ) {
378+
return array(
379+
'type' => array_is_list( $value ) ? 'array' : 'object',
380+
'count' => count( $value ),
381+
);
382+
}
383+
384+
if ( is_string( $value ) ) {
385+
return array( 'type' => 'string', 'size' => strlen( $value ) );
386+
}
387+
388+
return array( 'type' => strtolower( gettype( $value ) ) );
389+
}
390+
260391
/**
261392
* Build a machine-readable validation exception.
262393
*

‎tests/agents-chat-ability-smoke.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,8 @@ function smoke_assert( $expected, $actual, string $name, array &$failures, int &
186186

187187
$out_schema = agents_chat_output_schema();
188188
smoke_assert( array( 'session_id', 'reply' ), $out_schema['required'] ?? array(), 'output_schema_required_fields', $failures, $passes );
189+
smoke_assert( array( 1 ), $out_schema['properties']['metadata']['properties']['agents_api']['properties']['tool_observability']['properties']['version']['enum'] ?? array(), 'output_schema_documents_tool_observability_v1', $failures, $passes );
190+
smoke_assert( array( 'pending', 'succeeded', 'failed', 'rejected' ), $out_schema['properties']['metadata']['properties']['agents_api']['properties']['tool_observability']['properties']['calls']['items']['properties']['status']['enum'] ?? array(), 'output_schema_documents_tool_statuses', $failures, $passes );
189191

190192
// 9. Runtime principal input is normalized before dispatch and has a scoped permission filter.
191193
smoke_reset_chat_filters();

‎tests/conversation-loop-tool-execution-smoke.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,7 @@ static function ( array $messages ): array {
246246
agents_api_smoke_assert_equals( 'duplicate_tool_call', $reject_result['tool_audit_events'][0]['error_type'] ?? '', 'reject decision records audit error type', $failures, $passes );
247247
agents_api_smoke_assert_equals( array( 'tool_call', 'tool_result' ), array_column( $reject_result['tool_events'], 'type' ), 'reject decision records canonical tool events', $failures, $passes );
248248
agents_api_smoke_assert_equals( 'error', $reject_result['tool_events'][1]['status'] ?? '', 'reject decision records error tool event status', $failures, $passes );
249+
agents_api_smoke_assert_equals( 'rejected', $reject_result['tool_observability']['calls'][0]['status'] ?? '', 'tool observability distinguishes mediator rejection from execution failure', $failures, $passes );
249250

250251
$executor->executed = array();
251252
$replace_result = AgentsAPI\AI\WP_Agent_Conversation_Loop::run(
@@ -350,6 +351,8 @@ static function ( array $messages, array $context ) use ( &$turn_count ): array
350351
agents_api_smoke_assert_equals( 2, count( $executor->executed ), 'tool executor was called twice across turns', $failures, $passes );
351352
agents_api_smoke_assert_equals( 3, $turn_count, 'loop ran three turns (two with tools, one without)', $failures, $passes );
352353
agents_api_smoke_assert_equals( 2, count( $multi_result['tool_execution_results'] ), 'result contains two tool execution results', $failures, $passes );
354+
agents_api_smoke_assert_equals( array( 'tool-call-1-1', 'tool-call-2-1' ), array_column( $multi_result['tool_observability']['calls'] ?? array(), 'tool_call_id' ), 'tool observability preserves generated fallback ids across turns', $failures, $passes );
355+
agents_api_smoke_assert_equals( array( 1, 2 ), array_column( $multi_result['tool_observability']['calls'] ?? array(), 'sequence' ), 'tool observability assigns global call sequence across turns', $failures, $passes );
353356

354357
echo "\n[4] Tool validation errors are returned as error results without crashing:\n";
355358
$executor->executed = array();
@@ -644,6 +647,7 @@ static function ( array $messages ): array {
644647
agents_api_smoke_assert_equals( 'client/summarize', $pending_result['runtime_tool_pending']['tool_name'] ?? '', 'pending request carries tool name', $failures, $passes );
645648
agents_api_smoke_assert_equals( 'client-call-1', $pending_result['runtime_tool_pending']['tool_call_id'] ?? '', 'pending request carries tool call id', $failures, $passes );
646649
agents_api_smoke_assert_equals( array( 'tool_call', AgentsAPI\AI\WP_Agent_Runtime_Tool_Request::STATUS_PENDING ), array_column( $pending_result['tool_events'], 'type' ), 'pending request is recorded in canonical tool events', $failures, $passes );
650+
agents_api_smoke_assert_equals( 'pending', $pending_result['tool_observability']['calls'][0]['status'] ?? '', 'tool observability records pending runtime calls', $failures, $passes );
647651
agents_api_smoke_assert_equals( AgentsAPI\AI\WP_Agent_Conversation_Result::OUTCOME_STATUS_PENDING_RUNTIME_TOOL, $pending_result['run_outcome']['status'] ?? '', 'pending runtime tool run outcome is pending', $failures, $passes );
648652
agents_api_smoke_assert_equals( AgentsAPI\AI\WP_Agent_Runtime_Tool_Request::STATUS_PENDING, $pending_result['run_outcome']['stop_reason'] ?? '', 'pending runtime tool run outcome stop reason is pending', $failures, $passes );
649653

‎tests/default-agents-chat-handler-smoke.php‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -438,6 +438,13 @@ public function executeWP_Agent_Tool_Call( array $tool_call, array $tool_definit
438438
agents_api_smoke_assert_equals( true, in_array( 'user', $canonical_roles, true ), 'canonical messages include the user turn', $failures, $passes );
439439
agents_api_smoke_assert_equals( true, in_array( 'assistant', $canonical_roles, true ), 'canonical messages include the assistant reply', $failures, $passes );
440440
agents_api_smoke_assert_equals( true, ! in_array( 'tool', $canonical_roles, true ), 'canonical messages omit raw tool envelopes', $failures, $passes );
441+
$tool_observability = $output['metadata']['agents_api']['tool_observability'] ?? array();
442+
agents_api_smoke_assert_equals( 1, $tool_observability['version'] ?? null, 'canonical metadata projects tool observability v1', $failures, $passes );
443+
agents_api_smoke_assert_equals( 'call-1', $tool_observability['calls'][0]['tool_call_id'] ?? '', 'canonical metadata preserves the provider tool call id', $failures, $passes );
444+
agents_api_smoke_assert_equals( 'succeeded', $tool_observability['calls'][0]['status'] ?? '', 'canonical metadata projects the terminal tool status', $failures, $passes );
445+
agents_api_smoke_assert_equals( array( 'query' ), $tool_observability['calls'][0]['arguments']['keys'] ?? array(), 'canonical metadata exposes argument keys without values', $failures, $passes );
446+
$tool_observability_json = json_encode( $tool_observability );
447+
agents_api_smoke_assert_equals( false, is_string( $tool_observability_json ) && str_contains( $tool_observability_json, 'risotto' ), 'canonical tool observability omits argument values', $failures, $passes );
441448

442449
echo "\n[1b] Runtime-bundle agents declare their toolset as `enabled_tools` and the loop wires it:\n";
443450
// Native runtime agent bundles place their toolset under

0 commit comments

Comments
 (0)