Skip to content

Commit 97ecc0c

Browse files
Surface activity timeout-overdue age on dw system:operator-metrics
Renders the workflow-v2 activity timeout-overdue rollup pinned by the Phase 6 rollout-safety contract on `operator_metrics.activities.{timeout_overdue,oldest_timeout_overdue_at,max_timeout_overdue_age_ms}` so HTTP-only operators can read the worst-case stuck-activity duplicate-risk age (the activity-path counterpart of `tasks.lease_expired` — open Pending or Running activity executions whose `schedule_deadline_at`, `close_deadline_at`, `schedule_to_close_deadline_at`, or `heartbeat_deadline_at` has passed and is still waiting for the timeout sweep) directly from `dw system:operator-metrics` without walking `activity_executions` themselves. `OperatorMetricsCommand::renderActivities()` now emits three new rows right after the existing "Oldest retrying started at" row and before "Failed attempts": Timeout overdue: <count> Oldest timeout-overdue age: <ms> ms Oldest timeout-overdue at: <ISO> Each row degrades gracefully when the snapshot predates the contract: the renderer omits the row when its key is absent and the existing retrying-age and failed-attempts rendering still surfaces the rest of the activity-path coordination health. Pins `activities.timeout_overdue`, `activities.oldest_timeout_overdue_at`, and `activities.max_timeout_overdue_age_ms` on `schemas/output/operator-metrics.schema.json` next to the existing retrying-age keys, and adds `test_operator_metrics_schema_pins_activities_timeout_overdue_keys` and `test_operator_metrics_command_omits_activities_timeout_overdue_when_snapshot_predates_contract` plus assertions on the existing render smoke test to guard the schema declaration, the renderer, and the renderer fallback.
1 parent 79e98c2 commit 97ecc0c

3 files changed

Lines changed: 63 additions & 0 deletions

File tree

schemas/output/operator-metrics.schema.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,9 @@
125125
"retrying": { "type": ["integer", "null"] },
126126
"oldest_retrying_started_at": { "type": ["string", "null"] },
127127
"max_retrying_age_ms": { "type": ["integer", "null"] },
128+
"timeout_overdue": { "type": ["integer", "null"] },
129+
"oldest_timeout_overdue_at": { "type": ["string", "null"] },
130+
"max_timeout_overdue_age_ms": { "type": ["integer", "null"] },
128131
"failed_attempts": { "type": ["integer", "null"] },
129132
"max_attempt_count": { "type": ["integer", "null"] }
130133
},

src/Commands/SystemCommand/OperatorMetricsCommand.php

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -435,6 +435,21 @@ private function renderActivities(OutputInterface $output, array $activities): v
435435
if (is_string($activities['oldest_retrying_started_at'] ?? null)) {
436436
$output->writeln(sprintf(' Oldest retrying started at: %s', $activities['oldest_retrying_started_at']));
437437
}
438+
if (array_key_exists('timeout_overdue', $activities)) {
439+
$output->writeln(sprintf(
440+
' Timeout overdue: %d',
441+
(int) ($activities['timeout_overdue'] ?? 0),
442+
));
443+
}
444+
if (array_key_exists('max_timeout_overdue_age_ms', $activities)) {
445+
$output->writeln(sprintf(
446+
' Oldest timeout-overdue age: %d ms',
447+
(int) ($activities['max_timeout_overdue_age_ms'] ?? 0),
448+
));
449+
}
450+
if (is_string($activities['oldest_timeout_overdue_at'] ?? null)) {
451+
$output->writeln(sprintf(' Oldest timeout-overdue at: %s', $activities['oldest_timeout_overdue_at']));
452+
}
438453
$output->writeln(sprintf(
439454
' Failed attempts: %d (max attempts on a single execution: %d)',
440455
(int) ($activities['failed_attempts'] ?? 0),

tests/Commands/SystemCommandTest.php

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -572,6 +572,9 @@ public function test_operator_metrics_command_renders_rollout_safety_signals():
572572
self::assertStringContainsString('Open 12 (pending 8, running 4), retrying 3', $display);
573573
self::assertStringContainsString('Oldest retrying age: 270000 ms', $display);
574574
self::assertStringContainsString('Oldest retrying started at: 2026-04-24T11:25:30Z', $display);
575+
self::assertStringContainsString('Timeout overdue: 2', $display);
576+
self::assertStringContainsString('Oldest timeout-overdue age: 345000 ms', $display);
577+
self::assertStringContainsString('Oldest timeout-overdue at: 2026-04-24T11:24:15Z', $display);
575578
self::assertStringContainsString('Failed attempts: 7 (max attempts on a single execution: 5)', $display);
576579

577580
self::assertStringContainsString('Redispatch after: 120s', $display);
@@ -947,6 +950,45 @@ public function test_operator_metrics_command_omits_activities_block_when_payloa
947950
self::assertStringNotContainsString('Oldest retrying age', $display);
948951
}
949952

953+
public function test_operator_metrics_schema_pins_activities_timeout_overdue_keys(): void
954+
{
955+
$schema = json_decode(
956+
(string) file_get_contents(__DIR__.'/../../schemas/output/operator-metrics.schema.json'),
957+
true,
958+
flags: JSON_THROW_ON_ERROR,
959+
);
960+
961+
$activities = $schema['properties']['operator_metrics']['properties']['activities']['properties'];
962+
963+
self::assertSame(['integer', 'null'], $activities['timeout_overdue']['type']);
964+
self::assertSame(['string', 'null'], $activities['oldest_timeout_overdue_at']['type']);
965+
self::assertSame(['integer', 'null'], $activities['max_timeout_overdue_age_ms']['type']);
966+
}
967+
968+
public function test_operator_metrics_command_omits_activities_timeout_overdue_when_snapshot_predates_contract(): void
969+
{
970+
$payload = self::operatorMetricsPayload();
971+
unset(
972+
$payload['operator_metrics']['activities']['timeout_overdue'],
973+
$payload['operator_metrics']['activities']['oldest_timeout_overdue_at'],
974+
$payload['operator_metrics']['activities']['max_timeout_overdue_age_ms'],
975+
);
976+
977+
$command = new OperatorMetricsCommand();
978+
$command->setServerClient(new SystemFakeClient($payload));
979+
980+
$tester = new CommandTester($command);
981+
self::assertSame(Command::SUCCESS, $tester->execute([]));
982+
983+
$display = $tester->getDisplay();
984+
985+
self::assertStringNotContainsString('Timeout overdue:', $display);
986+
self::assertStringNotContainsString('Oldest timeout-overdue age', $display);
987+
self::assertStringNotContainsString('Oldest timeout-overdue at', $display);
988+
self::assertStringContainsString('Open 12 (pending 8, running 4), retrying 3', $display);
989+
self::assertStringContainsString('Failed attempts: 7', $display);
990+
}
991+
950992
public function test_operator_metrics_command_renders_dedicated_matching_role_shape(): void
951993
{
952994
$payload = self::operatorMetricsPayload();
@@ -1090,6 +1132,9 @@ private static function operatorMetricsPayload(): array
10901132
'retrying' => 3,
10911133
'oldest_retrying_started_at' => '2026-04-24T11:25:30Z',
10921134
'max_retrying_age_ms' => 270000,
1135+
'timeout_overdue' => 2,
1136+
'oldest_timeout_overdue_at' => '2026-04-24T11:24:15Z',
1137+
'max_timeout_overdue_age_ms' => 345000,
10931138
'failed_attempts' => 7,
10941139
'max_attempt_count' => 5,
10951140
],

0 commit comments

Comments
 (0)