Surface run-summary projection missing-run age on operator metrics

durable-workflow-ops · durable-workflow-ops · commit 3711c3b42da4 · 2026-04-25T02:25:13.000Z
Freezes `projections.run_summaries.oldest_missing_run_started_at`
(ISO-8601 or null) and `projections.run_summaries.max_missing_run_age_ms`
(integer ms) on `OperatorMetrics::snapshot()`. The pair mirrors the
existing `repair.oldest_missing_run_started_at` /
`repair.max_missing_run_age_ms` shape so operators can answer "how
long has the worst-case run been without a run-summary projection?"
— the primary projection-lag age indicator on the run-summary path —
from the metric alone without walking `workflow_runs`.

The selector uses `COALESCE(started_at, created_at)` so runs that
have not yet recorded a `started_at` still report the backlog age
they contribute to the projection lag (previously the signal would
have missed them entirely). The missing-projection set is defined
by `RunSummaryProjectionDrift::missingRunQuery()`: runs whose id is
not present in `workflow_run_summaries`. Runs with a projection and
orphaned/stale summaries are out of scope for this indicator; they
stay visible on the existing `missing`, `orphaned`, `stale`, and
`needs_rebuild` counts.

Pins the row on `docs/architecture/rollout-safety.md` and adds a
projection-lag bullet, guarded by
`RolloutSafetyDocumentationTest::testContractDocumentFreezesRunSummaryProjectionMissingRunAgeRow`.
diff --git a/docs/architecture/rollout-safety.md b/docs/architecture/rollout-safety.md
@@ -420,6 +420,7 @@ change.
 | `backlog` | `unhealthy_tasks`, `repair_needed_runs`, `claim_failed_runs`, `compatibility_blocked_runs` | stuck/blocked roll-ups |
 | `backlog` | `oldest_compatibility_blocked_started_at`, `max_compatibility_blocked_age_ms` | earliest wait-start timestamp among compatibility-blocked runs and the largest blocked age in milliseconds, mirroring the `repair.oldest_missing_run_started_at` / `max_missing_run_age_ms` shape so operators can answer "how stale is the worst mixed-build block?" from the metric alone |
 | `repair` | `missing_task_candidates`, `selected_missing_task_candidates`, `oldest_missing_run_started_at`, `max_missing_run_age_ms` | stuck-run detectors per `TaskRepairCandidates` |
+| `projections.run_summaries` | `oldest_missing_run_started_at`, `max_missing_run_age_ms` | earliest `COALESCE(workflow_runs.started_at, workflow_runs.created_at)` among runs whose id is not present in `workflow_run_summaries` and the largest missing-projection age in milliseconds, mirroring the `repair.oldest_missing_run_started_at` / `max_missing_run_age_ms` shape so operators can read "how long has the worst-case run been without a run-summary projection?" — the primary projection-lag age indicator on the run-summary path — from the metric alone without walking `workflow_runs` |
 | `workers` | `required_compatibility`, `active_workers`, `active_worker_scopes`, `active_workers_supporting_required` | routing-health signals per `WorkerCompatibilityFleet` |
 | `workers` | `fleet` | per-scope fleet entries (`worker_id`, `namespace`, `connection`, `queue`, `supported`, `supports_required`, `recorded_at`, `expires_at`, `source`, `host`, `process_id`) so mixed-build state is legible to Waterline and other consumers without reinferring it from the summary counts |
 | `schedules` | `active`, `paused`, `missed`, `oldest_overdue_at`, `max_overdue_ms`, `fires_total`, `failures_total` | scheduler-role health: active and paused schedules in namespace, active schedules whose `next_fire_at` is overdue at snapshot time, the earliest overdue `next_fire_at` among them, the largest overdue age in milliseconds, and running totals of fires and failures so scheduler lag and failure trends are legible without reading `workflow_schedules` directly |
diff --git a/src/V2/Support/OperatorMetrics.php b/src/V2/Support/OperatorMetrics.php
@@ -48,7 +48,7 @@ public static function snapshot(?CarbonInterface $now = null, ?string $namespace
             'starts' => self::startMetrics($now, $namespace),
             'history' => self::historyMetrics($namespace),
             'command_contracts' => self::commandContractMetrics($namespace),
-            'projections' => self::projectionMetrics($namespace),
+            'projections' => self::projectionMetrics($now, $namespace),
             'schedules' => self::scheduleMetrics($now, $namespace),
             'workers' => self::workerMetrics(),
             'backend' => BackendCapabilities::snapshot($now),
@@ -446,15 +446,20 @@ private static function commandContractMetrics(?string $namespace): array
     /**
      * @return array<string, array<string, int|string|null>>
      */
-    private static function projectionMetrics(?string $namespace): array
+    private static function projectionMetrics(CarbonInterface $now, ?string $namespace): array
     {
         $runSummaries = RunSummaryProjectionDrift::metrics($namespace);
+        $oldestMissingRunStartedAt = self::oldestMissingRunSummaryStartedAt($namespace);
 
         return [
             'run_summaries' => [
                 ...$runSummaries,
                 'oldest_updated_at' => self::jsonTimestamp(self::summaryQuery($namespace)->min('updated_at')),
                 'newest_updated_at' => self::jsonTimestamp(self::summaryQuery($namespace)->max('updated_at')),
+                'oldest_missing_run_started_at' => $oldestMissingRunStartedAt?->toJSON(),
+                'max_missing_run_age_ms' => $oldestMissingRunStartedAt === null
+                    ? 0
+                    : (int) $oldestMissingRunStartedAt->diffInMilliseconds($now),
             ],
             'run_waits' => self::runWaitProjectionMetrics($namespace),
             'run_timeline_entries' => self::runTimelineProjectionMetrics($namespace),
@@ -463,6 +468,31 @@ private static function projectionMetrics(?string $namespace): array
         ];
     }
 
+    /**
+     * Earliest `COALESCE(workflow_runs.started_at, workflow_runs.created_at)`
+     * among runs whose id is not present in `workflow_run_summaries`.
+     * Mirrors the `repair.oldest_missing_run_started_at` shape so
+     * rollout-safety consumers can read "how long has the worst-case run
+     * been without a run-summary projection?" — the primary projection-lag
+     * age indicator on the run-summary path — from the metric alone without
+     * walking `workflow_runs`. Falls back to `created_at` when the run has
+     * not yet recorded a `started_at` so not-yet-started runs still report
+     * the backlog age they contribute to the projection lag.
+     */
+    private static function oldestMissingRunSummaryStartedAt(?string $namespace): ?CarbonInterface
+    {
+        /** @var WorkflowRun|null $run */
+        $run = RunSummaryProjectionDrift::missingRunQuery($namespace)
+            ->orderByRaw('COALESCE(started_at, created_at) asc')
+            ->first();
+
+        if (! $run instanceof WorkflowRun) {
+            return null;
+        }
+
+        return $run->started_at ?? $run->created_at;
+    }
+
     /**
      * @return array<string, int|string|null>
      */
diff --git a/tests/Feature/V2/V2OperatorMetricsTest.php b/tests/Feature/V2/V2OperatorMetricsTest.php
@@ -1596,6 +1596,141 @@ public function testSnapshotReportsRetryingActivityAgeAsZeroWhenNoActivitiesAreR
         $this->assertSame(0, $snapshot['activities']['max_retrying_age_ms']);
     }
 
+    public function testSnapshotSurfacesMissingRunSummaryProjectionAgeFromOldestMissingRun(): void
+    {
+        Carbon::setTestNow('2026-04-09 12:00:00');
+        $this->beforeApplicationDestroyed(static function (): void {
+            Carbon::setTestNow();
+        });
+
+        $now = Carbon::now();
+
+        // Worst-case: run with a summary already exists (not counted). Its
+        // started_at must NOT win the "oldest missing run" selection even
+        // though it is the oldest started_at overall.
+        $this->createRunWithSummary(
+            instanceId: 'missing-summary-healthy-i',
+            runId: '01JMISSUMHEALTHYRUN000001',
+            status: 'running',
+            statusBucket: 'running',
+            livenessState: 'running',
+        );
+
+        // Missing-summary run A — started 180s ago, created 200s ago.
+        // Oldest started_at among missing runs — wins the selection.
+        $oldestMissingInstance = WorkflowInstance::query()->create([
+            'id' => 'missing-summary-oldest-i',
+            'workflow_class' => 'WorkflowClass',
+            'workflow_type' => 'workflow.test',
+            'run_count' => 1,
+        ]);
+
+        WorkflowRun::query()->create([
+            'id' => '01JMISSUMOLDESTRUN000001',
+            'workflow_instance_id' => $oldestMissingInstance->id,
+            'run_number' => 1,
+            'workflow_class' => 'WorkflowClass',
+            'workflow_type' => 'workflow.test',
+            'status' => 'running',
+            'started_at' => $now->copy()
+                ->subSeconds(180),
+            'created_at' => $now->copy()
+                ->subSeconds(200),
+            'updated_at' => $now->copy()
+                ->subSeconds(180),
+        ]);
+
+        // Missing-summary run B — started 30s ago, counted but must not
+        // win the "oldest at" selection.
+        $newerMissingInstance = WorkflowInstance::query()->create([
+            'id' => 'missing-summary-newer-i',
+            'workflow_class' => 'WorkflowClass',
+            'workflow_type' => 'workflow.test',
+            'run_count' => 1,
+        ]);
+
+        WorkflowRun::query()->create([
+            'id' => '01JMISSUMNEWERRUN000001',
+            'workflow_instance_id' => $newerMissingInstance->id,
+            'run_number' => 1,
+            'workflow_class' => 'WorkflowClass',
+            'workflow_type' => 'workflow.test',
+            'status' => 'running',
+            'started_at' => $now->copy()
+                ->subSeconds(30),
+            'created_at' => $now->copy()
+                ->subSeconds(30),
+            'updated_at' => $now->copy()
+                ->subSeconds(30),
+        ]);
+
+        // Missing-summary run C — started_at NULL, created 240s ago.
+        // Falls back to created_at for the age signal, but the 180s started
+        // A is still the oldest because 180s < 240s is false... wait: 240s
+        // IS older than 180s, so C must win the selection via the
+        // COALESCE(started_at, created_at) fallback.
+        $nullStartedInstance = WorkflowInstance::query()->create([
+            'id' => 'missing-summary-null-started-i',
+            'workflow_class' => 'WorkflowClass',
+            'workflow_type' => 'workflow.test',
+            'run_count' => 1,
+        ]);
+
+        WorkflowRun::query()->create([
+            'id' => '01JMISSUMNULLRUN00000001',
+            'workflow_instance_id' => $nullStartedInstance->id,
+            'run_number' => 1,
+            'workflow_class' => 'WorkflowClass',
+            'workflow_type' => 'workflow.test',
+            'status' => 'pending',
+            'started_at' => null,
+            'created_at' => $now->copy()
+                ->subSeconds(240),
+            'updated_at' => $now->copy()
+                ->subSeconds(240),
+        ]);
+
+        $snapshot = OperatorMetrics::snapshot($now);
+
+        $expectedOldestMissingAt = $now->copy()
+            ->subSeconds(240)
+            ->toJSON();
+
+        $this->assertSame(3, $snapshot['projections']['run_summaries']['missing']);
+        $this->assertSame(
+            $expectedOldestMissingAt,
+            $snapshot['projections']['run_summaries']['oldest_missing_run_started_at'],
+        );
+        $this->assertSame(
+            240 * 1000,
+            $snapshot['projections']['run_summaries']['max_missing_run_age_ms'],
+        );
+    }
+
+    public function testSnapshotReportsMissingRunSummaryProjectionAgeAsZeroWhenNoRunsAreMissing(): void
+    {
+        Carbon::setTestNow('2026-04-09 12:00:00');
+        $this->beforeApplicationDestroyed(static function (): void {
+            Carbon::setTestNow();
+        });
+
+        $now = Carbon::now();
+
+        $this->createRunWithSummary(
+            instanceId: 'no-missing-summary-i',
+            runId: '01JNOMISSUMRUN0000000001',
+            status: 'running',
+            statusBucket: 'running',
+            livenessState: 'running',
+        );
+
+        $snapshot = OperatorMetrics::snapshot($now);
+
+        $this->assertSame(0, $snapshot['projections']['run_summaries']['missing']);
+        $this->assertNull($snapshot['projections']['run_summaries']['oldest_missing_run_started_at']);
+        $this->assertSame(0, $snapshot['projections']['run_summaries']['max_missing_run_age_ms']);
+    }
+
     public function testSnapshotReportsInWorkerMatchingRoleShapeByDefault(): void
     {
         config()->set('workflows.v2.matching_role.queue_wake_enabled', true);
diff --git a/tests/Unit/V2/RolloutSafetyDocumentationTest.php b/tests/Unit/V2/RolloutSafetyDocumentationTest.php
@@ -411,6 +411,17 @@ public function testContractDocumentFreezesRetryingActivityAgeRow(): void
         );
     }
 
+    public function testContractDocumentFreezesRunSummaryProjectionMissingRunAgeRow(): void
+    {
+        $contents = $this->documentContents();
+
+        $this->assertMatchesRegularExpression(
+            '/\|\s*`projections\.run_summaries`\s*\|[^|]*`oldest_missing_run_started_at`[^|]*`max_missing_run_age_ms`/',
+            $contents,
+            'Rollout safety contract must pin the run-summary projection missing-run age row so operators can read "how long has the worst-case run been without a run-summary projection?" — the primary projection-lag age indicator on the run-summary path — from OperatorMetrics::snapshot() without walking workflow_runs.',
+        );
+    }
+
     public function testContractDocumentFreezesHealthCheckNames(): void
     {
         $contents = $this->documentContents();
