feat(ci): missed-run heartbeat for the daily sync cron (#72)

* feat(ci): missed-run heartbeat for the daily sync cron (#71)

A failed sync run surfaces on its own, but a run that never fires is silent:
GitHub can drop scheduled runs, auto-disable the workflow after inactivity, or
lose the schedule. Add a separate scheduled workflow (heartbeat.yml) that checks
the last successful scheduled sync-translation run and alerts when it is older
than a threshold, so a silently missed run is caught. It does not depend on the
sync workflow running, so it still fires once that workflow has stopped.

The staleness decision lives in a pure heartbeat_run_is_stale() in lib.sh with
bats coverage; HEARTBEAT_MAX_AGE_HOURS (30) carries margin above the 24h cadence.
Alerting is interim (fails the job); point it at the shared notification channel
once the failure-notification work lands.

* fix(ci): use the scheduled run's createdAt for the heartbeat age

updatedAt moves when an old run is rerun, so a stopped schedule whose last run
was manually rerun looked healthy. Query createdAt, the schedule-fire time, so
the age reflects when the run actually fired.

Author: timon0305

.github/workflows/assets/env.sh

@@ -15,6 +15,11 @@ if [[ -z "${_ENV_SH_LOADED:-}" ]]; then
   readonly PHASE_TRIGGER_WEBLATE="trigger-weblate"
   readonly PHASE_DISCOVER="discover"
   readonly PHASE_SYNC_POINTERS="sync-pointers"
+  readonly PHASE_HEARTBEAT="heartbeat"
+
+  # Missed-run heartbeat: alert when the last successful scheduled sync is older
+  # than this. Margin above the 24h cron cadence so a delayed run does not trip the alert.
+  readonly HEARTBEAT_MAX_AGE_HOURS=30
 
   # start-translation submodule mode (START_PHASE env).
   readonly START_PHASE_MIRRORS="mirrors"

.github/workflows/assets/lib.sh

@@ -559,3 +559,12 @@ validate_secrets() {
     _require_nonempty WEBLATE_TOKEN "WEBLATE_TOKEN secret is not set."
   fi
 }
+
+# True (exit 0) when the last successful scheduled sync is missing or older than
+# max_age_seconds, i.e. the daily run has silently stopped. Pure function; the
+# heartbeat workflow supplies the timestamps from the GitHub API.
+heartbeat_run_is_stale() {
+  local last_success_epoch="$1" now_epoch="$2" max_age_seconds="$3"
+  [[ -z "$last_success_epoch" || "$last_success_epoch" == "0" ]] && return 0
+  (( now_epoch - last_success_epoch > max_age_seconds ))
+}

.github/workflows/heartbeat.yml

@@ -0,0 +1,73 @@
+# Missed-run heartbeat for the daily "Sync translation" cron.
+#
+# A failed sync run surfaces on its own, but a run that never fires is silent:
+# GitHub can drop scheduled runs, auto-disable the workflow after repo inactivity,
+# or the schedule can be removed. This workflow runs on its own schedule and
+# alerts when the last *successful* scheduled sync is older than a threshold, so a
+# silently missed run is caught. It does not depend on sync-translation.yml
+# running, so it still fires once that workflow has stopped.
+
+name: Sync heartbeat
+
+on:
+  schedule:
+    # Noon UTC, offset from the sync's 00:00 run so a healthy day reads ~12h old.
+    - cron: "0 12 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  actions: read
+
+jobs:
+  check-heartbeat:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository (master)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Check the last successful scheduled sync
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -euo pipefail
+
+          # shellcheck source=assets/env.sh
+          source "$GITHUB_WORKSPACE/.github/workflows/assets/env.sh"
+          # shellcheck source=assets/lib.sh
+          source "$GITHUB_WORKSPACE/.github/workflows/assets/lib.sh"
+
+          begin_phase "$PHASE_HEARTBEAT" "Check daily sync heartbeat"
+          trap '[[ -n "$CURRENT_PHASE" ]] && end_phase' EXIT ERR
+
+          # createdAt is when the run was scheduled; updatedAt moves when an old
+          # run is rerun, which would make a stopped schedule look healthy.
+          last_ts="$(gh run list \
+            --repo "$GITHUB_REPOSITORY" \
+            --workflow sync-translation.yml \
+            --event schedule \
+            --status success \
+            --limit 1 \
+            --json createdAt \
+            --jq '.[0].createdAt // ""')"
+
+          last_epoch=""
+          if [[ -n "$last_ts" ]]; then
+            last_epoch="$(date -d "$last_ts" +%s)"
+          fi
+          now_epoch="$(date +%s)"
+          max_age_seconds=$(( HEARTBEAT_MAX_AGE_HOURS * 3600 ))
+
+          if heartbeat_run_is_stale "$last_epoch" "$now_epoch" "$max_age_seconds"; then
+            # Interim alert: fail the job. Swap this for the shared notification
+            # channel once the failure-notification work provides one.
+            last_desc="${last_ts:-none on record}"
+            echo "::error::Daily sync heartbeat: last successful scheduled run is stale (last=${last_desc}, threshold=${HEARTBEAT_MAX_AGE_HOURS}h). The scheduled sync may have stopped." >&2
+            end_phase
+            exit 1
+          fi
+
+          echo "Heartbeat healthy: last successful scheduled sync at ${last_ts} (within ${HEARTBEAT_MAX_AGE_HOURS}h)." >&2
+          end_phase

tests/test_lib.bats

@@ -568,3 +568,20 @@ teardown() {
   [[ "$output" == *"Failed — processing error (1): sync_failed"* ]]
   [[ "$output" != *"processing error (2)"* ]]
 }
+
+@test "heartbeat_run_is_stale: stale when no successful run is on record" {
+  heartbeat_run_is_stale "" 200000 100000
+  heartbeat_run_is_stale "0" 200000 100000
+}
+
+@test "heartbeat_run_is_stale: stale when the last success is older than the threshold" {
+  heartbeat_run_is_stale 50000 200000 100000
+}
+
+@test "heartbeat_run_is_stale: fresh when the last success is within the threshold" {
+  ! heartbeat_run_is_stale 150000 200000 100000
+}
+
+@test "heartbeat_run_is_stale: at the threshold boundary is still fresh (strict older-than)" {
+  ! heartbeat_run_is_stale 100000 200000 100000
+}