From 746027d97f4fd57abc890d2dcdbc8a3e707ad88d Mon Sep 17 00:00:00 2001 From: timon0305 Date: Thu, 30 Jul 2026 12:25:25 +0200 Subject: [PATCH 1/2] feat(ci): missed-run heartbeat for the daily sync cron (#71) A failed sync run surfaces on its own, but a run that never fires is silent: GitHub can drop scheduled runs, auto-disable the workflow after inactivity, or lose the schedule. Add a separate scheduled workflow (heartbeat.yml) that checks the last successful scheduled sync-translation run and alerts when it is older than a threshold, so a silently missed run is caught. It does not depend on the sync workflow running, so it still fires once that workflow has stopped. The staleness decision lives in a pure heartbeat_run_is_stale() in lib.sh with bats coverage; HEARTBEAT_MAX_AGE_HOURS (30) carries margin above the 24h cadence. Alerting is interim (fails the job); point it at the shared notification channel once the failure-notification work lands. --- .github/workflows/assets/env.sh | 5 +++ .github/workflows/assets/lib.sh | 9 +++++ .github/workflows/heartbeat.yml | 71 +++++++++++++++++++++++++++++++++ tests/test_lib.bats | 17 ++++++++ 4 files changed, 102 insertions(+) create mode 100644 .github/workflows/heartbeat.yml diff --git a/.github/workflows/assets/env.sh b/.github/workflows/assets/env.sh index e6e682b..80d6d4a 100644 --- a/.github/workflows/assets/env.sh +++ b/.github/workflows/assets/env.sh @@ -15,6 +15,11 @@ if [[ -z "${_ENV_SH_LOADED:-}" ]]; then readonly PHASE_TRIGGER_WEBLATE="trigger-weblate" readonly PHASE_DISCOVER="discover" readonly PHASE_SYNC_POINTERS="sync-pointers" + readonly PHASE_HEARTBEAT="heartbeat" + + # Missed-run heartbeat: alert when the last successful scheduled sync is older + # than this. Margin above the 24h cron cadence so a delayed run does not trip the alert. + readonly HEARTBEAT_MAX_AGE_HOURS=30 # start-translation submodule mode (START_PHASE env). readonly START_PHASE_MIRRORS="mirrors" diff --git a/.github/workflows/assets/lib.sh b/.github/workflows/assets/lib.sh index da24cbc..874d39f 100644 --- a/.github/workflows/assets/lib.sh +++ b/.github/workflows/assets/lib.sh @@ -558,3 +558,12 @@ validate_secrets() { _require_nonempty WEBLATE_TOKEN "WEBLATE_TOKEN secret is not set." fi } + +# True (exit 0) when the last successful scheduled sync is missing or older than +# max_age_seconds, i.e. the daily run has silently stopped. Pure function; the +# heartbeat workflow supplies the timestamps from the GitHub API. +heartbeat_run_is_stale() { + local last_success_epoch="$1" now_epoch="$2" max_age_seconds="$3" + [[ -z "$last_success_epoch" || "$last_success_epoch" == "0" ]] && return 0 + (( now_epoch - last_success_epoch > max_age_seconds )) +} diff --git a/.github/workflows/heartbeat.yml b/.github/workflows/heartbeat.yml new file mode 100644 index 0000000..ceb82fb --- /dev/null +++ b/.github/workflows/heartbeat.yml @@ -0,0 +1,71 @@ +# Missed-run heartbeat for the daily "Sync translation" cron. +# +# A failed sync run surfaces on its own, but a run that never fires is silent: +# GitHub can drop scheduled runs, auto-disable the workflow after repo inactivity, +# or the schedule can be removed. This workflow runs on its own schedule and +# alerts when the last *successful* scheduled sync is older than a threshold, so a +# silently missed run is caught. It does not depend on sync-translation.yml +# running, so it still fires once that workflow has stopped. + +name: Sync heartbeat + +on: + schedule: + # Noon UTC, offset from the sync's 00:00 run so a healthy day reads ~12h old. + - cron: "0 12 * * *" + workflow_dispatch: + +permissions: + contents: read + actions: read + +jobs: + check-heartbeat: + runs-on: ubuntu-latest + steps: + - name: Checkout repository (master) + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Check the last successful scheduled sync + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + + # shellcheck source=assets/env.sh + source "$GITHUB_WORKSPACE/.github/workflows/assets/env.sh" + # shellcheck source=assets/lib.sh + source "$GITHUB_WORKSPACE/.github/workflows/assets/lib.sh" + + begin_phase "$PHASE_HEARTBEAT" "Check daily sync heartbeat" + trap '[[ -n "$CURRENT_PHASE" ]] && end_phase' EXIT ERR + + last_ts="$(gh run list \ + --repo "$GITHUB_REPOSITORY" \ + --workflow sync-translation.yml \ + --event schedule \ + --status success \ + --limit 1 \ + --json updatedAt \ + --jq '.[0].updatedAt // ""')" + + last_epoch="" + if [[ -n "$last_ts" ]]; then + last_epoch="$(date -d "$last_ts" +%s)" + fi + now_epoch="$(date +%s)" + max_age_seconds=$(( HEARTBEAT_MAX_AGE_HOURS * 3600 )) + + if heartbeat_run_is_stale "$last_epoch" "$now_epoch" "$max_age_seconds"; then + # Interim alert: fail the job. Swap this for the shared notification + # channel once the failure-notification work provides one. + last_desc="${last_ts:-none on record}" + echo "::error::Daily sync heartbeat: last successful scheduled run is stale (last=${last_desc}, threshold=${HEARTBEAT_MAX_AGE_HOURS}h). The scheduled sync may have stopped." >&2 + end_phase + exit 1 + fi + + echo "Heartbeat healthy: last successful scheduled sync at ${last_ts} (within ${HEARTBEAT_MAX_AGE_HOURS}h)." >&2 + end_phase diff --git a/tests/test_lib.bats b/tests/test_lib.bats index 97d9799..118361f 100644 --- a/tests/test_lib.bats +++ b/tests/test_lib.bats @@ -568,3 +568,20 @@ teardown() { [[ "$output" == *"Failed — processing error (1): sync_failed"* ]] [[ "$output" != *"processing error (2)"* ]] } + +@test "heartbeat_run_is_stale: stale when no successful run is on record" { + heartbeat_run_is_stale "" 200000 100000 + heartbeat_run_is_stale "0" 200000 100000 +} + +@test "heartbeat_run_is_stale: stale when the last success is older than the threshold" { + heartbeat_run_is_stale 50000 200000 100000 +} + +@test "heartbeat_run_is_stale: fresh when the last success is within the threshold" { + ! heartbeat_run_is_stale 150000 200000 100000 +} + +@test "heartbeat_run_is_stale: at the threshold boundary is still fresh (strict older-than)" { + ! heartbeat_run_is_stale 100000 200000 100000 +} From af97b900633115a7d521efd3c6eea812991b6be3 Mon Sep 17 00:00:00 2001 From: timon0305 Date: Thu, 30 Jul 2026 18:13:00 +0200 Subject: [PATCH 2/2] fix(ci): use the scheduled run's createdAt for the heartbeat age updatedAt moves when an old run is rerun, so a stopped schedule whose last run was manually rerun looked healthy. Query createdAt, the schedule-fire time, so the age reflects when the run actually fired. --- .github/workflows/heartbeat.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/heartbeat.yml b/.github/workflows/heartbeat.yml index ceb82fb..cb3ae33 100644 --- a/.github/workflows/heartbeat.yml +++ b/.github/workflows/heartbeat.yml @@ -42,14 +42,16 @@ jobs: begin_phase "$PHASE_HEARTBEAT" "Check daily sync heartbeat" trap '[[ -n "$CURRENT_PHASE" ]] && end_phase' EXIT ERR + # createdAt is when the run was scheduled; updatedAt moves when an old + # run is rerun, which would make a stopped schedule look healthy. last_ts="$(gh run list \ --repo "$GITHUB_REPOSITORY" \ --workflow sync-translation.yml \ --event schedule \ --status success \ --limit 1 \ - --json updatedAt \ - --jq '.[0].updatedAt // ""')" + --json createdAt \ + --jq '.[0].createdAt // ""')" last_epoch="" if [[ -n "$last_ts" ]]; then