diff --git a/.github/workflows/assets/env.sh b/.github/workflows/assets/env.sh index e6e682b..80d6d4a 100644 --- a/.github/workflows/assets/env.sh +++ b/.github/workflows/assets/env.sh @@ -15,6 +15,11 @@ if [[ -z "${_ENV_SH_LOADED:-}" ]]; then readonly PHASE_TRIGGER_WEBLATE="trigger-weblate" readonly PHASE_DISCOVER="discover" readonly PHASE_SYNC_POINTERS="sync-pointers" + readonly PHASE_HEARTBEAT="heartbeat" + + # Missed-run heartbeat: alert when the last successful scheduled sync is older + # than this. Margin above the 24h cron cadence so a delayed run does not trip the alert. + readonly HEARTBEAT_MAX_AGE_HOURS=30 # start-translation submodule mode (START_PHASE env). readonly START_PHASE_MIRRORS="mirrors" diff --git a/.github/workflows/assets/lib.sh b/.github/workflows/assets/lib.sh index da24cbc..874d39f 100644 --- a/.github/workflows/assets/lib.sh +++ b/.github/workflows/assets/lib.sh @@ -558,3 +558,12 @@ validate_secrets() { _require_nonempty WEBLATE_TOKEN "WEBLATE_TOKEN secret is not set." fi } + +# True (exit 0) when the last successful scheduled sync is missing or older than +# max_age_seconds, i.e. the daily run has silently stopped. Pure function; the +# heartbeat workflow supplies the timestamps from the GitHub API. +heartbeat_run_is_stale() { + local last_success_epoch="$1" now_epoch="$2" max_age_seconds="$3" + [[ -z "$last_success_epoch" || "$last_success_epoch" == "0" ]] && return 0 + (( now_epoch - last_success_epoch > max_age_seconds )) +} diff --git a/.github/workflows/heartbeat.yml b/.github/workflows/heartbeat.yml new file mode 100644 index 0000000..cb3ae33 --- /dev/null +++ b/.github/workflows/heartbeat.yml @@ -0,0 +1,73 @@ +# Missed-run heartbeat for the daily "Sync translation" cron. +# +# A failed sync run surfaces on its own, but a run that never fires is silent: +# GitHub can drop scheduled runs, auto-disable the workflow after repo inactivity, +# or the schedule can be removed. This workflow runs on its own schedule and +# alerts when the last *successful* scheduled sync is older than a threshold, so a +# silently missed run is caught. It does not depend on sync-translation.yml +# running, so it still fires once that workflow has stopped. + +name: Sync heartbeat + +on: + schedule: + # Noon UTC, offset from the sync's 00:00 run so a healthy day reads ~12h old. + - cron: "0 12 * * *" + workflow_dispatch: + +permissions: + contents: read + actions: read + +jobs: + check-heartbeat: + runs-on: ubuntu-latest + steps: + - name: Checkout repository (master) + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Check the last successful scheduled sync + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + + # shellcheck source=assets/env.sh + source "$GITHUB_WORKSPACE/.github/workflows/assets/env.sh" + # shellcheck source=assets/lib.sh + source "$GITHUB_WORKSPACE/.github/workflows/assets/lib.sh" + + begin_phase "$PHASE_HEARTBEAT" "Check daily sync heartbeat" + trap '[[ -n "$CURRENT_PHASE" ]] && end_phase' EXIT ERR + + # createdAt is when the run was scheduled; updatedAt moves when an old + # run is rerun, which would make a stopped schedule look healthy. + last_ts="$(gh run list \ + --repo "$GITHUB_REPOSITORY" \ + --workflow sync-translation.yml \ + --event schedule \ + --status success \ + --limit 1 \ + --json createdAt \ + --jq '.[0].createdAt // ""')" + + last_epoch="" + if [[ -n "$last_ts" ]]; then + last_epoch="$(date -d "$last_ts" +%s)" + fi + now_epoch="$(date +%s)" + max_age_seconds=$(( HEARTBEAT_MAX_AGE_HOURS * 3600 )) + + if heartbeat_run_is_stale "$last_epoch" "$now_epoch" "$max_age_seconds"; then + # Interim alert: fail the job. Swap this for the shared notification + # channel once the failure-notification work provides one. + last_desc="${last_ts:-none on record}" + echo "::error::Daily sync heartbeat: last successful scheduled run is stale (last=${last_desc}, threshold=${HEARTBEAT_MAX_AGE_HOURS}h). The scheduled sync may have stopped." >&2 + end_phase + exit 1 + fi + + echo "Heartbeat healthy: last successful scheduled sync at ${last_ts} (within ${HEARTBEAT_MAX_AGE_HOURS}h)." >&2 + end_phase diff --git a/tests/test_lib.bats b/tests/test_lib.bats index 97d9799..118361f 100644 --- a/tests/test_lib.bats +++ b/tests/test_lib.bats @@ -568,3 +568,20 @@ teardown() { [[ "$output" == *"Failed — processing error (1): sync_failed"* ]] [[ "$output" != *"processing error (2)"* ]] } + +@test "heartbeat_run_is_stale: stale when no successful run is on record" { + heartbeat_run_is_stale "" 200000 100000 + heartbeat_run_is_stale "0" 200000 100000 +} + +@test "heartbeat_run_is_stale: stale when the last success is older than the threshold" { + heartbeat_run_is_stale 50000 200000 100000 +} + +@test "heartbeat_run_is_stale: fresh when the last success is within the threshold" { + ! heartbeat_run_is_stale 150000 200000 100000 +} + +@test "heartbeat_run_is_stale: at the threshold boundary is still fresh (strict older-than)" { + ! heartbeat_run_is_stale 100000 200000 100000 +}