|
| 1 | +#!/usr/bin/env bash |
| 2 | +set -euo pipefail |
| 3 | + |
| 4 | +run_id="${INPUT_RUN_ID}" |
| 5 | +run_attempt="${INPUT_RUN_ATTEMPT}" |
| 6 | +workflow_name="${INPUT_WORKFLOW_NAME}" |
| 7 | +max_run_attempts="${INPUT_MAX_RUN_ATTEMPTS:-2}" |
| 8 | + |
| 9 | +if [ -z "${GH_TOKEN:-}" ]; then |
| 10 | + echo "GH_TOKEN is required." >&2 |
| 11 | + |
| 12 | + exit 1 |
| 13 | +fi |
| 14 | + |
| 15 | +failed_jobs_csv="" |
| 16 | +matched_jobs_csv="" |
| 17 | +uninspectable_jobs_csv="" |
| 18 | + |
| 19 | +csv_append() { |
| 20 | + local current="$1" |
| 21 | + local value="$2" |
| 22 | + |
| 23 | + if [ -z "${current}" ]; then |
| 24 | + printf '%s' "${value}" |
| 25 | + |
| 26 | + return |
| 27 | + fi |
| 28 | + |
| 29 | + printf '%s,%s' "${current}" "${value}" |
| 30 | +} |
| 31 | + |
| 32 | +csv_to_summary_list() { |
| 33 | + local csv="$1" |
| 34 | + local rendered=() |
| 35 | + local item="" |
| 36 | + |
| 37 | + if [ -z "${csv}" ]; then |
| 38 | + return |
| 39 | + fi |
| 40 | + |
| 41 | + IFS=',' read -r -a rendered <<< "${csv}" |
| 42 | + |
| 43 | + for item in "${rendered[@]}"; do |
| 44 | + printf '`%s`' "${item}" |
| 45 | + |
| 46 | + if [ "${item}" != "${rendered[${#rendered[@]}-1]}" ]; then |
| 47 | + printf ', ' |
| 48 | + fi |
| 49 | + done |
| 50 | +} |
| 51 | + |
| 52 | +build_summary() { |
| 53 | + local status="$1" |
| 54 | + local lines=( |
| 55 | + "## Transient Failure Retry Summary" |
| 56 | + "" |
| 57 | + "- Workflow: \`${workflow_name}\`" |
| 58 | + "- Run ID: \`${run_id}\`" |
| 59 | + "- Run attempt: \`${run_attempt}\`" |
| 60 | + "- Retry status: \`${status}\`" |
| 61 | + ) |
| 62 | + |
| 63 | + if [ -n "${failed_jobs_csv}" ]; then |
| 64 | + lines+=("- Failed jobs inspected: $(csv_to_summary_list "${failed_jobs_csv}")") |
| 65 | + fi |
| 66 | + |
| 67 | + if [ -n "${matched_jobs_csv}" ]; then |
| 68 | + lines+=("- Jobs with transient GitHub failure signatures: $(csv_to_summary_list "${matched_jobs_csv}")") |
| 69 | + fi |
| 70 | + |
| 71 | + if [ -n "${uninspectable_jobs_csv}" ]; then |
| 72 | + lines+=("- Failed jobs with unreadable logs: $(csv_to_summary_list "${uninspectable_jobs_csv}")") |
| 73 | + fi |
| 74 | + |
| 75 | + case "${status}" in |
| 76 | + rerun-requested) |
| 77 | + lines+=("- Action: Requested a rerun of failed jobs because every inspectable failed job matched transient GitHub-side error signatures.") |
| 78 | + ;; |
| 79 | + skipped-run-attempt-limit) |
| 80 | + lines+=("- Action: Skipped rerun because the workflow already reached the configured retry limit.") |
| 81 | + ;; |
| 82 | + skipped-no-failed-jobs) |
| 83 | + lines+=("- Action: Skipped rerun because the workflow reported failure without failed jobs to inspect.") |
| 84 | + ;; |
| 85 | + skipped-no-transient-match) |
| 86 | + lines+=("- Action: Skipped rerun because at least one failed job did not match the transient GitHub-side signatures.") |
| 87 | + ;; |
| 88 | + skipped-uninspectable-logs) |
| 89 | + lines+=("- Action: Skipped rerun because at least one failed job log could not be downloaded through the GitHub Actions API.") |
| 90 | + ;; |
| 91 | + esac |
| 92 | + |
| 93 | + printf '%s\n' "${lines[@]}" |
| 94 | +} |
| 95 | + |
| 96 | +write_summary_output() { |
| 97 | + local summary="$1" |
| 98 | + local delimiter="SUMMARY_$(date +%s%N)" |
| 99 | + |
| 100 | + { |
| 101 | + printf 'summary<<%s\n' "${delimiter}" |
| 102 | + printf '%s\n' "${summary}" |
| 103 | + printf '%s\n' "${delimiter}" |
| 104 | + } >> "${GITHUB_OUTPUT}" |
| 105 | +} |
| 106 | + |
| 107 | +write_status_and_summary() { |
| 108 | + local status="$1" |
| 109 | + local summary |
| 110 | + |
| 111 | + summary="$(build_summary "${status}")" |
| 112 | + |
| 113 | + printf 'status=%s\n' "${status}" >> "${GITHUB_OUTPUT}" |
| 114 | + write_summary_output "${summary}" |
| 115 | +} |
| 116 | + |
| 117 | +log_matches_transient_signature() { |
| 118 | + local log_file="$1" |
| 119 | + |
| 120 | + grep -Eiq \ |
| 121 | + "RPC failed; HTTP 5[0-9][0-9]|expected flush after ref listing|expected 'packfile'|remote:[[:space:]]+Internal Server Error|requested URL returned error:[[:space:]]*5[0-9][0-9]|fatal:[[:space:]]+unable to access 'https://github\\.com/.*': The requested URL returned error:[[:space:]]*5[0-9][0-9]" \ |
| 122 | + "${log_file}" |
| 123 | +} |
| 124 | + |
| 125 | +download_job_logs() { |
| 126 | + local job_id="$1" |
| 127 | + local output_file="$2" |
| 128 | + |
| 129 | + curl \ |
| 130 | + -sS -L \ |
| 131 | + -H "Accept: application/vnd.github+json" \ |
| 132 | + -H "Authorization: Bearer ${GH_TOKEN}" \ |
| 133 | + -H "X-GitHub-Api-Version: 2022-11-28" \ |
| 134 | + -o "${output_file}" \ |
| 135 | + -w '%{http_code}' \ |
| 136 | + "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/jobs/${job_id}/logs" |
| 137 | +} |
| 138 | + |
| 139 | +if [ "${run_attempt}" -ge "${max_run_attempts}" ]; then |
| 140 | + write_status_and_summary "skipped-run-attempt-limit" |
| 141 | + |
| 142 | + exit 0 |
| 143 | +fi |
| 144 | + |
| 145 | +jobs_json="$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/jobs?per_page=100")" |
| 146 | +failed_jobs_json="$(jq -c '.jobs[] | select(.conclusion == "failure")' <<< "${jobs_json}")" |
| 147 | + |
| 148 | +if [ -z "${failed_jobs_json}" ]; then |
| 149 | + write_status_and_summary "skipped-no-failed-jobs" |
| 150 | + |
| 151 | + exit 0 |
| 152 | +fi |
| 153 | + |
| 154 | +while IFS= read -r failed_job; do |
| 155 | + [ -n "${failed_job}" ] || continue |
| 156 | + |
| 157 | + job_id="$(jq -r '.id' <<< "${failed_job}")" |
| 158 | + job_name="$(jq -r '.name' <<< "${failed_job}")" |
| 159 | + failed_jobs_csv="$(csv_append "${failed_jobs_csv}" "${job_name}")" |
| 160 | + |
| 161 | + temporary_log_file="$(mktemp)" |
| 162 | + |
| 163 | + log_status_code="$(download_job_logs "${job_id}" "${temporary_log_file}")" |
| 164 | + |
| 165 | + if [ "${log_status_code}" != "200" ]; then |
| 166 | + uninspectable_jobs_csv="$(csv_append "${uninspectable_jobs_csv}" "${job_name} (${log_status_code})")" |
| 167 | + rm -f "${temporary_log_file}" |
| 168 | + write_status_and_summary "skipped-uninspectable-logs" |
| 169 | + |
| 170 | + exit 0 |
| 171 | + fi |
| 172 | + |
| 173 | + if ! log_matches_transient_signature "${temporary_log_file}"; then |
| 174 | + rm -f "${temporary_log_file}" |
| 175 | + write_status_and_summary "skipped-no-transient-match" |
| 176 | + |
| 177 | + exit 0 |
| 178 | + fi |
| 179 | + |
| 180 | + matched_jobs_csv="$(csv_append "${matched_jobs_csv}" "${job_name}")" |
| 181 | + rm -f "${temporary_log_file}" |
| 182 | +done <<< "${failed_jobs_json}" |
| 183 | + |
| 184 | +gh api -X POST "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/rerun-failed-jobs" >/dev/null |
| 185 | + |
| 186 | +write_status_and_summary "rerun-requested" |
0 commit comments