Scheduled #26311
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Scheduled | |
| on: | |
| schedule: | |
| - cron: '45 * * * *' | |
| workflow_dispatch: | |
| concurrency: | |
| group: "scheduled" | |
| cancel-in-progress: true | |
| jobs: | |
| check: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| configs: ${{ steps.repo_check.outputs.configs }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - uses: mikefarah/yq@0f4fb8d35ec1a939d78dd6862f494d19ec589f19 # v4.52.5 | |
| - name: Set up Python | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| with: | |
| python-version: '3.10' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install pyyaml | |
| - name: Generate config.yaml | |
| run: | | |
| python generate_config.py | |
| echo "Generated config.yaml for workflow use" | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 | |
| with: | |
| username: ${{ vars.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| - id: repo_check | |
| env: | |
| CONCURRENT_IMAGE_PULL: ${{ vars.CONCURRENT_IMAGE_PULL }} | |
| run: | | |
| # This script reads config.yaml, platforms.yaml and runners.yaml to generate a list of configurations to build and deploy | |
| # It will: | |
| # - check the source respository for the latest commit hash | |
| # - check if the built image exists in our dockerhub registry | |
| # - generate a list of configurations to build and deploy | |
| CONFIG_FILE="config.yaml" | |
| PLATFORMS_FILE="platforms.yaml" | |
| RUNNERS_FILE="runners.yaml" | |
| OVERRIDES_FILE="runner_overrides.yaml" | |
| # Create a temporary directory for storing intermediate results | |
| TEMP_DIR=$(mktemp -d) | |
| # Ensure the temporary directory is removed when the script exits | |
| trap "rm -rf $TEMP_DIR" EXIT | |
| process_commits() { | |
| local LINE=$1 | |
| local SOURCE_REPOSITORY=$2 | |
| local SOURCE_REF=$3 | |
| local TARGET_REPOSITORY=$4 | |
| local TARGET_TAG=$5 | |
| local CLIENT="${TARGET_REPOSITORY#*/}" | |
| local RESPONSE=$(curl -s -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ | |
| "https://api.github.com/repos/${SOURCE_REPOSITORY}/commits/${SOURCE_REF}?per_page=1") | |
| local COMMIT_HASH_FULL=$(echo "$RESPONSE" | jq -r '.sha') | |
| local COMMIT_HASH=$(echo "$COMMIT_HASH_FULL" | cut -c1-7) | |
| if [[ -z "$COMMIT_HASH" || "$COMMIT_HASH" == "null" ]]; then | |
| # Log error but don't exit; just skip this configuration | |
| echo "[LINE:$LINE] Error fetching commit hash for ${SOURCE_REPOSITORY}#${SOURCE_REF}, skipping." | |
| return | |
| fi | |
| local configOutput="${TEMP_DIR}/${LINE}_commits.json" | |
| touch $configOutput | |
| echo "{\"line\": \"$LINE\", \"commit_hash\": \"$COMMIT_HASH\", \"commit_hash_full\": \"$COMMIT_HASH_FULL\"}," >> $configOutput | |
| } | |
| process_image() { | |
| local LINE=$1 | |
| local IMAGE=$2 | |
| local URL=$3 | |
| local IMAGE_TAG=$4 | |
| local imageOutput="${TEMP_DIR}/${LINE}_image.json" | |
| touch $imageOutput | |
| # Classify the Docker Hub response so we only trigger a build on an | |
| # authoritative "tag missing" answer. Retries absorb transient | |
| # failures (rate limits, 5xx, connection resets, non-JSON bodies). | |
| local exists="unknown" | |
| local attempt response http_code body count | |
| for attempt in 1 2 3; do | |
| # `|| true` is load-bearing: the surrounding script runs under | |
| # `bash -e`, which aborts the subshell on a failed command | |
| # substitution — so a curl timeout here would kill the background | |
| # process_image mid-function and leave an empty output file, | |
| # breaking the aggregator with "bad array subscript". | |
| response=$(curl -s --max-time 20 -w $'\n%{http_code}' "$URL" || true) | |
| if [ -z "$response" ]; then | |
| sleep $((attempt * 2)) | |
| continue | |
| fi | |
| http_code=$(printf '%s' "$response" | tail -n1) | |
| body=$(printf '%s' "$response" | sed '$d') | |
| if [ "$http_code" = "200" ]; then | |
| # Require a well-formed tags payload and exact-match the tag | |
| # name so a substring hit (e.g. `foo-abc` vs `foo-abcdef`) | |
| # can't mask a missing tag. `|| true` again: jq -e exits | |
| # non-zero on parse errors / null results and would otherwise | |
| # abort the subshell under bash -e. | |
| count=$(echo "$body" | jq -e --arg tag "$IMAGE_TAG" '[.results[]? | select(.name == $tag)] | length' 2>/dev/null || true) | |
| if [ -n "$count" ]; then | |
| if [ "$count" -gt 0 ]; then | |
| exists=true | |
| else | |
| exists=false | |
| fi | |
| break | |
| fi | |
| elif [ "$http_code" = "404" ]; then | |
| # Repository itself doesn't exist yet on Docker Hub — first build needs to push. | |
| if echo "$body" | jq -e '.message == "object not found"' >/dev/null 2>&1; then | |
| exists=false | |
| break | |
| fi | |
| fi | |
| echo "[LINE:$LINE] Docker Hub check attempt $attempt inconclusive for ${IMAGE} (http=$http_code), retrying." >&2 | |
| sleep $((attempt * 2)) | |
| done | |
| if [ "$exists" = "unknown" ]; then | |
| echo "[LINE:$LINE] WARN: Docker Hub check inconclusive for ${IMAGE} after retries, skipping build." >&2 | |
| fi | |
| echo "{\"line\": \"$LINE\", \"image\": \"$IMAGE\", \"exists\": \"$exists\"}" >> $imageOutput | |
| } | |
| # Get commit hashes for each configuration in parallel | |
| while IFS=$'\t' read -r LINE SOURCE_REPOSITORY SOURCE_REF TARGET_REPOSITORY TARGET_TAG; do | |
| process_commits "$LINE" "$SOURCE_REPOSITORY" "$SOURCE_REF" "$TARGET_REPOSITORY" "$TARGET_TAG" & | |
| done < <(yq -r 'to_entries | map_values({"value":.value, "index":.key}) | .[] | [.index, .value.source.repository, .value.source.ref, .value.target.repository, .value.target.tag] | @tsv' "$CONFIG_FILE") | |
| wait | |
| # Initialize JSON arrays | |
| COMMITS="[" | |
| # Concatenate results, ensuring files exist before attempting to read | |
| for file in $TEMP_DIR/*_commits.json; do | |
| if [ -f "$file" ]; then | |
| COMMITS+=$(cat "$file") | |
| fi | |
| done | |
| # Remove trailing commas and close JSON arrays | |
| COMMITS="${COMMITS%,}]" | |
| echo "Checking if images exist in dockerhub..." | |
| while IFS=$'\t' read -r LINE SOURCE_REPOSITORY SOURCE_REF TARGET_REPOSITORY TARGET_TAG; do | |
| # get the image commit hash from LINE | |
| COMMIT_HASH=$(echo "$COMMITS" | jq -r --arg LINE "$LINE" '.[] | select(.line == $LINE) | .commit_hash') | |
| IMAGE_TAG="${TARGET_TAG}-${COMMIT_HASH}" | |
| IMAGE="${TARGET_REPOSITORY}:${IMAGE_TAG}" | |
| URL="https://hub.docker.com/v2/repositories/${TARGET_REPOSITORY}/tags?page_size=25&page=1&ordering=&name=${IMAGE_TAG}" | |
| process_image "$LINE" "$IMAGE" "$URL" "$IMAGE_TAG" & | |
| done < <(yq -r 'to_entries | map_values({"value":.value, "index":.key}) | .[] | [.index, .value.source.repository, .value.source.ref, .value.target.repository, .value.target.tag] | @tsv' "$CONFIG_FILE") | |
| wait | |
| declare -A images | |
| # Concatenate results, ensuring files exist before attempting to read. | |
| # Skip empty / malformed files so a single aborted process_image subshell | |
| # can't crash the whole check job with "bad array subscript". | |
| for file in $TEMP_DIR/*_image.json; do | |
| if [ -f "$file" ] && [ -s "$file" ]; then | |
| LINE=$(jq -r '.line // empty' "$file" 2>/dev/null || true) | |
| IMAGE=$(jq -r '.image // empty' "$file" 2>/dev/null || true) | |
| EXISTS=$(jq -r '.exists // empty' "$file" 2>/dev/null || true) | |
| if [ -n "$IMAGE" ]; then | |
| images[$IMAGE]=$EXISTS | |
| else | |
| echo "WARN: dropping malformed image-check result $(basename "$file"): $(head -c 200 "$file")" >&2 | |
| fi | |
| else | |
| echo "WARN: image-check result $(basename "$file") is empty; skipping" >&2 | |
| fi | |
| done | |
| CONFIGS="configs=[" | |
| echo "Generating configuration files..." | |
| while IFS=$'\t' read -r LINE SOURCE_REPOSITORY SOURCE_REF SOURCE_PATCH TARGET_REPOSITORY TARGET_TAG; do | |
| # get the image commit hash from LINE | |
| COMMIT_HASH=$(echo "$COMMITS" | jq -r --arg LINE "$LINE" '.[] | select(.line == $LINE) | .commit_hash') | |
| COMMIT_HASH_FULL=$(echo "$COMMITS" | jq -r --arg LINE "$LINE" '.[] | select(.line == $LINE) | .commit_hash_full') | |
| IMAGE_TAG="${TARGET_TAG}-${COMMIT_HASH}" | |
| IMAGE="${TARGET_REPOSITORY}:${IMAGE_TAG}" | |
| CLIENT="${TARGET_REPOSITORY#*/}" | |
| # Only build when Docker Hub authoritatively reported the tag as missing. | |
| # "unknown" (check failed after retries) or missing entries are skipped to | |
| # avoid burning runner time on rebuilds that the existence check couldn't verify. | |
| if [ "${images[$IMAGE]}" != "true" ] && [ "${images[$IMAGE]}" != "false" ]; then | |
| echo "[LINE:$LINE] Skipping ${IMAGE}: Docker Hub existence check was inconclusive." | |
| fi | |
| if [ "${images[$IMAGE]}" == "false" ]; then | |
| # Handle platforms and runners, ensuring output files are created even if empty | |
| platforms=$(yq e ".$CLIENT[]" "$PLATFORMS_FILE") | |
| platformsArr="" | |
| for platform in $platforms; do | |
| runner=$(yq e ".\"$platform\"" "$RUNNERS_FILE") | |
| override=$(yq e ".\"$CLIENT\".\"$platform\"" "$OVERRIDES_FILE") | |
| if [ "$override" != "null" ] && [ -n "$override" ]; then | |
| runner=$override | |
| fi | |
| slug=$(echo "$platform" | tr '/' '-') | |
| platformsArr+="{\\\"platform\\\": \\\"$platform\\\", \\\"runner\\\": \\\"$runner\\\", \\\"slug\\\": \\\"$slug\\\"}," | |
| done | |
| platformsArr="${platformsArr%,}" | |
| # convert to string | |
| platformsOutput="{\"platforms\": \"[$platformsArr]\"}" | |
| # Convert NONE placeholder back to empty string for patch | |
| if [ "$SOURCE_PATCH" == "NONE" ]; then | |
| SOURCE_PATCH="" | |
| fi | |
| CONFIGS+=$(echo "$(yq -r -o=json ".[${LINE}]" "$CONFIG_FILE" | jq --argjson plat "$platformsOutput" --arg commit "$COMMIT_HASH_FULL" --arg patch "$SOURCE_PATCH" '. + $plat + {source_commit: $commit, source_patch: $patch}'),") | |
| fi | |
| done < <(yq -r 'to_entries | map_values({"value":.value, "index":.key}) | .[] | [.index, .value.source.repository, .value.source.ref, .value.source.patch // "NONE", .value.target.repository, .value.target.tag] | @tsv' "$CONFIG_FILE") | |
| # Remove trailing commas and close JSON arrays | |
| CONFIGS="${CONFIGS%,}]" | |
| echo "CONFIGS: $CONFIGS" | |
| echo $CONFIGS >> $GITHUB_OUTPUT | |
| deploy: | |
| needs: check | |
| if: ${{ needs.check.outputs.configs != '[]' && needs.check.outputs.configs != '' }} | |
| uses: ./.github/workflows/deploy.yml | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: ${{fromJson(needs.check.outputs.configs)}} | |
| name: ${{ matrix.config.source.repository }}#${{ matrix.config.source.ref }} ${{ matrix.config.target.tag }} | |
| with: | |
| source_repository: ${{ matrix.config.source.repository }} | |
| source_ref: ${{ matrix.config.source.ref }} | |
| source_commit: ${{ matrix.config.source_commit }} | |
| source_patch: ${{ matrix.config.source_patch }} | |
| build_script: ${{ matrix.config.build_script }} | |
| build_args: "${{ matrix.config.build_args }}" | |
| target_tag: ${{ matrix.config.target.tag }} | |
| target_repository: ${{ matrix.config.target.repository }} | |
| target_dockerfile: ${{ matrix.config.target.dockerfile }} | |
| platforms: ${{ matrix.config.platforms }} | |
| harbor_registry: "${{ vars.HARBOR_REGISTRY }}" | |
| HARBOR_USERNAME: "${{ vars.HARBOR_USERNAME }}" | |
| DOCKER_USERNAME: "${{ vars.DOCKER_USERNAME }}" | |
| GOPROXY: "${{ vars.GOPROXY }}" | |
| secrets: | |
| DOCKER_PASSWORD: "${{ secrets.DOCKER_PASSWORD }}" | |
| HARBOR_PASSWORD: "${{ secrets.HARBOR_PASSWORD }}" | |
| MACOS_PASSWORD: "${{ secrets.MACOS_PASSWORD }}" | |
| notify: | |
| name: Discord Notification | |
| runs-on: ubuntu-latest | |
| needs: | |
| - check | |
| - deploy | |
| if: cancelled() || failure() | |
| steps: | |
| - name: Notify | |
| uses: nobrayner/discord-webhook@1766a33bf571acdcc0678f00da4fb83aad01ebc7 # v1 | |
| with: | |
| github-token: ${{ secrets.github_token }} | |
| discord-webhook: ${{ secrets.DISCORD_WEBHOOK }} |