From 083ead6a9ef8ee206bfbf8f937d6bfad19ad20cb Mon Sep 17 00:00:00 2001 From: Costas Papastathis Date: Wed, 28 May 2025 16:51:30 +0200 Subject: [PATCH 1/3] adding the script files --- stack/scripts/create.sh | 6 +- stack/scripts/publish.sh | 194 ++++++++++++++++++++++++++++++++++++++ stack/scripts/receipts.sh | 6 +- stack/scripts/test.sh | 63 +++++++++---- 4 files changed, 242 insertions(+), 27 deletions(-) create mode 100755 stack/scripts/publish.sh diff --git a/stack/scripts/create.sh b/stack/scripts/create.sh index b15a8a89..a25c3018 100755 --- a/stack/scripts/create.sh +++ b/stack/scripts/create.sh @@ -6,7 +6,7 @@ set -o pipefail readonly PROG_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" readonly ROOT_DIR="$(cd "${PROG_DIR}/.." && pwd)" readonly BIN_DIR="${ROOT_DIR}/.bin" -readonly IMAGES_JSON="${ROOT_DIR}/stacks/images.json" +readonly IMAGES_JSON="${ROOT_DIR}/images.json" # shellcheck source=SCRIPTDIR/.util/tools.sh source "${PROG_DIR}/.util/tools.sh" @@ -67,7 +67,9 @@ function main() { if [ -f "${IMAGES_JSON}" ]; then # we need to copy images.json for inclusion in the build image defaultStackPath=$(jq -r '.images[] | select(.name == "default") | .config_dir' "${IMAGES_JSON}") - cp $IMAGES_JSON $ROOT_DIR/$defaultStackPath/images.json + if [ -n "$defaultStackPath" ]; then + cp $IMAGES_JSON "${ROOT_DIR}/${defaultStackPath}/images.json" + fi fi # if stack or build argument is provided but not both, then throw an error diff --git a/stack/scripts/publish.sh b/stack/scripts/publish.sh new file mode 100755 index 00000000..58a98b01 --- /dev/null +++ b/stack/scripts/publish.sh @@ -0,0 +1,194 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +readonly PROG_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly ROOT_DIR="$(cd "${PROG_DIR}/.." && pwd)" +readonly BIN_DIR="${ROOT_DIR}/.bin" + +# shellcheck source=SCRIPTDIR/.util/tools.sh +source "${PROG_DIR}/.util/tools.sh" + +# shellcheck source=SCRIPTDIR/.util/print.sh +source "${PROG_DIR}/.util/print.sh" + +if [[ $BASH_VERSINFO -lt 4 ]]; then + util::print::error "Before running this script please update Bash to v4 or higher (e.g. on OSX: \$ brew install bash)" +fi + +function main() { + local build_ref=() + local run_ref=() + local image_ref=() + local build_archive="" + local run_archive="" + local image_archive="" + + while [[ "${#}" != 0 ]]; do + case "${1}" in + --help|-h) + shift 1 + usage + exit 0 + ;; + + --build-ref) + build_ref+=("${2}") + shift 2 + ;; + + --run-ref) + run_ref+=("${2}") + shift 2 + ;; + + --build-archive) + build_archive=${2} + shift 2 + ;; + + --run-archive) + run_archive=${2} + shift 2 + ;; + + --image-ref) + image_ref+=("${2}") + shift 2 + ;; + + --image-archive) + image_archive=${2} + shift 2 + ;; + + "") + # skip if the argument is empty + shift 1 + ;; + + *) + util::print::error "unknown argument \"${1}\"" + esac + done + + if [[ ${#image_ref[@]} != 0 || -n "$image_archive" ]]; then + if ((${#image_ref[@]} == 0)); then + util::print::error "--image-ref is required [Example: docker.io/paketobuildpacks/foo:latest]" + fi + + if [ -z "$image_archive" ]; then + util::print::error "--image-archive is required [Example: ./path/to/image.oci]" + fi + else + if ((${#build_ref[@]} == 0)); then + util::print::error "--build-ref is required [Example: docker.io/paketobuildpacks/foo:latest]" + fi + + if ((${#run_ref[@]} == 0)); then + util::print::error "--run-ref is required [Example: docker.io/paketobuildpacks/foo:1.0.0]" + fi + + if ((${#run_ref[@]} != ${#build_ref[@]})); then + util::print::error "must have the same number of --build-ref and --run-ref arguments" + fi + + if [ -z "$build_archive" ]; then + util::print::error "--build-archive is required [Example: ./path/to/build.oci]" + fi + + if [ -z "$run_archive" ]; then + util::print::error "--run-archive is required [Example: ./path/to/run.oci]" + fi + fi + + tools::install + + if [[ ${#image_ref[@]} != 0 || -n "$image_archive" ]]; then + stack::publish::image \ + "$image_archive" \ + "${#image_ref[@]}" \ + "${image_ref[@]}" + else + stack::publish \ + "$build_archive" \ + "$run_archive" \ + "${#build_ref[@]}" \ + "${build_ref[@]}" \ + "${#run_ref[@]}" \ + "${run_ref[@]}" + fi +} + +function usage() { + cat <<-USAGE +publish.sh [OPTIONS] + +Publishes the stack using the existing OCI image archives. + +OPTIONS + --build-ref list of build references to publish to [Required if --image-ref is not provided] + --run-ref list of run references to publish to [Required if --image-ref is not provided] + --build-archive path to the build OCI archive file [Required if --image-ref is not provided] + --run-archive path to the run OCI archive file [Required if --image-ref is not provided] + --image-ref list of image references to publish to [Required if --build-ref and --run-ref are not provided] + --image-archive path to the image OCI archive file [Required if --build-ref and --run-ref are not provided] + --help -h prints the command usage +USAGE +} + +function tools::install() { + util::tools::jam::install \ + --directory "${BIN_DIR}" +} + +function stack::publish() { + local build_archive="$1" + local run_archive="$2" + + # bash can't easily pass arrays, they all get merged into one list of arguments + # so we pass the lengths & extract the arrays from the single argument list + local build_ref_len="$3" # length of build ref array + local build_ref=("${@:4:$build_ref_len}") # pull out build_ref array + local run_len_slot=$(( 4 + build_ref_len)) # location of run_ref length + local run_ref_len="${*:$run_len_slot:1}" # length of run ref array + local run_ref_slot=$(( 1 + run_len_slot)) # location of run_ref array + local run_ref=("${@:$run_ref_slot:$run_ref_len}") # pull out run_ref array + + # iterate over build_ref & run_ref, they will be the same length + local len=${#build_ref[@]} + for (( i=0; i -t Token used to download assets from GitHub (e.g. jam, pack, etc) (optional) - --test-only-stacks Runs the tests of the stacks passed to this argument (e.g. java-8 nodejs-16) (optional) - --help -h prints the command usage + --clean -c Clears contents of stack output directory before running tests + --token -t Token used to download assets from GitHub (e.g. jam, pack, etc) (optional) + --test-only-stacks Runs the tests of the stacks passed to this argument (e.g. java-8 nodejs-16) (optional) + --validate-stack-builds Validates that the stack builds are present before running tests (optional) + --help -h Prints the command usage USAGE } @@ -175,17 +192,17 @@ function tools::install() { token="${1}" util::tools::jam::install \ - --directory "${STACK_DIR}/.bin" \ + --directory "${ROOT_DIR}/.bin" \ --token "${token}" util::tools::pack::install \ - --directory "${STACK_DIR}/.bin" \ + --directory "${ROOT_DIR}/.bin" \ --token "${token}" util::tools::skopeo::check util::tools::crane::install \ - --directory "${STACK_DIR}/.bin" \ + --directory "${ROOT_DIR}/.bin" \ --token "${token}" } @@ -193,8 +210,9 @@ function tests::run() { util::print::title "Run Stack Acceptance Tests" export CGO_ENABLED=0 + export JAM_PATH="${ROOT_DIR}/.bin/jam" testout=$(mktemp) - pushd "${STACK_DIR}" > /dev/null + pushd "${ROOT_DIR}" > /dev/null if GOMAXPROCS="${GOMAXPROCS:-4}" go test -count=1 -timeout 0 ./... -v -run Acceptance | tee "${testout}"; then util::tools::tests::checkfocus "${testout}" util::print::success "** GO Test Succeeded **" @@ -210,7 +228,12 @@ function stack_builds_exist() { while IFS= read -r image; do stack_output_dir=$(echo "${image}" | jq -r '.output_dir') - if ! [[ -f "${STACK_DIR}/${stack_output_dir}/build.oci" ]] || ! [[ -f "${STACK_DIR}/${stack_output_dir}/run.oci" ]]; then + is_build_image_necessary=$(echo "${image}" | jq -r '.create_build_image // false') + + if ! [[ -f "${ROOT_DIR}/${stack_output_dir}/run.oci" ]]; then + stack_output_builds_exist="false" + fi + if [[ ! -f "${ROOT_DIR}/${stack_output_dir}/build.oci" && "${is_build_image_necessary}" == true ]]; then stack_output_builds_exist="false" fi done <<<"$STACK_IMAGES" @@ -221,7 +244,7 @@ function stack_builds_exist() { function clean::stacks(){ while read -r image; do output_dir=$(echo "${image}" | jq -r '.output_dir') - rm -rf "${STACK_DIR}/${output_dir}" + rm -rf "${ROOT_DIR}/${output_dir}" done <<<"$STACK_IMAGES" } From b382dfa0ced8339694d2736c9313b9d81754a411 Mon Sep 17 00:00:00 2001 From: Costas Papastathis Date: Wed, 28 May 2025 16:56:05 +0200 Subject: [PATCH 2/3] adding github workflows --- stack/.github/workflows/create-release.yml | 1416 ++++++++++++++--- stack/.github/workflows/push-image.yml | 268 +++- stack/.github/workflows/test-pull-request.yml | 6 +- 3 files changed, 1377 insertions(+), 313 deletions(-) diff --git a/stack/.github/workflows/create-release.yml b/stack/.github/workflows/create-release.yml index d47550f6..c022f70f 100644 --- a/stack/.github/workflows/create-release.yml +++ b/stack/.github/workflows/create-release.yml @@ -2,7 +2,7 @@ name: Create Release on: schedule: - - cron: '27 2,14 * * *' # daily at 02:27 and 14:27 UTC + - cron: '27 2,14 * * *' # daily at 02:27 and 14:27 UTC push: branches: - main @@ -22,102 +22,512 @@ on: concurrency: release env: - BUILD_RECEIPT_FILENAME: "build-receipt.cyclonedx.json" - RUN_RECEIPT_FILENAME: "run-receipt.cyclonedx.json" + STACKS_FILEPATH: "images.json" PATCHED_USNS_FILENAME: "patched-usns.json" - BUILD_DIFF_ADDED_FILENAME: "build-diff-added.json" - BUILD_DIFF_MODIFIED_FILENAME: "build-diff-modified.json" - BUILD_DIFF_REMOVED_FILENAME: "build-diff-removed.json" - RUN_DIFF_ADDED_FILENAME: "run-diff-added.json" - RUN_DIFF_MODIFIED_FILENAME: "run-diff-modified.json" - RUN_DIFF_REMOVED_FILENAME: "run-diff-removed.json" jobs: + preparation: + name: Preparation + runs-on: ubuntu-22.04 + outputs: + architectures: ${{ steps.lookup.outputs.platforms }} + archs_added: ${{ steps.lookup.outputs.platforms_added }} + github_repo_name: ${{ steps.repo.outputs.github_repo_name }} + os_codename: ${{ steps.repo.outputs.os_codename }} + os_name: ${{ steps.repo.outputs.os_name }} + repo_type: ${{ steps.repo.outputs.repo_type }} + polling_type: ${{ steps.polling-os-type.outputs.polling_type }} + registry_repo_name: ${{ steps.repo.outputs.registry_repo_name }} + repo_owner: ${{ steps.repo.outputs.repo_owner }} + stacks: ${{ steps.get-stacks.outputs.stacks }} + stacks_added: ${{ steps.get-stacks.outputs.stacks_added }} + stack_files_dir: ${{ steps.get-stacks.outputs.stack_files_dir }} + support_usns: ${{ steps.polling-os-type.outputs.support_usns }} + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 # gets full history + + - name: Get stacks images + id: get-stacks + run: | + + if [ -f ${{ env.STACKS_FILEPATH }} ]; then + current_stacks=$(jq -c '[.images[] ]' ${{ env.STACKS_FILEPATH }} | jq -c '.[]') + stack_files_dir="$(dirname "${{ env.STACKS_FILEPATH }}")" + else + current_stacks=$(jq -n -c '[ + { + "name": "default", + "config_dir": "stack", + "output_dir": "build", + "build_image": "build", + "run_image": "run", + "create_build_image": true, + "is_new": false, + "pattern_image_registry_name": "build_image_run_image-os_codename-stack_type", + "pattern_assets_prefix": "os_codename-stack_type-repo_type-version-arch-build_image_run_image" + } + ]' | jq -c '.[]') + stack_files_dir="stack" + fi + + latest_tag_or_commit=$(git describe --tags --abbrev=0 2>/dev/null || git rev-parse HEAD) + + previous_images_json_exists=$(git ls-tree --name-only "$latest_tag_or_commit" "$stack_files_dir/images.json") + + if [[ -n $previous_images_json_exists ]]; then + git show $latest_tag_or_commit:"$stack_files_dir/images.json" >./previous_images.json + else + echo '{"images":[]}' >./previous_images.json + fi + + # Start with an empty array + stacks=$(jq -n -c '[]') + + for c_stack in $current_stacks; do + c_stack_name=$(echo "$c_stack" | jq -r '.name') + stack_exists=$(jq --arg c_stack_name "$c_stack_name" '.images[] | select(.name == $c_stack_name)' ./previous_images.json) + c_stack=$(jq --argjson c_stack "$c_stack" '. + {is_new: false}' <<< "$c_stack") + stacks=$(jq --argjson c_stack "$c_stack" '. + [$c_stack]' <<< "$stacks") + done + + # Filter stacks array to include the minimum number of attributes + stacks=$(echo "$stacks" | jq 'map({ + name, + config_dir, + output_dir, + build_image, + run_image, + create_build_image, + base_build_container_image, + base_run_container_image, + is_new, + stack_type, + pattern_image_registry_name, + pattern_assets_prefix + })') + + stacks=$(jq -c <<< "$stacks" ) + echo "stacks=$stacks" + echo "stacks=$stacks" >> "$GITHUB_OUTPUT" + echo "stack_files_dir=$stack_files_dir" >> "$GITHUB_OUTPUT" + + - name: Polling OS type + id: polling-os-type + run: | + support_usns=true + + if [[ -f ${{ env.STACKS_FILEPATH }} ]]; then + support_usns=$( jq '.support_usns' ${{ env.STACKS_FILEPATH }} ) + fi + + if [ $support_usns == true ]; then + echo "polling_type=usn" >> "$GITHUB_OUTPUT" + else + echo "polling_type=hash" >> "$GITHUB_OUTPUT" + fi + + echo "support_usns=${support_usns}" >> "$GITHUB_OUTPUT" + + - name: Get Repository Name + id: repo + run: | + full=${{ github.repository }} + # Strip off the org and slash from repo name + # paketo-buildpacks/jammy-base-stack --> jammy-base-stack + repo=$(echo "${full}" | sed 's/^.*\///') + echo "github_repo_name=${repo}" >> "$GITHUB_OUTPUT" + + # Strip off 'stack' suffix from repo name + # paketo-buildpacks/jammy-base-stack --> jammy-base + registry_repo="${repo//-stack/}" + echo "registry_repo_name=${registry_repo}" >> "$GITHUB_OUTPUT" + + # translates 'paketo-buildpacks' to 'paketobuildpacks' + repo_owner="${GITHUB_REPOSITORY_OWNER/-/}" + printf "repo_owner=%s\n" "${repo_owner}" >> "$GITHUB_OUTPUT" + + os_codename="" + if [[ -f ${{ env.STACKS_FILEPATH }} ]]; then + os_codename=$( jq -r '.os_codename' ${{ env.STACKS_FILEPATH }} ) + os_name=$( jq -r '.os_name' ${{ env.STACKS_FILEPATH }} ) + repo_type=$( jq -r '.repo_type' ${{ env.STACKS_FILEPATH }} ) + else + # Extract codename from repo name: + # paketo-buildpacks/jammy-tiny-stack --> jammy + os_codename="$(echo "${{ github.repository }}" | sed 's/^.*\///' | sed 's/\-.*$//')" + os_name="" + repo_type="stack" + fi + + echo "repo_type=${repo_type}" >> "$GITHUB_OUTPUT" + echo "os_codename=${os_codename}" >> "$GITHUB_OUTPUT" + echo "os_name=${os_name}" >> "$GITHUB_OUTPUT" + + - name: Lookup Supported Architectures + id: lookup + run: | + + set -euo pipefail + shopt -s inherit_errexit + + if [[ -f "$STACKS_FILEPATH" ]]; then + current_platforms="$(jq -c '[.platforms[] | { name: sub("linux/"; "") , is_new: true }]' $STACKS_FILEPATH )" + else + current_platforms='[{"name": "amd64", "is_new": true}]' + fi + + latest_tag_or_commit=$(git describe --tags --abbrev=0 2>/dev/null || git rev-parse HEAD) + + exists_previous_stacks_filepath=$(git ls-tree --name-only "$latest_tag_or_commit" "$STACKS_FILEPATH") + + if [[ -n $exists_previous_stacks_filepath ]]; then + git show "$latest_tag_or_commit:$STACKS_FILEPATH" >./previous_stacks.json + previous_platforms=$(jq -c '[.platforms[] | { name: sub("linux/"; "") , is_new: false }]' ./previous_stacks.json) + else + previous_platforms='[{"name": "amd64", "is_new": false}]' + fi + + platforms=$(echo "$current_platforms" | jq -c --argjson prev "$previous_platforms" ' + map( + . as $curr | + ($prev[] | select(.name == $curr.name)) // $curr + ) + ') + + echo "current_platforms=$current_platforms" + echo "previous_platforms=$previous_platforms" + echo "platforms=$platforms" + echo "platforms=${platforms}" >> "$GITHUB_OUTPUT" + + # The following job is specific to Ubuntu images. It checks for new + # USNs (Ubuntu Security Notices) and triggers the flow to create + # a new release with the latest images that have the USNs patched. poll_usns: name: Poll USNs - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 + needs: [preparation] + if: ${{ needs.preparation.outputs.polling_type == 'usn' }} + strategy: + matrix: + stacks: ${{ fromJSON(needs.preparation.outputs.stacks) }} + arch: ${{ fromJSON(needs.preparation.outputs.architectures) }} outputs: - usns: ${{ steps.usns.outputs.usns }} + usns: ${{ steps.new_usns.outputs.usns }} steps: + - name: Check for Previous Releases + id: check_previous + run: | + gh auth status + # shellcheck disable=SC2046 + if [ $(gh api "/repos/${{ github.repository }}/releases" | jq -r 'length') -eq 0 ]; then + echo "exists=false" + echo "exists=false" >> "$GITHUB_OUTPUT" + else + echo "exists=true" + echo "exists=true" >> "$GITHUB_OUTPUT" + fi + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Generate receipt asset patterns + id: receipt_pattern + run: | + os_name="${{ needs.preparation.outputs.os_name}}" + os_codename="${{ needs.preparation.outputs.os_codename }}" + stack_type="${{ matrix.stacks.stack_type }}" + repo_type="${{ needs.preparation.outputs.repo_type }}" + build_image="${{ matrix.stacks.build_image }}" + run_image="${{ matrix.stacks.run_image }}" + pattern_assets_prefix="${{ matrix.stacks.pattern_assets_prefix }}" + arch=${{ matrix.arch.name }} + + if [[ "${{ matrix.arch.name }}" == "amd64" ]]; then + if [[ "$pattern_assets_prefix" == "os_codename-stack_type-repo_type-version-arch-build_image_run_image" ]]; then + build="${os_codename}-${stack_type}-${repo_type}-\\d+\\.\\d+(\\.\\d+)?-${build_image}-receipt.cyclonedx.json" + run="${os_codename}-${stack_type}-${repo_type}-\\d+\\.\\d+(\\.\\d+)?-${run_image}-receipt.cyclonedx.json" + elif [[ "$pattern_assets_prefix" == "os_name-os_codename-build_image_run_image-stack_type-version-arch" ]]; then + build="${os_name}-${os_codename}-${build_image}-${stack_type}-\\d+\\.\\d+(\\.\\d+)?-receipt.cyclonedx.json" + run="${os_name}-${os_codename}-${run_image}-${stack_type}-\\d+\\.\\d+(\\.\\d+)?-receipt.cyclonedx.json" + elif [[ "$pattern_assets_prefix" == "os_name-os_codename-build_image_run_image-version-arch" ]]; then + build="${os_name}-${os_codename}-${build_image}-\\d+\\.\\d+(\\.\\d+)?-receipt.cyclonedx.json" + run="${os_name}-${os_codename}-${run_image}-\\d+\\.\\d+(\\.\\d+)?-receipt.cyclonedx.json" + fi + else + if [[ "$pattern_assets_prefix" == "os_codename-stack_type-repo_type-version-arch-build_image_run_image" ]]; then + build="${os_codename}-${stack_type}-${repo_type}-\\d+\\.\\d+(\\.\\d+)?-${arch}-${build_image}-receipt.cyclonedx.json" + run="${os_codename}-${stack_type}-${repo_type}-\\d+\\.\\d+(\\.\\d+)?-${arch}-${run_image}-receipt.cyclonedx.json" + elif [[ "$pattern_assets_prefix" == "os_name-os_codename-build_image_run_image-stack_type-version-arch" ]]; then + build="${os_name}-${os_codename}-${build_image}-${stack_type}-\\d+\\.\\d+(\\.\\d+)?-${arch}-receipt.cyclonedx.json" + run="${os_name}-${os_codename}-${run_image}-${stack_type}-\\d+\\.\\d+(\\.\\d+)?-${arch}-receipt.cyclonedx.json" + elif [[ "$pattern_assets_prefix" == "os_name-os_codename-build_image_run_image-version-arch" ]]; then + build="${os_name}-${os_codename}-${build_image}-\\d+\\.\\d+(\\.\\d+)?-${arch}-receipt.cyclonedx.json" + run="${os_name}-${os_codename}-${run_image}-\\d+\\.\\d+(\\.\\d+)?-${arch}-receipt.cyclonedx.json" + fi + fi + + echo "build=${build}" >> "$GITHUB_OUTPUT" + echo "run=${run}" >> "$GITHUB_OUTPUT" + + - name: Write Empty Previous Build Receipt + if: ${{ matrix.stacks.create_build_image == false || matrix.arch.is_new == true || matrix.stacks.is_new == true || steps.check_previous.outputs.exists == 'false' }} + run: | + echo '{"components":[]}' > "./${{ matrix.arch.name }}-previous-build-receipt-${{ matrix.stacks.name }}" + + - name: Write Empty Previous Run Receipt + if: ${{ matrix.arch.is_new == true || matrix.stacks.is_new == true || steps.check_previous.outputs.exists == 'false' }} + run: | + echo '{"components":[]}' > "./${{ matrix.arch.name }}-previous-run-receipt-${{ matrix.stacks.name }}" + - name: Find and Download Previous Build Receipt + if: ${{ matrix.stacks.create_build_image == true && matrix.arch.is_new == false && matrix.stacks.is_new == false && steps.check_previous.outputs.exists == 'true' }} id: previous_build uses: paketo-buildpacks/github-config/actions/release/find-and-download-asset@main with: - asset_pattern: "${{ env.BUILD_RECEIPT_FILENAME }}" + asset_pattern: "${{ steps.receipt_pattern.outputs.build }}" search_depth: 1 repo: ${{ github.repository }} - output_path: "/github/workspace/${{ env.BUILD_RECEIPT_FILENAME }}" + output_path: "/github/workspace/${{ matrix.arch.name }}-previous-build-receipt-${{ matrix.stacks.name }}" token: ${{ secrets.PAKETO_BOT_GITHUB_TOKEN }} + strict: true - name: Find and Download Previous Run Receipt + if: ${{ matrix.arch.is_new == false && matrix.stacks.is_new == false && steps.check_previous.outputs.exists == 'true' }} id: previous_run uses: paketo-buildpacks/github-config/actions/release/find-and-download-asset@main with: - asset_pattern: "${{ env.RUN_RECEIPT_FILENAME }}" + asset_pattern: "${{ steps.receipt_pattern.outputs.run }}" search_depth: 1 repo: ${{ github.repository }} - output_path: "/github/workspace/${{ env.RUN_RECEIPT_FILENAME }}" + output_path: "/github/workspace/${{ matrix.arch.name }}-previous-run-receipt-${{ matrix.stacks.name }}" token: ${{ secrets.PAKETO_BOT_GITHUB_TOKEN }} + strict: true + + - name: Write Empty Package list + if: ${{ steps.previous_run.outputs.output_path == '' }} + run: | + echo '[]' > "./${{ matrix.arch.name }}-package-list-${{ matrix.stacks.name }}" - name: Get Package List id: packages - if: ${{ steps.previous_build.outputs.output_path != '' && steps.previous_run.outputs.output_path != '' }} + if: ${{ steps.previous_run.outputs.output_path != '' }} uses: paketo-buildpacks/github-config/actions/stack/generate-package-list@main with: - build_receipt: "${{ github.workspace }}/${{ env.BUILD_RECEIPT_FILENAME }}" - run_receipt: "${{ github.workspace }}/${{ env.RUN_RECEIPT_FILENAME }}" + build_receipt: "${{ github.workspace }}/${{ matrix.arch.name }}-previous-build-receipt-${{ matrix.stacks.name }}" + run_receipt: "${{ github.workspace }}/${{ matrix.arch.name }}-previous-run-receipt-${{ matrix.stacks.name }}" + output_path: "${{ github.workspace }}/${{ matrix.arch.name }}-package-list-${{ matrix.stacks.name }}" + + - name: Generate USNs download asset pattern + id: usn_download_pattern + run: | + os_name="${{ needs.preparation.outputs.os_name }}" + os_codename="${{ needs.preparation.outputs.os_codename }}" + stack_type="${{ matrix.stacks.stack_type }}" + repo_type="${{ needs.preparation.outputs.repo_type }}" + build_image="${{ matrix.stacks.build_image }}" + run_image="${{ matrix.stacks.run_image }}" + pattern_assets_prefix="${{ matrix.stacks.pattern_assets_prefix }}" + arch_name="${{ matrix.arch.name }}" + + asset_prefix="" + if [[ "$pattern_assets_prefix" == "os_name-os_codename-build_image_run_image-stack_type-version-arch" ]]; then + asset_prefix="${os_name}-${os_codename}-${stack_type}" + elif [[ "$pattern_assets_prefix" == "os_name-os_codename-build_image_run_image-version-arch" ]]; then + asset_prefix="${os_name}-${os_codename}" + fi + + arch_prefix="" + if [[ "${arch_name}" = "amd64" ]]; then + arch_prefix="" + else + arch_prefix="${arch_name}" + fi + + pattern=$( + echo '[ + "'"$asset_prefix"'", + "\\d+.\\d+(.\\d+)?", + "'"$arch_prefix"'", + "${{ env.PATCHED_USNS_FILENAME }}" + ]' | jq -r 'map(select(length > 0)) | join("-")' + ) + echo "pattern=$pattern" >> "$GITHUB_OUTPUT" + + - name: Write Empty Previous Patched USNs + if: ${{ matrix.arch.is_new == true || matrix.stacks.is_new == true || steps.check_previous.outputs.exists == 'false' }} + run: | + echo '[]' > "./${{ matrix.arch.name }}-${{ matrix.stacks.name }}-${{ env.PATCHED_USNS_FILENAME }}-previous" - name: Find and Download Previous Patched USNs + if: ${{ steps.check_previous.outputs.exists == 'true' && matrix.arch.is_new == false && matrix.stacks.is_new == false }} id: download_patched uses: paketo-buildpacks/github-config/actions/release/find-and-download-asset@main with: - asset_pattern: "${{ env.PATCHED_USNS_FILENAME }}" + asset_pattern: "${{ steps.usn_download_pattern.outputs.pattern }}" search_depth: "-1" # Search all releases repo: ${{ github.repository }} - output_path: "/github/workspace/${{ env.PATCHED_USNS_FILENAME }}" + output_path: "/github/workspace/${{ matrix.arch.name }}-${{ matrix.stacks.name }}-${{ env.PATCHED_USNS_FILENAME }}-previous" + token: ${{ secrets.PAKETO_BOT_GITHUB_TOKEN }} + + - name: Get New USNs + id: usns + uses: paketo-buildpacks/github-config/actions/stack/get-usns@main + with: + distribution: ${{ needs.preparation.outputs.os_codename }} + packages_filepath: "./${{ matrix.arch.name }}-package-list-${{ matrix.stacks.name }}" + last_usns_filepath: "./${{ matrix.arch.name }}-${{ matrix.stacks.name }}-${{ env.PATCHED_USNS_FILENAME }}-previous" + usns_output_path: "./${{ matrix.arch.name }}-${{ matrix.stacks.name }}-${{ env.PATCHED_USNS_FILENAME }}" + + - name: Upload USNs file + uses: actions/upload-artifact@v4 + with: + name: "${{ matrix.arch.name }}-${{ matrix.stacks.name }}-${{ env.PATCHED_USNS_FILENAME }}" + path: "${{ matrix.arch.name }}-${{ matrix.stacks.name }}-${{ env.PATCHED_USNS_FILENAME }}" + + - name: Are any new USNs + id: new_usns + run: | + are_new_usns=$(cat "${{ matrix.arch.name }}-${{ matrix.stacks.name }}-${{ env.PATCHED_USNS_FILENAME }}" | jq 'length > 0') + + if [ $are_new_usns == true ]; then + echo "usns=true" + echo "usns=true" >> "$GITHUB_OUTPUT" + fi + + # The job below checks if new images are available on the registry + # based on the sha256 checksum. If yes, it triggers the flow + # to create a new release with the latest images + poll_images: + name: Poll Images based on the hash code + runs-on: ubuntu-22.04 + if: ${{ needs.preparation.outputs.polling_type == 'hash' }} + needs: preparation + strategy: + matrix: + stacks: ${{ fromJSON(needs.preparation.outputs.stacks) }} + arch: ${{ fromJSON(needs.preparation.outputs.architectures) }} + outputs: + images_need_update: ${{ steps.compare_previous_and_current_sha256_hash_codes.outputs.images_need_update }} + steps: + - name: Generate hash code asset patterns + id: hashcode_pattern + run: | + os_codename="${{ needs.preparation.outputs.os_codename }}" + stack_type="${{ matrix.stacks.stack_type }}" + repo_type="${{ needs.preparation.outputs.repo_type }}" + build_image="${{ matrix.stacks.build_image }}" + run_image="${{ matrix.stacks.run_image }}" + pattern_assets_prefix="${{ matrix.stacks.pattern_assets_prefix }}" + arch=${{ matrix.arch.name }} + + if [[ "${{ matrix.arch.name }}" = "amd64" ]]; then + if [[ "$pattern_assets_prefix" == "os_codename-stack_type-repo_type-version-arch-build_image_run_image" ]]; then + build="${os_codename}-${stack_type}-${repo_type}-\\d+.\\d+(.\\d+)?-${build_image}.oci.sha256" + run="${os_codename}-${stack_type}-${repo_type}-\\d+.\\d+(.\\d+)?-${run_image}.oci.sha256" + fi + else + if [[ "$pattern_assets_prefix" == "os_codename-stack_type-repo_type-version-arch-build_image_run_image" ]]; then + build="${os_codename}-${stack_type}-${repo_type}-\\d+.\\d+(.\\d+)?-${arch}-${build_image}.oci.sha256" + run="${os_codename}-${stack_type}-${repo_type}-\\d+.\\d+(.\\d+)?-${arch}-${run_image}.oci.sha256" + fi + fi + + echo "build=${build}" >> "$GITHUB_OUTPUT" + echo "run=${run}" >> "$GITHUB_OUTPUT" + + - name: Find and Download Previous build image hash code of stack ${{ matrix.stacks.build_image }} + if: ${{ matrix.stacks.create_build_image == true }} + uses: paketo-buildpacks/github-config/actions/release/find-and-download-asset@main + with: + asset_pattern: "${{ steps.hashcode_pattern.outputs.build }}" + search_depth: 1 + repo: ${{ github.repository }} + output_path: "./previous_${{ matrix.arch.name }}-${{ matrix.stacks.build_image }}.oci.sha256" token: ${{ secrets.PAKETO_BOT_GITHUB_TOKEN }} - - name: Output Patched USNs as JSON String - id: patched - if: ${{ steps.download_patched.outputs.output_path != '' }} + - name: Find and Download Previous run image hash code of stack ${{ matrix.stacks.run_image }} + uses: paketo-buildpacks/github-config/actions/release/find-and-download-asset@main + with: + asset_pattern: "${{ steps.hashcode_pattern.outputs.run }}" + search_depth: 1 + repo: ${{ github.repository }} + output_path: "./previous_${{ matrix.arch.name }}-${{ matrix.stacks.run_image }}.oci.sha256" + token: ${{ secrets.PAKETO_BOT_GITHUB_TOKEN }} + + - name: Get current build image hash code of ${{ matrix.stacks.name }} stack with arch ${{ matrix.arch.name }} + if: ${{ matrix.stacks.create_build_image == true }} run: | - patched=$(jq --compact-output . < "${GITHUB_WORKSPACE}/${{ env.PATCHED_USNS_FILENAME }}") - printf "patched=%s\n" "${patched}" >> "$GITHUB_OUTPUT" + skopeo inspect --format "{{.Digest}}" ${{ matrix.stacks.base_build_container_image }} > ./hash-code-current-build-image-${{ matrix.arch.name }}-${{ matrix.stacks.name }} - - name: Get Stack Distribution Name - id: distro + - name: Get current run image hash code of ${{ matrix.stacks.name }} stack with arch ${{ matrix.arch.name }} run: | - # Extract distro from repo name: - # paketo-buildpacks/jammy-tiny-stack --> jammy - distro="$(echo "${{ github.repository }}" | sed 's/^.*\///' | sed 's/\-.*$//')" - echo "Ubuntu distribution: ${distro}" - printf "distro=%s\n" "${distro}" >> "$GITHUB_OUTPUT" + skopeo inspect --format "{{.Digest}}" ${{ matrix.stacks.base_run_container_image }} > ./hash-code-current-run-image-${{ matrix.arch.name }}-${{ matrix.stacks.name }} + + - name: Create empty image hash codes + run: | + if [ ! -f "./hash-code-current-build-image-${{ matrix.arch.name }}-${{ matrix.stacks.name }}" ]; then + touch "./hash-code-current-build-image-${{ matrix.arch.name }}-${{ matrix.stacks.name }}" + fi + if [ ! -f "./hash-code-current-run-image-${{ matrix.arch.name }}-${{ matrix.stacks.name }}" ]; then + touch "./hash-code-current-run-image-${{ matrix.arch.name }}-${{ matrix.stacks.name }}" + fi - - name: Get New USNs - id: usns - uses: paketo-buildpacks/github-config/actions/stack/get-usns@main + - name: Upload run image hash code + uses: actions/upload-artifact@v4 with: - distribution: ${{ steps.distro.outputs.distro }} - packages: ${{ steps.packages.outputs.packages }} - last_usns: ${{ steps.patched.outputs.patched }} + name: hash-code-current-run-image-${{ matrix.arch.name }}-${{ matrix.stacks.name }} + path: hash-code-current-run-image-${{ matrix.arch.name }}-${{ matrix.stacks.name }} + if-no-files-found: error - create_stack: - name: Create Stack - needs: poll_usns - if: ${{ ! ( needs.poll_usns.outputs.usns == '[]' && github.event_name == 'schedule' ) }} + - name: Upload build image hash code + if: ${{ matrix.stacks.create_build_image == true }} + uses: actions/upload-artifact@v4 + with: + name: hash-code-current-build-image-${{ matrix.arch.name }}-${{ matrix.stacks.name }} + path: hash-code-current-build-image-${{ matrix.arch.name }}-${{ matrix.stacks.name }} + if-no-files-found: error + + - name: Compare previous and current hash codes + id: compare_previous_and_current_sha256_hash_codes + run: | + if [ "$(cat previous_${{ matrix.arch.name }}-${{ matrix.stacks.run_image }}.oci.sha256)" != "$(cat hash-code-current-run-image-${{ matrix.arch.name }}-${{ matrix.stacks.name }})" ]; then + echo "images_need_update=true" >> "$GITHUB_OUTPUT" + fi + + if [ "${{ matrix.stacks.create_build_image }}" == "true" ]; then + + if [ "$(cat previous_${{ matrix.arch.name }}-${{ matrix.stacks.build_image }}.oci.sha256)" != "$(cat hash-code-current-build-image-${{ matrix.arch.name }}-${{ matrix.stacks.name }})" ]; then + echo "images_need_update=true" >> "$GITHUB_OUTPUT" + fi + fi + + # If there is no change on the usns, and there is no change on the image hash codes + # and the event is schedule, then there is no need to run below workflow as nothing has changed + stack_files_changed: + name: Determine If Stack Files Changed + runs-on: ubuntu-22.04 + needs: [ preparation, poll_usns, poll_images ] + if: ${{ + !failure() && !cancelled() && + !( + (needs.poll_images.outputs.images_need_update == null && + needs.poll_usns.outputs.usns == null ) && + github.event_name == 'schedule' + ) }} outputs: stack_files_changed: ${{ steps.compare.outputs.stack_files_changed }} - runs-on: ubuntu-24.04 steps: - - name: Checkout - uses: actions/checkout@v3 + - name: Checkout With History + uses: actions/checkout@v4 with: fetch-depth: 0 # gets full history - - name: Determine If Stack Files Changed + - name: Compare With Previous Release id: compare run: | # shellcheck disable=SC2046 - changed="$(git diff --name-only $(git describe --tags --abbrev=0) -- stack)" + changed=$(git diff --name-only $(git describe --tags --abbrev=0) -- "${{ needs.preparation.outputs.stack_files_dir }}") if [ -z "${changed}" ] then echo "No relevant files changed since previous release." @@ -128,62 +538,132 @@ jobs: echo "stack_files_changed=true" >> "$GITHUB_OUTPUT" fi - - name: Create stack + usns_or_sha_changed: + name: USNs or SHAs have changed + runs-on: ubuntu-22.04 + outputs: + changed: ${{ steps.usns_or_sha_changed.outputs.changed }} + needs: [ preparation, poll_usns, poll_images ] + if: ${{ !failure() && !cancelled() }} + steps: + - name: Check USNs or SHAs have changed + id: usns_or_sha_changed + run: | + if [ '${{ needs.poll_images.result }}' != 'skipped' ]; then + echo "Poll images job did not skip" + if [ '${{ needs.poll_images.outputs.images_need_update }}' = 'true' ]; then + echo "SHAs have changed" + echo "changed=true" >> "$GITHUB_OUTPUT" + else + echo "SHAs have not changed" + echo "changed=false" >> "$GITHUB_OUTPUT" + fi + exit 0 + fi + + if [ '${{ needs.poll_usns.result }}' != 'skipped' ]; then + echo "Poll USNs did not skip" + if [ '${{ needs.poll_usns.outputs.usns }}' = 'true' ]; then + echo "USNs have changed" + echo "changed=true" >> "$GITHUB_OUTPUT" + else + echo "USNs have not changed" + echo "changed=false" >> "$GITHUB_OUTPUT" + fi + exit 0 + fi + + create_stack: + name: Create Stack + needs: [ preparation, usns_or_sha_changed ] + # If there is no change on the usns, and there is no change on the image hash codes + # and the event is schedule, then there is no need to run below workflow as nothing has changed + if: ${{ !failure() && !cancelled() && !( needs.usns_or_sha_changed.outputs.changed == 'false' && github.event_name == 'schedule') }} + runs-on: ubuntu-22.04 + strategy: + matrix: + stacks: ${{ fromJSON(needs.preparation.outputs.stacks) }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + # https://github.com/docker/setup-qemu-action + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Create stack ${{ matrix.stacks.name }} id: create-stack run: | - scripts/create.sh + scripts/create.sh --stack-dir ${{ matrix.stacks.config_dir }} \ + --build-dir ${{ matrix.stacks.output_dir }} - name: Generate Package Receipts id: receipts run: | - scripts/receipts.sh --build-image "${GITHUB_WORKSPACE}/build/build.oci" \ - --run-image "${GITHUB_WORKSPACE}/build/run.oci" \ - --build-receipt ${{ env.BUILD_RECEIPT_FILENAME }} \ - --run-receipt ${{ env.RUN_RECEIPT_FILENAME }} - echo "build_receipt=${{ env.BUILD_RECEIPT_FILENAME }}" >> "$GITHUB_OUTPUT" - echo "run_receipt=${{ env.RUN_RECEIPT_FILENAME }}" >> "$GITHUB_OUTPUT" + scripts/receipts.sh --build-image "${{ matrix.stacks.output_dir }}/build.oci" \ + --run-image "${{ matrix.stacks.output_dir }}/run.oci" \ + --build-receipt current-build-receipt-${{ matrix.stacks.name }} \ + --run-receipt current-run-receipt-${{ matrix.stacks.name }} - name: Upload run image uses: actions/upload-artifact@v4 with: - name: current-run-image - path: build/run.oci + name: current-run-image-${{ matrix.stacks.name }} + path: "${{ matrix.stacks.output_dir }}/run.oci" + if-no-files-found: error - name: Upload build image + if: ${{ matrix.stacks.create_build_image == true }} uses: actions/upload-artifact@v4 with: - name: current-build-image - path: build/build.oci + name: current-build-image-${{ matrix.stacks.name }} + path: "${{ matrix.stacks.output_dir }}/build.oci" + if-no-files-found: error - name: Upload Build receipt + if: ${{ matrix.stacks.create_build_image == true }} uses: actions/upload-artifact@v4 with: - name: current-build-receipt - path: ${{ steps.receipts.outputs.build_receipt }} + name: current-build-receipt-${{ matrix.stacks.name }} + path: "*current-build-receipt-${{ matrix.stacks.name }}" + if-no-files-found: error - name: Upload Run receipt uses: actions/upload-artifact@v4 with: - name: current-run-receipt - path: ${{ steps.receipts.outputs.run_receipt }} + name: current-run-receipt-${{ matrix.stacks.name }} + path: "*current-run-receipt-${{ matrix.stacks.name }}" + if-no-files-found: error diff: name: Diff Packages + env: + BUILD_DIFF_ADDED_FILENAME: "build-diff-added.json" + BUILD_DIFF_MODIFIED_FILENAME: "build-diff-modified.json" + BUILD_DIFF_REMOVED_FILENAME: "build-diff-removed.json" + RUN_DIFF_ADDED_FILENAME: "run-diff-added.json" + RUN_DIFF_MODIFIED_FILENAME: "run-diff-modified.json" + RUN_DIFF_REMOVED_FILENAME: "run-diff-removed.json" + if: ${{ !cancelled() && !failure() && needs.create_stack.result != 'skipped' }} outputs: removed_with_force: ${{ steps.removed_with_force.outputs.packages_removed }} - packages_changed: ${{ steps.compare.outputs.packages_changed }} - needs: [ create_stack ] - runs-on: ubuntu-24.04 + needs: [ create_stack, preparation ] + runs-on: ubuntu-22.04 + strategy: + matrix: + stacks: ${{ fromJSON(needs.preparation.outputs.stacks) }} + arch: ${{ fromJSON(needs.preparation.outputs.architectures) }} steps: - - name: Download Build Receipt - uses: actions/download-artifact@v4 + - name: Checkout With History + uses: actions/checkout@v4 with: - name: current-build-receipt + fetch-depth: 0 # gets full history - - name: Download Run Receipt + - name: Download Current Receipt(s) uses: actions/download-artifact@v4 with: - name: current-run-receipt + pattern: current-*-receipt-${{ matrix.stacks.name }} + path: receipt-files - name: Check for Previous Releases id: check_previous @@ -192,215 +672,423 @@ jobs: # shellcheck disable=SC2046 if [ $(gh api "/repos/${{ github.repository }}/releases" | jq -r 'length') -eq 0 ]; then echo "exists=false" >> "$GITHUB_OUTPUT" - exit 0 + else + echo "exists=true" >> "$GITHUB_OUTPUT" fi - echo "exists=true" >> "$GITHUB_OUTPUT" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Write Empty Previous Receipts - if: ${{ steps.check_previous.outputs.exists == 'false' }} + if: ${{ steps.check_previous.outputs.exists == 'false' || matrix.arch.is_new == true || matrix.stacks.is_new == true }} + run: | + if [ ${{ matrix.stacks.create_build_image }} == true ]; then + if [ ! -f "./${{ matrix.arch.name }}-previous-build-receipt-${{ matrix.stacks.name }}" ]; then + echo '{"components":[]}' > "./${{ matrix.arch.name }}-previous-build-receipt-${{ matrix.stacks.name }}" + fi + fi + if [ ! -f "./${{ matrix.arch.name }}-previous-run-receipt-${{ matrix.stacks.name }}" ]; then + echo '{"components":[]}' > "./${{ matrix.arch.name }}-previous-run-receipt-${{ matrix.stacks.name }}" + fi + + - name: Previous receipt download filename pattern + if: ${{ steps.check_previous.outputs.exists == 'true' && matrix.arch.is_new == false && matrix.stacks.is_new == false }} + id: previous_receipt_download_pattern run: | - echo '{"components":[]}' > "${{ github.workspace }}/previous-${{ env.BUILD_RECEIPT_FILENAME }}" - echo '{"components":[]}' > "${{ github.workspace }}/previous-${{ env.RUN_RECEIPT_FILENAME }}" + os_name="${{ needs.preparation.outputs.os_name }}" + os_codename="${{ needs.preparation.outputs.os_codename }}" + stack_type="${{ matrix.stacks.stack_type }}" + repo_type="${{ needs.preparation.outputs.repo_type }}" + build_image="${{ matrix.stacks.build_image }}" + run_image="${{ matrix.stacks.run_image }}" + pattern_assets_prefix="${{ matrix.stacks.pattern_assets_prefix }}" + arch=${{ matrix.arch.name }} + + if [[ "${{ matrix.arch.name }}" == "amd64" ]]; then + if [[ "$pattern_assets_prefix" == "os_codename-stack_type-repo_type-version-arch-build_image_run_image" ]]; then + build="${os_codename}-${stack_type}-${repo_type}-\\d+\\.\\d+(\\.\\d+)?-${build_image}-receipt.cyclonedx.json" + run="${os_codename}-${stack_type}-${repo_type}-\\d+\\.\\d+(\\.\\d+)?-${run_image}-receipt.cyclonedx.json" + elif [[ "$pattern_assets_prefix" == "os_name-os_codename-build_image_run_image-stack_type-version-arch" ]]; then + build="${os_name}-${os_codename}-${build_image}-${stack_type}-\\d+\\.\\d+(\\.\\d+)?-receipt.cyclonedx.json" + run="${os_name}-${os_codename}-${run_image}-${stack_type}-\\d+\\.\\d+(\\.\\d+)?-receipt.cyclonedx.json" + elif [[ "$pattern_assets_prefix" == "os_name-os_codename-build_image_run_image-version-arch" ]]; then + build="${os_name}-${os_codename}-${build_image}-\\d+\\.\\d+(\\.\\d+)?-receipt.cyclonedx.json" + run="${os_name}-${os_codename}-${run_image}-\\d+\\.\\d+(\\.\\d+)?-receipt.cyclonedx.json" + fi + else + if [[ "$pattern_assets_prefix" == "os_codename-stack_type-repo_type-version-arch-build_image_run_image" ]]; then + build="${os_codename}-${stack_type}-${repo_type}-\\d+\\.\\d+(\\.\\d+)?-${arch}-${build_image}-receipt.cyclonedx.json" + run="${os_codename}-${stack_type}-${repo_type}-\\d+\\.\\d+(\\.\\d+)?-${arch}-${run_image}-receipt.cyclonedx.json" + elif [[ "$pattern_assets_prefix" == "os_name-os_codename-build_image_run_image-stack_type-version-arch" ]]; then + build="${os_name}-${os_codename}-${build_image}-${stack_type}-\\d+\\.\\d+(\\.\\d+)?-${arch}-receipt.cyclonedx.json" + run="${os_name}-${os_codename}-${run_image}-${stack_type}-\\d+\\.\\d+(\\.\\d+)?-${arch}-receipt.cyclonedx.json" + elif [[ "$pattern_assets_prefix" == "os_name-os_codename-build_image_run_image-version-arch" ]]; then + build="${os_name}-${os_codename}-${build_image}-\\d+\\.\\d+(\\.\\d+)?-${arch}-receipt.cyclonedx.json" + run="${os_name}-${os_codename}-${run_image}-\\d+\\.\\d+(\\.\\d+)?-${arch}-receipt.cyclonedx.json" + fi + fi + + echo "build_filename=${build}" >> "$GITHUB_OUTPUT" + echo "run_filename=${run}" >> "$GITHUB_OUTPUT" - name: Find and Download Previous Build Receipt - if: ${{ steps.check_previous.outputs.exists == 'true' }} + if: ${{ matrix.stacks.create_build_image == true && steps.check_previous.outputs.exists == 'true' && matrix.arch.is_new == false && matrix.stacks.is_new == false }} uses: paketo-buildpacks/github-config/actions/release/find-and-download-asset@main with: - asset_pattern: "${{ env.BUILD_RECEIPT_FILENAME }}" + asset_pattern: "${{ steps.previous_receipt_download_pattern.outputs.build_filename }}" search_depth: 1 repo: ${{ github.repository }} - output_path: "/github/workspace/previous-${{ env.BUILD_RECEIPT_FILENAME }}" + output_path: "/github/workspace/${{ matrix.arch.name }}-previous-build-receipt-${{ matrix.stacks.name }}" token: ${{ secrets.PAKETO_BOT_GITHUB_TOKEN }} - name: Find and Download Previous Run Receipt - if: ${{ steps.check_previous.outputs.exists == 'true' }} + if: ${{ steps.check_previous.outputs.exists == 'true' && matrix.arch.is_new == false && matrix.stacks.is_new == false }} uses: paketo-buildpacks/github-config/actions/release/find-and-download-asset@main with: - asset_pattern: "${{ env.RUN_RECEIPT_FILENAME }}" + asset_pattern: "${{ steps.previous_receipt_download_pattern.outputs.run_filename }}" search_depth: 1 repo: ${{ github.repository }} - output_path: "/github/workspace/previous-${{ env.RUN_RECEIPT_FILENAME }}" + output_path: "/github/workspace/${{ matrix.arch.name }}-previous-run-receipt-${{ matrix.stacks.name }}" token: ${{ secrets.PAKETO_BOT_GITHUB_TOKEN }} - name: Compare Build Packages id: build_diff + if: ${{ matrix.stacks.create_build_image == true }} uses: paketo-buildpacks/github-config/actions/stack/diff-package-receipts@main with: - previous: "/github/workspace/previous-${{ env.BUILD_RECEIPT_FILENAME }}" - current: "/github/workspace/${{ env.BUILD_RECEIPT_FILENAME }}" + previous: "/github/workspace/${{ matrix.arch.name }}-previous-build-receipt-${{ matrix.stacks.name }}" + current: "receipt-files/current-build-receipt-${{ matrix.stacks.name }}/${{ matrix.arch.name }}-current-build-receipt-${{ matrix.stacks.name }}" added_diff_file: "/github/workspace/${{ env.BUILD_DIFF_ADDED_FILENAME }}" modified_diff_file: "/github/workspace/${{ env.BUILD_DIFF_MODIFIED_FILENAME }}" removed_diff_file: "/github/workspace/${{ env.BUILD_DIFF_REMOVED_FILENAME }}" - - name: Upload Build Diff Files - uses: actions/upload-artifact@v4 - with: - name: build-diff-files - path: | - ${{ env.BUILD_DIFF_ADDED_FILENAME }} - ${{ env.BUILD_DIFF_MODIFIED_FILENAME }} - ${{ env.BUILD_DIFF_REMOVED_FILENAME }} - - name: Compare Run Packages id: run_diff uses: paketo-buildpacks/github-config/actions/stack/diff-package-receipts@main with: - previous: "/github/workspace/previous-${{ env.RUN_RECEIPT_FILENAME }}" - current: "/github/workspace/${{ env.RUN_RECEIPT_FILENAME }}" + previous: "/github/workspace/${{ matrix.arch.name }}-previous-run-receipt-${{ matrix.stacks.name }}" + current: "receipt-files/current-run-receipt-${{ matrix.stacks.name }}/${{ matrix.arch.name }}-current-run-receipt-${{ matrix.stacks.name }}" added_diff_file: "/github/workspace/${{ env.RUN_DIFF_ADDED_FILENAME }}" modified_diff_file: "/github/workspace/${{ env.RUN_DIFF_MODIFIED_FILENAME }}" removed_diff_file: "/github/workspace/${{ env.RUN_DIFF_REMOVED_FILENAME }}" - - name: Upload Run Diff Files - uses: actions/upload-artifact@v4 - with: - name: run-diff-files - path: | - ${{ env.RUN_DIFF_ADDED_FILENAME }} - ${{ env.RUN_DIFF_MODIFIED_FILENAME }} - ${{ env.RUN_DIFF_REMOVED_FILENAME }} - - - name: Fail If Packages Removed id: removed_with_force run: | - build=$(jq '. | length' "${BUILD_REMOVED}") - echo "Build packages removed: ${build}" + if [ "${{ matrix.stacks.create_build_image }}" == "true" ]; then + build=$(jq '. | length' "${BUILD_REMOVED}") + echo "Build packages removed: ${build}" + else + build=0 + fi run=$(jq '. | length' "${RUN_REMOVED}") echo "Run packages removed: ${run}" # only fail if packages are removed AND the release has not been forced - if [ "${build}" -gt 0 ] || [ "${run}" -gt 0 ]; then - if [ "${{ github.event.inputs.force }}" != 'true' ]; then + if ( [ "${build}" -gt 0 ] && [ "${{ matrix.stacks.create_build_image }}" == "true" ] ) || [ "${run}" -gt 0 ]; then + if [ "${{ github.event.inputs.force }}" != "true" ]; then echo "Packages removed without authorization. Stack cannot be released." exit 1 else echo "packages_removed=true" >> "$GITHUB_OUTPUT" fi - else - echo "packages_removed=false" >> "$GITHUB_OUTPUT" fi env: BUILD_REMOVED: "${{ github.workspace }}/${{ env.BUILD_DIFF_REMOVED_FILENAME }}" RUN_REMOVED: "${{ github.workspace }}/${{ env.RUN_DIFF_REMOVED_FILENAME }}" - - name: Determine If Packages Changed + - name: Create/Upload variable artifacts + id: variable_artifacts + run: | + diffs_dir="diff-${{ matrix.arch.name }}-${{ matrix.stacks.name }}" + mkdir -p "$diffs_dir" + + if [ "${{ matrix.stacks.create_build_image }}" == "true" ]; then + cp "${{ github.workspace }}/${{ env.BUILD_DIFF_ADDED_FILENAME }}" "$diffs_dir/build_added" + cp "${{ github.workspace }}/${{ env.BUILD_DIFF_MODIFIED_FILENAME }}" "$diffs_dir/build_modified" + cp "${{ github.workspace }}/${{ env.BUILD_DIFF_REMOVED_FILENAME }}" "$diffs_dir/build_removed_with_force" + else + # We generate empty build diffs for the release notes action not to break + echo null > "${{ github.workspace }}/${{ env.BUILD_DIFF_ADDED_FILENAME }}" + echo null > "${{ github.workspace }}/${{ env.BUILD_DIFF_MODIFIED_FILENAME }}" + echo null > "${{ github.workspace }}/${{ env.BUILD_DIFF_REMOVED_FILENAME }}" + fi + + cp "${{ github.workspace }}/${{ env.RUN_DIFF_ADDED_FILENAME }}" "$diffs_dir/run_added" + cp "${{ github.workspace }}/${{ env.RUN_DIFF_MODIFIED_FILENAME }}" "$diffs_dir/run_modified" + cp "${{ github.workspace }}/${{ env.RUN_DIFF_REMOVED_FILENAME }}" "$diffs_dir/run_removed_with_force" + + - name: Upload diff-${{ matrix.arch.name }}-${{ matrix.stacks.name }} + uses: actions/upload-artifact@v4 + with: + name: diff-${{ matrix.arch.name }}-${{ matrix.stacks.name }} + path: diff-${{ matrix.arch.name }}-${{ matrix.stacks.name }} + if-no-files-found: error + + - name: Download USN File(s) + if: ${{ needs.preparation.outputs.polling_type == 'usn' }} + uses: actions/download-artifact@v4 + with: + path: "patched-usns" + pattern: ${{ matrix.arch.name }}-*${{ env.PATCHED_USNS_FILENAME }} + merge-multiple: true + + - name: Get USNs + id: get-usns + if: ${{ needs.preparation.outputs.polling_type == 'usn' }} + run: | + usns=$(cat "patched-usns/${{ matrix.arch.name }}-${{ matrix.stacks.name }}-${{ env.PATCHED_USNS_FILENAME }}" | jq -c ) + echo "usns=$usns" >> "$GITHUB_OUTPUT" + + - name: Get Repository Name + id: repo_name + run: | + full=${{ github.repository }} + # Strip off the org and slash from repo name + # paketo-buildpacks/repo-name --> repo-name + repo=$(echo "${full}" | sed 's/^.*\///') + echo "github_repo_name=${repo}" >> "$GITHUB_OUTPUT" + + - name: Increment Tag + if: github.event.inputs.version == '' + id: semver + uses: paketo-buildpacks/github-config/actions/tag/increment-tag@main + with: + allow_head_tagged: true + + - name: Set Release Tag + id: tag + run: | + tag="${{ github.event.inputs.version }}" + if [ -z "${tag}" ]; then + tag="${{ steps.semver.outputs.tag }}" + fi + echo "tag=${tag}" >> "$GITHUB_OUTPUT" + + - name: Get registry build and run image names + id: registry_names + run: | + os_name="${{ needs.preparation.outputs.os_name }}" + os_codename="${{ needs.preparation.outputs.os_codename }}" + stack_type="${{ matrix.stacks.stack_type }}" + + pattern_image_registry_name="${{ matrix.stacks.pattern_image_registry_name }}" + + if [ $pattern_image_registry_name == "os_name-os_codename-build_image_run_image-stack_type" ]; then + build_image="${{ needs.preparation.outputs.repo_owner }}/${os_name}-${os_codename}-${{ matrix.stacks.build_image }}-${stack_type}:${{ steps.tag.outputs.tag }}" + run_image="${{ needs.preparation.outputs.repo_owner }}/${os_name}-${os_codename}-${{ matrix.stacks.run_image }}-${stack_type}:${{ steps.tag.outputs.tag }}" + elif [ $pattern_image_registry_name == "os_name-os_codename-build_image_run_image" ]; then + build_image="${{ needs.preparation.outputs.repo_owner }}/${os_name}-${os_codename}-${{ matrix.stacks.build_image }}:${{ steps.tag.outputs.tag }}" + run_image="${{ needs.preparation.outputs.repo_owner }}/${os_name}-${os_codename}-${{ matrix.stacks.run_image }}:${{ steps.tag.outputs.tag }}" + elif [ $pattern_image_registry_name == "build_image_run_image-os_codename-stack_type" ]; then + build_image="${{ needs.preparation.outputs.repo_owner }}/${{ matrix.stacks.build_image }}-${os_codename}-${stack_type}:${{ steps.tag.outputs.tag }}" + run_image="${{ needs.preparation.outputs.repo_owner }}/${{ matrix.stacks.run_image }}-${os_codename}-${stack_type}:${{ steps.tag.outputs.tag }}" + fi + + if [ "${{ matrix.stacks.create_build_image }}" != "true" ]; then + build_image="" + fi + + echo "build_image=${build_image}" >> "$GITHUB_OUTPUT" + echo "run_image=${run_image}" >> "$GITHUB_OUTPUT" + + - name: Create Release Notes per Arch and per Stack + id: notes_arc_stack + uses: paketo-buildpacks/github-config/actions/stack/release-notes@main + with: + build_image: ${{ steps.registry_names.outputs.build_image }} + run_image: ${{ steps.registry_names.outputs.run_image }} + build_packages_added: "/github/workspace/${{ env.BUILD_DIFF_ADDED_FILENAME }}" + build_packages_modified: "/github/workspace/${{ env.BUILD_DIFF_MODIFIED_FILENAME }}" + build_packages_removed_with_force: "/github/workspace/${{ env.BUILD_DIFF_REMOVED_FILENAME }}" + run_packages_added: "/github/workspace/${{ env.RUN_DIFF_ADDED_FILENAME }}" + run_packages_modified: "/github/workspace/${{ env.RUN_DIFF_MODIFIED_FILENAME }}" + run_packages_removed_with_force: "/github/workspace/${{ env.RUN_DIFF_REMOVED_FILENAME }}" + supports_usns: ${{ needs.preparation.outputs.support_usns }} + patched_usns: ${{ needs.get-usns.outputs.usns }} + release_body_file: "${{ matrix.arch.name }}-${{ matrix.stacks.name }}-release-notes.md" + + - name: Upload ${{ matrix.arch.name }} release notes file for stack ${{ matrix.stacks.name }} + uses: actions/upload-artifact@v4 + with: + name: "${{ matrix.arch.name }}-${{ matrix.stacks.name }}-release-notes.md" + path: "${{ matrix.arch.name }}-${{ matrix.stacks.name }}-release-notes.md" + + packages_changed: + name: Determine If Packages Changed + needs: [ diff, preparation ] + runs-on: ubuntu-22.04 + if: ${{ !cancelled() && !failure() && needs.diff.result != 'skipped' }} + strategy: + matrix: + stacks: ${{ fromJSON(needs.preparation.outputs.stacks) }} + arch: ${{ fromJSON(needs.preparation.outputs.architectures) }} + outputs: + packages_changed: ${{ steps.compare.outputs.packages_changed }} + steps: + - name: Download diff-${{ matrix.arch.name }}-${{ matrix.stacks.name }} + uses: actions/download-artifact@v4 + with: + name: diff-${{ matrix.arch.name }}-${{ matrix.stacks.name }} + + - name: Compare With Previous Release id: compare run: | # shellcheck disable=SC2153 - build_added=$(jq '. | length' "${BUILD_ADDED}") - echo "Build packages added: ${build_added}" - # shellcheck disable=SC2153 - build_modified=$(jq '. | length' "${BUILD_MODIFIED}") - echo "Build packages modified: ${build_modified}" + if [ "${{ matrix.stacks.create_build_image }}" == "true" ]; then + build_added=$(cat ./build_added | jq 'length') + echo "Build packages added: ${build_added}" + + # shellcheck disable=SC2153 + build_modified=$(cat ./build_modified | jq 'length') + echo "Build packages modified: ${build_modified}" + fi # shellcheck disable=SC2153 - run_added=$(jq '. | length' "${RUN_ADDED}") + run_added=$(cat ./run_added | jq 'length') echo "Run packages added: ${run_added}" # shellcheck disable=SC2153 - run_modified=$(jq '. | length' "${RUN_MODIFIED}") + run_modified=$(cat ./run_modified | jq 'length') echo "Run packages modified: ${run_modified}" - if [ "${build_added}" -eq 0 ] && [ "${build_modified}" -eq 0 ] && [ "${run_added}" -eq 0 ] && [ "${run_modified}" -eq 0 ]; then + if [ "${build_added:-0}" -eq 0 ] && [ "${build_modified:-0}" -eq 0 ] && [ "${run_added}" -eq 0 ] && [ "${run_modified}" -eq 0 ]; then echo "No packages changed." - echo "packages_changed=false" >> "$GITHUB_OUTPUT" + # We ommit setting "packages_changed" variable to false, + # as there is an edge case scenario overriding any true value due to parallelization else echo "Packages changed." echo "packages_changed=true" >> "$GITHUB_OUTPUT" fi - env: - BUILD_ADDED: "${{ github.workspace }}/${{ env.BUILD_DIFF_ADDED_FILENAME }}" - BUILD_MODIFIED: "${{ github.workspace }}/${{ env.BUILD_DIFF_MODIFIED_FILENAME }}" - RUN_ADDED: "${{ github.workspace }}/${{ env.RUN_DIFF_ADDED_FILENAME }}" - RUN_MODIFIED: "${{ github.workspace }}/${{ env.RUN_DIFF_MODIFIED_FILENAME }}" - test: name: Acceptance Test - needs: [ create_stack ] - runs-on: ubuntu-24.04 + needs: [ preparation, create_stack ] + if: ${{ !cancelled() && !failure() && needs.create_stack.result != 'skipped' }} + runs-on: ubuntu-22.04 steps: - name: Setup Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: - go-version: 'stable' + go-version: stable - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - - name: Create OCI artifacts destination directory - run: | - mkdir -p build - - - name: Download Build Image + - name: Download Build Image(s) uses: actions/download-artifact@v4 with: - name: current-build-image - path: build + pattern: current-build-image-* - - name: Download Run Image + - name: Download Run Image(s) uses: actions/download-artifact@v4 with: - name: current-run-image - path: build + pattern: current-run-image-* + + - name: Create OCI artifacts destination directory + run: | + echo '${{ needs.preparation.outputs.stacks }}' | jq -c '.[]' | while read -r stack; do + name=$(echo "$stack" | jq -r '.name') + output_dir=$(echo "$stack" | jq -r '.output_dir') + create_build_image=$(echo "$stack" | jq -r '.create_build_image') + mkdir -p $output_dir + mv "current-run-image-${name}/run.oci" "${output_dir}/run.oci" + if [ $create_build_image == 'true' ]; then + mv "current-build-image-${name}/build.oci" "${output_dir}/build.oci" + fi + done - name: Run Acceptance Tests - run: ./scripts/test.sh + run: ./scripts/test.sh --validate-stack-builds release: name: Release - runs-on: ubuntu-24.04 - needs: [poll_usns, create_stack, diff, test ] - if: ${{ always() && needs.diff.result == 'success' && needs.test.result == 'success' && (needs.diff.outputs.packages_changed == 'true' || needs.create_stack.outputs.stack_files_changed == 'true' || github.event.inputs.force == 'true' ) }} + runs-on: ubuntu-22.04 + needs: [create_stack, diff, stack_files_changed, test, preparation, packages_changed] + if: ${{ always() && needs.diff.result == 'success' && needs.test.result == 'success' && ( needs.packages_changed.outputs.packages_changed == 'true' || needs.stack_files_changed.outputs.stack_files_changed == 'true' || github.event.inputs.force == 'true' ) }} + outputs: + tag: ${{ steps.tag.outputs.tag }} steps: - name: Print Release Reasoning run: | printf "Diff Packages: %s\n" "${{ needs.diff.result }}" printf "Acceptance Tests: %s\n" "${{ needs.test.result }}" - printf "Packages Changed: %s\n" "${{ needs.diff.outputs.packages_changed }}" - printf "Packages Removed With Force: %s\n" "${{ needs.diff.outputs.packages_removed }}" - printf "Run If Stack Files Changed: %s\n" "${{ needs.create_stack.outputs.stack_files_changed }}" + printf "Packages Changed: %s\n" "${{ needs.packages_changed.outputs.packages_changed }}" + printf "Packages Removed With Force: %s\n" "${{ needs.diff.outputs.removed_with_force == 'true' }}" + printf "Stack Files Changed: %s\n" "${{ needs.stack_files_changed.outputs.stack_files_changed }}" printf "Force Release: %s\n" "${{ github.event.inputs.force }}" - name: Checkout With History - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 # gets full history - - name: Download current build image + - name: Download current build image(s) uses: actions/download-artifact@v4 with: - name: current-build-image + path: image-files + pattern: current-build-image-* + + - name: Download current run image(s) + uses: actions/download-artifact@v4 + with: + path: image-files + pattern: current-run-image-* + + - name: Display Build and Run Images + run: ls image-files - - name: Download current run image + - name: Download Build Receipt(s) uses: actions/download-artifact@v4 with: - name: current-run-image + path: receipt-files + pattern: current-build-receipt-* + merge-multiple: true - - name: Download Build Receipt + - name: Download Run Receipt(s) uses: actions/download-artifact@v4 with: - name: current-build-receipt + path: receipt-files + pattern: current-run-receipt-* + merge-multiple: true - - name: Download Run Receipt + - name: Display Receipts + run: ls receipt-files + + - name: Download Release Note File(s) uses: actions/download-artifact@v4 with: - name: current-run-receipt + path: release-notes + pattern: "*release-notes.md" + merge-multiple: true + + - name: Display Release Note Files + run: ls release-notes - - name: Download Build Diff Files + - name: Download hash code Files + if: ${{ needs.preparation.outputs.polling_type == 'hash' }} uses: actions/download-artifact@v4 with: - name: build-diff-files + path: hash-code-files + pattern: hash-code-* + merge-multiple: true + + - name: Display hash code Files + if: ${{ needs.preparation.outputs.polling_type == 'hash' }} + run: ls hash-code-files - - name: Download Run Diff Files + - name: Download USN Files + if: ${{ needs.preparation.outputs.polling_type == 'usn' }} uses: actions/download-artifact@v4 with: - name: run-diff-files + path: usn-files + pattern: "*${{ env.PATCHED_USNS_FILENAME }}" + merge-multiple: true + + - name: Display USN Files + if: ${{ needs.preparation.outputs.polling_type == 'usn' }} + run: ls usn-files - name: Increment Tag if: github.event.inputs.version == '' @@ -418,92 +1106,332 @@ jobs: fi echo "tag=${tag}" >> "$GITHUB_OUTPUT" - - name: Write USNs File - id: write_usns - if: ${{ needs.poll_usns.outputs.usns != '[]' }} + - name: Setup Release Assets + id: assets run: | - jq . <<< "${USNS}" > "${USNS_PATH}" - echo "usns=${USNS_PATH}" >> "$GITHUB_OUTPUT" - env: - USNS_PATH: "${{ env.PATCHED_USNS_FILENAME }}" - USNS: ${{ needs.poll_usns.outputs.usns }} - - name: Get Repository Name - id: repo_name - run: | - full=${{ github.repository }} - # Strip off the org and slash from repo name - # paketo-buildpacks/repo-name --> repo-name - repo=$(echo "${full}" | sed 's/^.*\///') - echo "github_repo_name=${repo}" >> "$GITHUB_OUTPUT" + stacks=$(echo '${{ needs.preparation.outputs.stacks }}' | jq -c '.[]') + archs=$(echo '${{ needs.preparation.outputs.architectures }}' | jq -c -r '.[]') + version="${{ steps.tag.outputs.tag }}" + os_codename="${{ needs.preparation.outputs.os_codename }}" + os_name="${{needs.preparation.outputs.os_name}}" + repo_type="${{ needs.preparation.outputs.repo_type }}" - # Strip off 'stack' suffix from repo name - # some-name-stack --> some-name - registry_repo="${repo//-stack/}" - echo "registry_repo_name=${registry_repo}" >> "$GITHUB_OUTPUT" + # Start with an empty array + assets=$(jq -n -c '[]') + # Adding the .oci artifacts + for stack in $stacks; do + stack_name=$(echo "$stack" | jq -r '.name') + create_build_image=$(echo "$stack" | jq -r '.create_build_image // false') + stack_type=$(echo "$stack" | jq -r '.stack_type // ""') + pattern_assets_prefix=$(echo "$stack" | jq -r '.pattern_assets_prefix // ""') - - name: Create Release Notes - id: notes - uses: paketo-buildpacks/github-config/actions/stack/release-notes@main - with: - build_image: "paketobuildpacks/build-${{ steps.repo_name.outputs.registry_repo_name }}:${{ steps.tag.outputs.tag }}" - run_image: "paketobuildpacks/run-${{ steps.repo_name.outputs.registry_repo_name }}:${{ steps.tag.outputs.tag }}" - build_packages_added: "/github/workspace/${{ env.BUILD_DIFF_ADDED_FILENAME }}" - build_packages_modified: "/github/workspace/${{ env.BUILD_DIFF_MODIFIED_FILENAME }}" - build_packages_removed_with_force: "/github/workspace/${{ env.BUILD_DIFF_REMOVED_FILENAME }}" - run_packages_added: "/github/workspace/${{ env.RUN_DIFF_ADDED_FILENAME }}" - run_packages_modified: "/github/workspace/${{ env.RUN_DIFF_MODIFIED_FILENAME }}" - run_packages_removed_with_force: "/github/workspace/${{ env.RUN_DIFF_REMOVED_FILENAME }}" - patched_usns: ${{ needs.poll_usns.outputs.usns }} - release_body_file: "/github/workspace/release-body.md" + # .oci artifacts for the run image + run_image=$(echo "$stack" | jq -r '.run_image') + if [[ "$pattern_assets_prefix" == "os_name-os_codename-build_image_run_image-stack_type-version-arch" ]]; then + asset_prefix="${os_name}-${os_codename}-${run_image}-${stack_type}-${version}" + elif [[ "$pattern_assets_prefix" == "os_name-os_codename-build_image_run_image-version-arch" ]]; then + asset_prefix="${os_name}-${os_codename}-${run_image}-${version}" + elif [[ "$pattern_assets_prefix" == "os_codename-stack_type-repo_type-version-arch-build_image_run_image" ]]; then + asset_prefix="${os_codename}-${stack_type}-${repo_type}-${version}-${run_image}" + fi - - name: Setup Release Assets - id: assets - run: | - assets="$(jq --null-input --compact-output \ - --arg tag "${{ steps.tag.outputs.tag }}" \ - --arg repo "${{ steps.repo_name.outputs.github_repo_name }}" \ - --arg build_receipt "${{ env.BUILD_RECEIPT_FILENAME }}" \ - --arg run_receipt "${{ env.RUN_RECEIPT_FILENAME }}" \ - '[ - { - "path": "build.oci", - "name": ($repo + "-" + $tag + "-" + "build.oci"), - "content_type": "application/gzip" - }, - { - "path": "run.oci", - "name": ($repo + "-" + $tag + "-" + "run.oci"), - "content_type": "application/gzip" - }, - { - "path": $build_receipt, - "name": ($repo + "-" + $tag + "-" + $build_receipt), - "content_type": "text/plain" - }, - { - "path": $run_receipt, - "name": ($repo + "-" + $tag + "-" + $run_receipt), - "content_type": "text/plain" - }]')" - - if [ -n "${{ steps.write_usns.outputs.usns }}" ]; then - assets="$(jq --compact-output \ - --arg tag "${{ steps.tag.outputs.tag }}" \ - --arg repo "${{ steps.repo_name.outputs.github_repo_name }}" \ - --arg usn_path "${{ steps.write_usns.outputs.usns }}" \ - --arg usn_name "${{ env.PATCHED_USNS_FILENAME }}" \ + assets="$(jq -c \ + --arg image_filepath "image-files" \ + --arg stack_name "${stack_name}" \ + --arg asset_prefix "${asset_prefix}" \ '. += [ - { - "path": $usn_path, - "name": ($repo + "-" + $tag + "-" + $usn_name), - "content_type": "text/plain" - } - ]' <<< "${assets}")" + { + "path": ($image_filepath + "/" + "current-run-image-" + $stack_name + "/run" + ".oci"), + "name": ($asset_prefix + ".oci"), + "content_type": "application/gzip" + } + ]' <<<"${assets}")" + + # .oci artifacts for the build image + if [[ "$create_build_image" == "true" ]]; then + build_image=$(echo "$stack" | jq -r '.build_image') + if [[ "$pattern_assets_prefix" == "os_name-os_codename-build_image_run_image-stack_type-version-arch" ]]; then + asset_prefix="${os_name}-${os_codename}-${build_image}-${stack_type}-${version}" + elif [[ "$pattern_assets_prefix" == "os_name-os_codename-build_image_run_image-version-arch" ]]; then + asset_prefix="${os_name}-${os_codename}-${build_image}-${version}" + elif [[ "$pattern_assets_prefix" == "os_codename-stack_type-repo_type-version-arch-build_image_run_image" ]]; then + asset_prefix="${os_codename}-${stack_type}-${repo_type}-${version}-${build_image}" + fi + + assets="$(jq -c \ + --arg image_filepath "image-files" \ + --arg stack_name "${stack_name}" \ + --arg asset_prefix "$asset_prefix" \ + '. += [ + { + "path": ($image_filepath + "/" + "current-build-image-" + $stack_name + "/build" + ".oci"), + "name": ($asset_prefix + ".oci"), + "content_type": "application/gzip" + } + ]' <<<"${assets}")" + fi + done + + # Adding hash code files to the assets + if [[ "${{ needs.preparation.outputs.polling_type }}" == "hash" ]]; then + for stack in $stacks; do + stack_name=$(echo "$stack" | jq -r '.name') + create_build_image=$(echo "$stack" | jq -r '.create_build_image // false') + stack_type=$(echo "$stack" | jq -r '.stack_type // ""') + pattern_assets_prefix=$(echo "$stack" | jq -r '.pattern_assets_prefix // ""') + + for arch in $archs; do + arch_name=$(echo "$arch" | jq -r '.name') + arch_prefix="-${arch_name}-" + if [[ $arch_name == "amd64" ]]; then + arch_prefix="-" + fi + + # hash code for the run image + run_image=$(echo "$stack" | jq -r '.run_image') + if [[ "$pattern_assets_prefix" == "os_codename-stack_type-repo_type-version-arch-build_image_run_image" ]]; then + asset_prefix="${os_codename}-${stack_type}-${repo_type}-${version}" + fi + + run_image=$(echo "$stack" | jq -r '.run_image') + assets="$(jq -c \ + --arg hash_code_filepath "hash-code-files" \ + --arg stack_name "${stack_name}" \ + --arg run_image "${run_image}" \ + --arg asset_prefix "$asset_prefix" \ + --arg arch "${arch_name}" \ + --arg arch_prefix "${arch_prefix}" \ + '. += [ + { + "path": ($hash_code_filepath + "/" + "hash-code-current-run-image-"+ $arch + "-" + $stack_name ), + "name": ($asset_prefix + $arch_prefix + $run_image + ".oci.sha256"), + "content_type": "text/plain" + } + ]' <<<"${assets}")" + + # hash code for the build image + if [[ $create_build_image == true ]]; then + + build_image=$(echo "$stack" | jq -r '.build_image') + if [[ "$pattern_assets_prefix" == "os_codename-stack_type-repo_type-version-arch-build_image_run_image" ]]; then + asset_prefix="${os_codename}-${stack_type}-${repo_type}-${version}" + fi + + build_image=$(echo "$stack" | jq -r '.build_image') + assets="$(jq -c \ + --arg hash_code_filepath "hash-code-files" \ + --arg stack_name "${stack_name}" \ + --arg build_image "${build_image}" \ + --arg asset_prefix "$asset_prefix" \ + --arg arch "${arch_name}" \ + --arg arch_prefix "${arch_prefix}" \ + '. += [ + { + "path": ($hash_code_filepath + "/" + "hash-code-current-build-image-"+ $arch + "-" + $stack_name ), + "name": ($asset_prefix + "-" + $arch_prefix + $build_image + ".oci.sha256"), + "content_type": "text/plain" + } + ]' <<<"${assets}")" + fi + done + done + fi + + # SBOM/receipt.cyclonedx.json receipt assets + for stack in $stacks; do + stack_name=$(echo "$stack" | jq -r '.name') + create_build_image=$(echo "$stack" | jq -r '.create_build_image // false') + stack_type=$(echo "$stack" | jq -r '.stack_type // ""') + pattern_assets_prefix=$(echo "$stack" | jq -r '.pattern_assets_prefix // ""') + + for arch in $archs; do + arch_name=$(echo "$arch" | jq -r '.name') + arch_prefix=$arch_name + if [[ $arch_name == "amd64" ]]; then + arch_prefix="" + fi + + # SBOM for the run image + run_image=$(echo "$stack" | jq -r '.run_image') + if [[ "$pattern_assets_prefix" == "os_name-os_codename-build_image_run_image-stack_type-version-arch" ]]; then + sbom_asset_name_run_image=$( echo '[ "'"$os_name"'", "'"$os_codename"'", "'"$run_image"'", "'"$stack_type"'", + "'"$version"'", "'"$arch_prefix"'", "receipt.cyclonedx.json" ]' | jq -r 'map(select(length > 0)) | join("-")' ) + sbom_asset_name_build_image=$( echo '[ "'"$os_name"'", "'"$os_codename"'", "'"$build_image"'", "'"$stack_type"'", "'"$version"'", + "'"$arch_prefix"'", "receipt.cyclonedx.json" ]' | jq -r 'map(select(length > 0)) | join("-")') + elif [[ "$pattern_assets_prefix" == "os_name-os_codename-build_image_run_image-version-arch" ]]; then + sbom_asset_name_run_image=$( echo '[ "'"$os_name"'", "'"$os_codename"'", "'"$run_image"'", "'"$stack_type"'", "'"$version"'", + "'"$arch_prefix"'", "receipt.cyclonedx.json" ]' | jq -r 'map(select(length > 0)) | join("-")' ) + sbom_asset_name_build_image=$(echo '[ "'"$os_name"'", "'"$os_codename"'", "'"$build_image"'", "'"$version"'", "'"$arch_prefix"'", + "receipt.cyclonedx.json" ]' | jq -r 'map(select(length > 0)) | join("-")') + elif [[ "$pattern_assets_prefix" == "os_codename-stack_type-repo_type-version-arch-build_image_run_image" ]]; then + sbom_asset_name_run_image=$(echo '[ "'"$os_codename"'", "'"$stack_type"'", "'"$repo_type"'", "'"$version"'", "'"$arch_prefix"'", + "'"$run_image"'", "receipt.cyclonedx.json"]' | jq -r 'map(select(length > 0)) | join("-")') + sbom_asset_name_build_image=$(echo '[ "'"$os_codename"'", "'"$stack_type"'", "'"$repo_type"'", "'"$version"'", "'"$arch_prefix"'", + "'"$build_image"'", "receipt.cyclonedx.json" ]' | jq -r 'map(select(length > 0)) | join("-")') + fi + + assets="$(jq -c \ + --arg sbom_filepath "receipt-files" \ + --arg stack_name "${stack_name}" \ + --arg arch "${arch_name}" \ + --arg sbom_asset_name "${sbom_asset_name_run_image}" \ + '. += [ + { + "path": ($sbom_filepath + "/" + $arch + "-" + "current-run-receipt"+ "-" + $stack_name), + "name": $sbom_asset_name, + "content_type": "text/plain" + } + ]' <<<"${assets}")" + + # SBOM for the build image + if [[ "$create_build_image" == "true" ]]; then + assets="$(jq -c \ + --arg sbom_filepath "receipt-files" \ + --arg stack_name "${stack_name}" \ + --arg arch "${arch_name}" \ + --arg sbom_asset_name "${sbom_asset_name_build_image}" \ + '. += [ + { + "path": ($sbom_filepath + "/" + $arch + "-" + "current-build-receipt"+ "-" + $stack_name), + "name": $sbom_asset_name, + "content_type": "text/plain" + } + ]' <<<"${assets}")" + fi + done + done + + ## Add the usn files to the assets + if [ "${{ needs.preparation.outputs.polling_type }}" = "usn" ]; then + for stack in $stacks; do + stack_name=$(echo "$stack" | jq -r '.name') + os_codename="${{ needs.preparation.outputs.os_codename }}" + os_name="${{needs.preparation.outputs.os_name}}" + stack_type=$(echo "$stack" | jq -r '.stack_type') + pattern_assets_prefix=$(echo "$stack" | jq -r '.pattern_assets_prefix // ""') + version="${{ steps.tag.outputs.tag }}" + + asset_prefix="" + if [[ "$pattern_assets_prefix" == "os_name-os_codename-build_image_run_image-stack_type-version-arch" ]]; then + asset_prefix="${os_name}-${os_codename}-${stack_type}-${version}" + elif [[ "$pattern_assets_prefix" == "os_name-os_codename-build_image_run_image-version-arch" ]]; then + asset_prefix="${os_name}-${os_codename}-${version}" + elif [[ "$pattern_assets_prefix" == "os_codename-stack_type-repo_type-version-arch-build_image_run_image" ]]; then + asset_prefix="${os_codename}-${stack_type}-${repo_type}-${version}" + fi + + for arch in $archs; do + arch_name=$(echo "$arch" | jq -r '.name') + arch_prefix="${arch_name}" + if [[ $arch_name == "amd64" ]]; then + arch_prefix="" + fi + + usn_asset_name=$( + echo '[ + "'"$asset_prefix"'", + "'"$arch_prefix"'", + "${{ env.PATCHED_USNS_FILENAME }}" + ]' | jq -r 'map(select(length > 0)) | join("-")' + ) + + assets="$(jq -c \ + --arg asset_prefix "${asset_prefix}" \ + --arg stack_name "${stack_name}" \ + --arg arch "${arch_name}" \ + --arg usn_asset_name "${usn_asset_name}" \ + --arg patched_usns_suffix "${{ env.PATCHED_USNS_FILENAME }}" \ + '. += [ + { + "path": ("usn-files" + "/" + $arch + "-" + $stack_name + "-" + $patched_usns_suffix), + "name": $usn_asset_name, + "content_type": "text/plain" + } + ]' <<< "${assets}")" + done + done + fi + + # Merge release notes per architecture and add them to the assets + release_notes_dir="release-notes" + for arch in $archs; do + arch_name=$(echo "$arch" | jq -r '.name') + arch_prefix="-${arch_name}-" + if [[ $arch_name == "amd64" ]]; then + arch_prefix="-" + fi + + # Merge the release notes per architecture + for stack in $stacks; do + stack_name=$(echo "$stack" | jq -r '.name') + filename="${arch_name}-${stack_name}-release-notes.md" + cat "${release_notes_dir}/${filename}" >>"${release_notes_dir}/${arch_name}-release-notes" + done + + # add release notes of the arch on the assets + assets="$(jq -c \ + --arg release_notes_dir "${release_notes_dir}" \ + --arg tag "${version}" \ + --arg repo "${{ needs.preparation.outputs.github_repo_name }}" \ + --arg arch "${arch_name}" \ + --arg arch_prefix "${arch_prefix}" \ + '. += [ + { + "path": ($release_notes_dir + "/" + $arch + "-" + "release-notes"), + "name": ($repo + "-" + $tag + $arch_prefix + "release-notes.md"), + "content_type": "text/plain" + } + ]' <<<"${assets}")" + done + + echo "assets=${assets}" >> "$GITHUB_OUTPUT" + + - name: Generate Release Notes Description + id: notes + run: | + + # If there is only one architecture + archs_length=$(echo '${{ needs.preparation.outputs.architectures }}' | jq 'length') + if [ $archs_length -eq 1 ]; then + arch_name=$(echo '${{ needs.preparation.outputs.architectures }}' | jq -r '.[0].name') + cat "release-notes/${arch_name}-release-notes" > release_notes.md + else + stacks=$(echo '${{ needs.preparation.outputs.stacks }}' | jq -c -r '.[]') + os_codename="${{ needs.preparation.outputs.os_codename }}" + os_name="${{ needs.preparation.outputs.os_name }}" + + # Construct the release notes document + echo "## Images" > release_notes.md + echo "" >> release_notes.md + + for stack in $stacks; do + run_image=$(echo "$stack" | jq -r '.run_image') + build_image=$(echo "$stack" | jq -r '.build_image') + stack_type=$(echo "$stack" | jq -r '.stack_type // ""') + pattern_image_registry_name=$(echo "$stack" | jq -r '.pattern_image_registry_name') + create_build_image=$(echo "$stack" | jq -r '.create_build_image // false') + + if [ $pattern_image_registry_name == "os_name-os_codename-build_image_run_image-stack_type" ]; then + build_image_registry_name="${{ needs.preparation.outputs.repo_owner }}/${os_name}-${os_codename}-${build_image}-${stack_type}:${{ steps.tag.outputs.tag }}" + run_image_registry_name="${{ needs.preparation.outputs.repo_owner }}/${os_name}-${os_codename}-${run_image}-${stack_type}:${{ steps.tag.outputs.tag }}" + elif [ $pattern_image_registry_name == "os_name-os_codename-build_image_run_image" ]; then + build_image_registry_name="${{ needs.preparation.outputs.repo_owner }}/${os_name}-${os_codename}-${build_image}:${{ steps.tag.outputs.tag }}" + run_image_registry_name="${{ needs.preparation.outputs.repo_owner }}/${os_name}-${os_codename}-${run_image}:${{ steps.tag.outputs.tag }}" + elif [ $pattern_image_registry_name == "build_image_run_image-os_codename-stack_type" ]; then + build_image_registry_name="${{ needs.preparation.outputs.repo_owner }}/${build_image}-${os_codename}-${stack_type}:${{ steps.tag.outputs.tag }}" + run_image_registry_name="${{ needs.preparation.outputs.repo_owner }}/${run_image}-${os_codename}-${stack_type}:${{ steps.tag.outputs.tag }}" + fi + + if [[ "$create_build_image" == "true" ]]; then + echo "Build: \`${build_image_registry_name}\`" >> release_notes.md + fi + echo "Run: \`${run_image_registry_name}\`" >> release_notes.md + done fi - printf "assets=%s\n" "${assets}" >> "$GITHUB_OUTPUT" + echo "release_body=release_notes.md" >> "$GITHUB_OUTPUT" - name: Create Release uses: paketo-buildpacks/github-config/actions/release/create@main @@ -513,15 +1441,15 @@ jobs: tag_name: v${{ steps.tag.outputs.tag }} target_commitish: ${{ github.sha }} name: v${{ steps.tag.outputs.tag }} - body_filepath: "/github/workspace/release-body.md" + body_filepath: ${{ steps.notes.outputs.release_body }} draft: false assets: ${{ steps.assets.outputs.assets }} failure: name: Alert on Failure - runs-on: ubuntu-24.04 - needs: [poll_usns, create_stack, diff, test, release, ] - if: ${{ always() && needs.poll_usns.result == 'failure' || needs.create_stack.result == 'failure' || needs.diff.result == 'failure' || needs.test.result == 'failure' || needs.release.result == 'failure' }} + runs-on: ubuntu-22.04 + needs: [ preparation, poll_usns, poll_images, create_stack, diff, test, release ] + if: ${{ always() && needs.preparation.result == 'failure' || needs.poll_images.result == 'failure' || needs.poll_usns.result == 'failure' || needs.create_stack.result == 'failure' || needs.diff.result == 'failure' || needs.test.result == 'failure' || needs.release.result == 'failure' }} steps: - name: File Failure Alert Issue uses: paketo-buildpacks/github-config/actions/issue/file@main @@ -533,6 +1461,6 @@ jobs: issue_title: "Failure: Create Release workflow" issue_body: | Create Release workflow [failed](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}). - Please take a look to ensure CVE patches can be released. (cc @paketo-buildpacks/stacks-maintainers). + Unable to update images. comment_body: | Another failure occurred: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} diff --git a/stack/.github/workflows/push-image.yml b/stack/.github/workflows/push-image.yml index 243f3050..203e658f 100644 --- a/stack/.github/workflows/push-image.yml +++ b/stack/.github/workflows/push-image.yml @@ -4,54 +4,187 @@ on: release: types: - published + + workflow_dispatch: + inputs: + version: + description: 'Version of the stack to push' + required: false + env: - REGISTRIES_FILENAME: "registries.json" + REGISTRIES_FILEPATH: "registries.json" + STACKS_FILEPATH: "images.json" + GCR_REGISTRY: "gcr.io" + GCR_USERNAME: "_json_key" jobs: preparation: name: Preparation - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} - DOCKERHUB_ORG: ${{ steps.set-dockerhub-org-namespace.outputs.DOCKERHUB_ORG}} - push_to_gcr: ${{ steps.parse_configs.outputs.push_to_gcr}} - push_to_dockerhub: ${{ steps.parse_configs.outputs.push_to_dockerhub}} + DOCKERHUB_ORG: ${{ steps.set-dockerhub-org-namespace.outputs.DOCKERHUB_ORG }} + push_to_gcr: ${{ steps.parse_configs.outputs.push_to_gcr }} + push_to_dockerhub: ${{ steps.parse_configs.outputs.push_to_dockerhub }} + tag: ${{ steps.event.outputs.tag }} + os_codename: ${{ steps.repo.outputs.os_codename }} + os_name: ${{ steps.repo.outputs.os_name }} + repo_type: ${{ steps.repo.outputs.repo_type }} + stacks: ${{ steps.get-stacks.outputs.stacks }} steps: - name: Checkout uses: actions/checkout@v4 + - name: Parse Event + id: event + run: | + set -euo pipefail + shopt -s inherit_errexit + + # If the workflow has been triggered from dispatch event + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "tag=${{ github.event.inputs.version }}" >> "$GITHUB_OUTPUT" + else #The workflow has been triggered from publish event + echo "tag=$(jq -r '.release.tag_name' "${GITHUB_EVENT_PATH}" | sed s/^v//)" >> "$GITHUB_OUTPUT" + fi + - name: Set matrix id: set-matrix run: | - release_version="$(jq -r '.release.tag_name' "${GITHUB_EVENT_PATH}" | sed s/^v//)" + stacks=$(jq -c '.images | .[]' ${STACKS_FILEPATH}) + version="${{ steps.event.outputs.tag }}" + os_name=$( jq -r '.os_name' ${STACKS_FILEPATH} ) + os_codename=$( jq -r '.os_codename' ${STACKS_FILEPATH} ) + repo_type=$( jq -r '.repo_type' ${STACKS_FILEPATH} ) + + # Start with an empty array + asset_names=$(jq -n -c '[]') + + for stack in $stacks; do + stack_name=$(echo "$stack" | jq -r '.name') + stack_type=$(echo "$stack" | jq -r '.stack_type // ""') + + # .oci artifacts for the run image + run_image=$(echo "$stack" | jq -r '.run_image') + build_image=$(echo "$stack" | jq -r '.build_image') + + if [ $pattern_assets_prefix == "os_codename-stack_type-repo_type" ]; then + asset_prefix="${os_codename}-${stack_type}-${repo_type}" + elif [ $pattern_assets_prefix == "os_codename-repo_type" ]; then + asset_prefix="${os_codename}-${repo_type}" + fi + + pattern_assets_prefix=$(echo "$stack" | jq -r '.pattern_assets_prefix // ""') + if [[ "$pattern_assets_prefix" == "os_name-os_codename-build_image_run_image-stack_type-version-arch" ]]; then + run_image_asset_prefix="${os_name}-${os_codename}-${run_image}-${stack_type}-${version}" + build_image_asset_prefix="${os_name}-${os_codename}-${build_image}-${stack_type}-${version}" + elif [[ "$pattern_assets_prefix" == "os_name-os_codename-build_image_run_image-version-arch" ]]; then + run_image_asset_prefix="${os_name}-${os_codename}-${run_image}-${version}" + build_image_asset_prefix="${os_name}-${os_codename}-${build_image}-${version}" + elif [[ "$pattern_assets_prefix" == "os_codename-stack_type-repo_type-version-arch-build_image_run_image" ]]; then + run_image_asset_prefix="${os_codename}-${stack_type}-${repo_type}-${version}-${run_image}" + build_image_asset_prefix="${os_codename}-${stack_type}-${repo_type}-${version}-${build_image}" + fi + + pattern_image_registry_name=$(echo "$stack" | jq -r '.pattern_image_registry_name // ""') + if [ $pattern_image_registry_name == "os_name-os_codename-build_image_run_image-stack_type" ]; then + build_image_registry_name="${os_name}-${os_codename}-${build_image}-${stack_type}" + run_image_registry_name="${os_name}-${os_codename}-${run_image}-${stack_type}" + elif [ $pattern_image_registry_name == "os_name-os_codename-build_image_run_image" ]; then + build_image_registry_name="${os_name}-${os_codename}-${build_image}" + run_image_registry_name="${os_name}-${os_codename}-${run_image}" + elif [ $pattern_image_registry_name == "build_image_run_image-os_codename-stack_type" ]; then + build_image_registry_name="${build_image}-${os_codename}-${stack_type}" + run_image_registry_name="${run_image}-${os_codename}-${stack_type}" + fi + + asset_names="$(jq -c \ + --arg image_filepath "image-files" \ + --arg stack_name "${stack_name}" \ + --arg run_image_asset_prefix "${run_image_asset_prefix}" \ + --arg run_image_registry_name "${run_image_registry_name}" \ + '. += [ + { + "name": $run_image_asset_prefix, + "path": ($image_filepath + "/" + "current-run-image-" + $stack_name + "/run" + ".oci"), + "registry_name": $run_image_registry_name + } + ]' <<<"${asset_names}")" + + # .oci artifacts for the build image + create_build_image=$(echo "$stack" | jq -r '.create_build_image // false') + if [[ $create_build_image == true ]]; then + + asset_names="$(jq -c \ + --arg image_filepath "image-files" \ + --arg stack_name "${stack_name}" \ + --arg build_image_asset_prefix "${build_image_asset_prefix}" \ + --arg build_image_registry_name "${build_image_registry_name}" \ + '. += [ + { + "name": $build_image_asset_prefix, + "path": ($image_filepath + "/" + "current-build-image-" + $stack_name + "/build" + ".oci"), + "registry_name": $build_image_registry_name + + } + ]' <<<"${asset_names}")" + fi + done + + release_version="${{ steps.event.outputs.tag }}" + release_info=$(curl -s "https://api.github.com/repos/${{ github.repository }}/releases/tags/v${release_version}") + oci_release_assets=$(echo "$release_info" | jq -c \ + '[ .assets[] + | select(.name | endswith(".oci")) + | {name: (.name | split(".oci") | .[0]), oci_asset_url: .url} + ]') - # Strip off the org and slash from repo name - # paketo-buildpacks/repo-name --> repo-name - repo_name=$(echo "${{ github.repository }}" | sed 's/^.*\///') + oci_release_assets_length=$(echo "$oci_release_assets" | jq 'length') + asset_names_length=$(echo "$asset_names" | jq 'length') - asset_prefix="${repo_name}-${release_version}-" - oci_images=$(jq -c --arg asset_prefix "$asset_prefix" '[.release.assets[].name | select(endswith(".oci")) | split(".oci") | .[0] | split($asset_prefix) | .[1]]' "${GITHUB_EVENT_PATH}") - printf "matrix=%s\n" "${oci_images}" >> "$GITHUB_OUTPUT" + if [ "$oci_release_assets_length" != "$asset_names_length" ]; then + echo "Produced assets do not match the release assets." + exit 1 + fi + + matrix=$(jq -n -c --argjson a "$oci_release_assets" --argjson b "$asset_names" ' + $a | map( + . as $oci_item + | ($b[] | select(.name == $oci_item.name)) as $asset_item + | $oci_item + $asset_item + ) + ') + + matrix_length=$(echo "$matrix" | jq 'length') + if [ "$matrix_length" != "$asset_names_length" ]; then + echo "Produced assets do not match the release assets." + exit 1 + fi + + echo "matrix=${matrix}" + echo "matrix=${matrix}" >> "$GITHUB_OUTPUT" - name: Set DOCKERHUB_ORG namespace id: set-dockerhub-org-namespace - run: echo "DOCKERHUB_ORG=${GITHUB_REPOSITORY_OWNER//-/}" >> "$GITHUB_OUTPUT" + run: | + echo "DOCKERHUB_ORG=${GITHUB_REPOSITORY_OWNER//-/}" >> "$GITHUB_OUTPUT" - name: Parse Configs id: parse_configs - run: | - registries_filename="${{ env.REGISTRIES_FILENAME }}" + registries_filepath="${{ env.REGISTRIES_FILEPATH }}" push_to_dockerhub=true push_to_gcr=false - if [[ -f $registries_filename ]]; then - if jq 'has("dockerhub")' $registries_filename > /dev/null; then - push_to_dockerhub=$(jq '.dockerhub' $registries_filename) + if [[ -f $registries_filepath ]]; then + + if jq 'has("dockerhub")' $registries_filepath > /dev/null; then + push_to_dockerhub=$(jq '.dockerhub' $registries_filepath) fi - if jq 'has("GCR")' $registries_filename > /dev/null; then - push_to_gcr=$(jq '.GCR' $registries_filename) + + if jq 'has("GCR")' $registries_filepath > /dev/null; then + push_to_gcr=$(jq '.GCR' $registries_filepath) fi fi @@ -68,69 +201,68 @@ jobs: oci_image: ${{ fromJSON(needs.preparation.outputs.matrix) }} steps: - - name: Parse Event - id: event - run: | - echo "tag=$(jq -r '.release.tag_name' "${GITHUB_EVENT_PATH}" | sed s/^v//)" >> "$GITHUB_OUTPUT" - echo "${{ matrix.oci_image }}_download_url=$(jq -r '.release.assets[] | select(.name | endswith("${{ matrix.oci_image }}.oci")) | .url' "${GITHUB_EVENT_PATH}")" >> "$GITHUB_OUTPUT" - - name: Checkout uses: actions/checkout@v4 - - name: Download ${{ matrix.oci_image }} Image + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up buildx + uses: docker/setup-buildx-action@v3 + + - name: Download ${{ matrix.oci_image.name }} Image uses: paketo-buildpacks/github-config/actions/release/download-asset@main with: - url: ${{ steps.event.outputs[format('{0}_download_url', matrix.oci_image)] }} - output: "/github/workspace/${{ matrix.oci_image }}.oci" + url: "${{ matrix.oci_image.oci_asset_url }}" + output: "./saved_image.oci" token: ${{ secrets.PAKETO_BOT_GITHUB_TOKEN }} - - name: Get Registry Repo Name - id: registry-repo - run: | - # Strip off the Github org prefix and 'stack' suffix from repo name - # paketo-buildpacks/some-name-stack --> some-name - echo "name=$(echo "${{ github.repository }}" | sed 's/^.*\///' | sed 's/\-stack$//')" >> "$GITHUB_OUTPUT" + - name: Docker login docker.io + uses: docker/login-action@v3 + if: ${{ needs.preparation.outputs.push_to_dockerhub == 'true' }} + with: + username: ${{ secrets.PAKETO_BUILDPACKS_DOCKERHUB_USERNAME }} + password: ${{ secrets.PAKETO_BUILDPACKS_DOCKERHUB_PASSWORD }} + registry: docker.io + + - name: Docker login gcr.io + uses: docker/login-action@v3 + if: ${{ needs.preparation.outputs.push_to_gcr == 'true' }} + with: + username: ${{ env.GCR_USERNAME }} + password: ${{ secrets.GCR_PUSH_BOT_JSON_KEY }} + registry: ${{ env.GCR_REGISTRY }} - - name: Push ${{ matrix.oci_image }} Image to DockerHub + - name: Push ${{ matrix.oci_image.name }} Image to registries id: push env: - DOCKERHUB_USERNAME: ${{ secrets.PAKETO_BUILDPACKS_DOCKERHUB_USERNAME }} - DOCKERHUB_PASSWORD: ${{ secrets.PAKETO_BUILDPACKS_DOCKERHUB_PASSWORD }} DOCKERHUB_ORG: "${{ needs.preparation.outputs.DOCKERHUB_ORG }}" - GCR_USERNAME: _json_key - GCR_PASSWORD: ${{ secrets.GCR_PUSH_BOT_JSON_KEY }} GCR_PROJECT: "${{ github.repository_owner }}" run: | + # Ensure other scripts can access the .bin directory to install their own + # tools after we install them as whatever user we are. + mkdir -p ./.bin/ + chmod 777 ./.bin/ - if [ "${{ needs.preparation.outputs.push_to_dockerhub }}" == "true" ]; then - echo "${DOCKERHUB_PASSWORD}" | sudo skopeo login --username "${DOCKERHUB_USERNAME}" --password-stdin index.docker.io - sudo skopeo copy "oci-archive:${GITHUB_WORKSPACE}/${{ matrix.oci_image }}.oci" "docker://${DOCKERHUB_ORG}/${{ matrix.oci_image }}-${{ steps.registry-repo.outputs.name }}:${{ steps.event.outputs.tag }}" - sudo skopeo copy "oci-archive:${GITHUB_WORKSPACE}/${{ matrix.oci_image }}.oci" "docker://${DOCKERHUB_ORG}/${{ matrix.oci_image }}-${{ steps.registry-repo.outputs.name }}:latest" - fi + image_name_registry=${{ matrix.oci_image.registry_name }} - if [ "${{ needs.preparation.outputs.push_to_gcr }}" == "true" ]; then - echo "${GCR_PASSWORD}" | sudo skopeo login --username "${GCR_USERNAME}" --password-stdin gcr.io - sudo skopeo copy "oci-archive:${GITHUB_WORKSPACE}/${{ matrix.oci_image }}.oci" "docker://gcr.io/${GCR_PROJECT}/${{ matrix.oci_image }}-${{ steps.registry-repo.outputs.name }}:${{ steps.event.outputs.tag }}" - sudo skopeo copy "oci-archive:${GITHUB_WORKSPACE}/${{ matrix.oci_image }}.oci" "docker://gcr.io/${GCR_PROJECT}/${{ matrix.oci_image }}-${{ steps.registry-repo.outputs.name }}:latest" - fi - # If the repository name contains 'bionic', let's push it to legacy image locations as well: - # paketobuildpacks/{build/run}:{version}-{variant} - # paketobuildpacks/{build/run}:{version}-{variant}-cnb - # paketobuildpacks/{build/run}:{variant}-cnb - # paketobuildpacks/{build/run}:{variant} - registry_repo="${{ steps.registry-repo.outputs.name }}" - if [[ ${registry_repo} == "bionic"-* ]]; - then - # Strip the final part from a repo name after the `-` - # bionic-tiny --> tiny - variant="${registry_repo#bionic-}" - - sudo skopeo copy "oci-archive:${GITHUB_WORKSPACE}/${{ matrix.oci_image }}.oci" "docker://${DOCKERHUB_ORG}/${{ matrix.oci_image }}:${{ steps.event.outputs.tag }}-${variant}" - sudo skopeo copy "oci-archive:${GITHUB_WORKSPACE}/${{ matrix.oci_image }}.oci" "docker://${DOCKERHUB_ORG}/${{ matrix.oci_image }}:${{ steps.event.outputs.tag }}-${variant}-cnb" - sudo skopeo copy "oci-archive:${GITHUB_WORKSPACE}/${{ matrix.oci_image }}.oci" "docker://${DOCKERHUB_ORG}/${{ matrix.oci_image }}:${variant}-cnb" - sudo skopeo copy "oci-archive:${GITHUB_WORKSPACE}/${{ matrix.oci_image }}.oci" "docker://${DOCKERHUB_ORG}/${{ matrix.oci_image }}:${variant}" - - sudo skopeo copy "docker://${DOCKERHUB_ORG}/${{ matrix.oci_image }}:${variant}-cnb" "docker://gcr.io/${GCR_PROJECT}/${{ matrix.oci_image }}:${variant}-cnb" + ./scripts/publish.sh \ + --image-ref "docker.io/${DOCKERHUB_ORG}/${image_name_registry}:${{ needs.preparation.outputs.tag }}" \ + --image-ref "docker.io/${DOCKERHUB_ORG}/${image_name_registry}:latest" \ + --image-archive "./saved_image.oci" + + if [ "${{ needs.preparation.outputs.push_to_gcr }}" = "true" ]; then + + platforms=$(docker manifest inspect "docker.io/${DOCKERHUB_ORG}/${image_name_registry}:${{ needs.preparation.outputs.tag }}" | + jq -r '[.manifests[].platform] | [.[] | .os + "/" + .architecture] | join(",")') + + echo "FROM docker.io/${DOCKERHUB_ORG}/${image_name_registry}:${{ needs.preparation.outputs.tag }}" | \ + docker buildx build -f - . \ + --tag "${{ env.GCR_REGISTRY }}/${GCR_PROJECT}/${image_name_registry}:${{ needs.preparation.outputs.tag }}" \ + --tag "${{ env.GCR_REGISTRY }}/${GCR_PROJECT}/${image_name_registry}:latest" \ + --platform "$platforms" \ + --provenance=false \ + --push fi failure: diff --git a/stack/.github/workflows/test-pull-request.yml b/stack/.github/workflows/test-pull-request.yml index 2e7c070a..23ae4430 100644 --- a/stack/.github/workflows/test-pull-request.yml +++ b/stack/.github/workflows/test-pull-request.yml @@ -18,10 +18,14 @@ jobs: - name: Checkout uses: actions/checkout@v3 + # https://github.com/docker/setup-qemu-action + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Create stack id: create-stack run: | - scripts/create.sh + ./scripts/create.sh - name: Run Acceptance Tests run: ./scripts/test.sh From 59b19858113bbb532265a0bd3f31fac8482b6161 Mon Sep 17 00:00:00 2001 From: Costas Papastathis Date: Wed, 28 May 2025 17:45:46 +0200 Subject: [PATCH 3/3] fix:lint errors --- stack/.github/workflows/create-release.yml | 32 +++++++++---------- .../workflows/update-github-config.yml | 2 +- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/stack/.github/workflows/create-release.yml b/stack/.github/workflows/create-release.yml index c022f70f..84e9a30d 100644 --- a/stack/.github/workflows/create-release.yml +++ b/stack/.github/workflows/create-release.yml @@ -2,7 +2,7 @@ name: Create Release on: schedule: - - cron: '27 2,14 * * *' # daily at 02:27 and 14:27 UTC + - cron: '27 2,14 * * *' # daily at 02:27 and 14:27 UTC push: branches: - main @@ -463,7 +463,7 @@ jobs: - name: Get current run image hash code of ${{ matrix.stacks.name }} stack with arch ${{ matrix.arch.name }} run: | skopeo inspect --format "{{.Digest}}" ${{ matrix.stacks.base_run_container_image }} > ./hash-code-current-run-image-${{ matrix.arch.name }}-${{ matrix.stacks.name }} - + - name: Create empty image hash codes run: | if [ ! -f "./hash-code-current-build-image-${{ matrix.arch.name }}-${{ matrix.stacks.name }}" ]; then @@ -478,7 +478,7 @@ jobs: with: name: hash-code-current-run-image-${{ matrix.arch.name }}-${{ matrix.stacks.name }} path: hash-code-current-run-image-${{ matrix.arch.name }}-${{ matrix.stacks.name }} - if-no-files-found: error + if-no-files-found: error - name: Upload build image hash code if: ${{ matrix.stacks.create_build_image == true }} @@ -508,11 +508,11 @@ jobs: name: Determine If Stack Files Changed runs-on: ubuntu-22.04 needs: [ preparation, poll_usns, poll_images ] - if: ${{ + if: ${{ !failure() && !cancelled() && !( - (needs.poll_images.outputs.images_need_update == null && - needs.poll_usns.outputs.usns == null ) && + (needs.poll_images.outputs.images_need_update == null && + needs.poll_usns.outputs.usns == null ) && github.event_name == 'schedule' ) }} outputs: @@ -541,7 +541,7 @@ jobs: usns_or_sha_changed: name: USNs or SHAs have changed runs-on: ubuntu-22.04 - outputs: + outputs: changed: ${{ steps.usns_or_sha_changed.outputs.changed }} needs: [ preparation, poll_usns, poll_images ] if: ${{ !failure() && !cancelled() }} @@ -626,14 +626,14 @@ jobs: with: name: current-build-receipt-${{ matrix.stacks.name }} path: "*current-build-receipt-${{ matrix.stacks.name }}" - if-no-files-found: error + if-no-files-found: error - name: Upload Run receipt uses: actions/upload-artifact@v4 with: name: current-run-receipt-${{ matrix.stacks.name }} path: "*current-run-receipt-${{ matrix.stacks.name }}" - if-no-files-found: error + if-no-files-found: error diff: name: Diff Packages @@ -823,7 +823,7 @@ jobs: with: name: diff-${{ matrix.arch.name }}-${{ matrix.stacks.name }} path: diff-${{ matrix.arch.name }}-${{ matrix.stacks.name }} - if-no-files-found: error + if-no-files-found: error - name: Download USN File(s) if: ${{ needs.preparation.outputs.polling_type == 'usn' }} @@ -886,7 +886,7 @@ jobs: fi if [ "${{ matrix.stacks.create_build_image }}" != "true" ]; then - build_image="" + build_image="" fi echo "build_image=${build_image}" >> "$GITHUB_OUTPUT" @@ -929,7 +929,7 @@ jobs: - name: Download diff-${{ matrix.arch.name }}-${{ matrix.stacks.name }} uses: actions/download-artifact@v4 with: - name: diff-${{ matrix.arch.name }}-${{ matrix.stacks.name }} + name: diff-${{ matrix.arch.name }}-${{ matrix.stacks.name }} - name: Compare With Previous Release id: compare @@ -955,7 +955,7 @@ jobs: if [ "${build_added:-0}" -eq 0 ] && [ "${build_modified:-0}" -eq 0 ] && [ "${run_added}" -eq 0 ] && [ "${run_modified}" -eq 0 ]; then echo "No packages changed." - # We ommit setting "packages_changed" variable to false, + # We ommit setting "packages_changed" variable to false, # as there is an edge case scenario overriding any true value due to parallelization else echo "Packages changed." @@ -1230,7 +1230,7 @@ jobs: '. += [ { "path": ($hash_code_filepath + "/" + "hash-code-current-build-image-"+ $arch + "-" + $stack_name ), - "name": ($asset_prefix + "-" + $arch_prefix + $build_image + ".oci.sha256"), + "name": ($asset_prefix + $arch_prefix + $build_image + ".oci.sha256"), "content_type": "text/plain" } ]' <<<"${assets}")" @@ -1261,9 +1261,9 @@ jobs: sbom_asset_name_build_image=$( echo '[ "'"$os_name"'", "'"$os_codename"'", "'"$build_image"'", "'"$stack_type"'", "'"$version"'", "'"$arch_prefix"'", "receipt.cyclonedx.json" ]' | jq -r 'map(select(length > 0)) | join("-")') elif [[ "$pattern_assets_prefix" == "os_name-os_codename-build_image_run_image-version-arch" ]]; then - sbom_asset_name_run_image=$( echo '[ "'"$os_name"'", "'"$os_codename"'", "'"$run_image"'", "'"$stack_type"'", "'"$version"'", + sbom_asset_name_run_image=$( echo '[ "'"$os_name"'", "'"$os_codename"'", "'"$run_image"'", "'"$version"'", "'"$arch_prefix"'", "receipt.cyclonedx.json" ]' | jq -r 'map(select(length > 0)) | join("-")' ) - sbom_asset_name_build_image=$(echo '[ "'"$os_name"'", "'"$os_codename"'", "'"$build_image"'", "'"$version"'", "'"$arch_prefix"'", + sbom_asset_name_build_image=$(echo '[ "'"$os_name"'", "'"$os_codename"'", "'"$build_image"'", "'"$version"'", "'"$arch_prefix"'", "receipt.cyclonedx.json" ]' | jq -r 'map(select(length > 0)) | join("-")') elif [[ "$pattern_assets_prefix" == "os_codename-stack_type-repo_type-version-arch-build_image_run_image" ]]; then sbom_asset_name_run_image=$(echo '[ "'"$os_codename"'", "'"$stack_type"'", "'"$repo_type"'", "'"$version"'", "'"$arch_prefix"'", diff --git a/stack/.github/workflows/update-github-config.yml b/stack/.github/workflows/update-github-config.yml index 0287559a..0b3c4244 100644 --- a/stack/.github/workflows/update-github-config.yml +++ b/stack/.github/workflows/update-github-config.yml @@ -2,7 +2,7 @@ name: Update shared github-config on: schedule: - - cron: '44 3 * * *' # daily at 3:44 AM UTC + - cron: '44 3 * * *' # daily at 3:44 AM UTC workflow_dispatch: {} concurrency: github_config_update