Skip to content

Commit 056274f

Browse files
NickLarsenNZdervoeti
NickLarsenNZ
and
dervoeti
authored
ci(dev): Extract digests from structured output (#801)
* ci(release): revert bad change * ci(actions/publish-image): revert bad change * ci(dev/airflow): revert debug change to workflow (#800)" This reverts commit 5a4b83a. * ci(action/publish-image): extract digest from structured data * ci(action/publish-manifest): extract digest from structured data * Update .github/actions/publish-manifest/action.yml Co-authored-by: Lukas Voetmand <lukas.voetmand@stackable.tech> --------- Co-authored-by: Lukas Voetmand <lukas.voetmand@stackable.tech>
1 parent 5a4b83a commit 056274f

4 files changed

Lines changed: 105 additions & 33 deletions

File tree

‎.github/actions/publish-image/action.yml‎

Lines changed: 81 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -59,62 +59,124 @@ runs:
5959
echo "IMAGE_NAME=$IMAGE_NAME" >> $GITHUB_ENV
6060
echo "TAG_NAME=$TAG_NAME" >> $GITHUB_ENV
6161
62+
# TODO (@NickLarsenNZ): Make this a reusable action in .github/actions/push-and-sign/action.yml
6263
- name: Push Image to repo.stackable.tech and sign via cosign
6364
shell: bash
6465
run: |
6566
set -euo pipefail
66-
docker image push "$(< bake-target-tags)"
67+
TARGET_TAG="$(< bake-target-tags)"
68+
docker image push "$TARGET_TAG"
6769
# Obtain the digest of the image, because signing by tag is deprecated and will be removed from cosign in the future
68-
DIGEST=$(docker images --digests "$(< bake-target-tags)" --format '{{.Digest}}')
69-
echo "DIGEST=$DIGEST" >> $GITHUB_ENV
70+
# This must be done after a push, otherwise the repo digest is empty.
71+
# Find the digest for the image. Notice the architecture at the end of TAG
72+
# IMAGE_VERSION does not contain the architecture (otherwise we could have used ${IMAGE_NAME}:${IMAGE_VERSION})
73+
# Given:
74+
# IMAGE_NAME: docker.stackable.tech/stackable/hello-world
75+
# TARGET_TAG: docker.stackable.tech/stackable/hello-world:0.0.1-SNAPSHOT-stackable0.0.0-dev-arm64
76+
# Expect:
77+
# STDOUT: docker.stackable.tech/stackable/hello-world@sha256:917f800259ef4915f976e93987b752fd64debf347568610d7f685d20220fc88a
78+
REPO_DIGEST=$(
79+
docker inspect "$TARGET_TAG" --format json | \
80+
jq -r \
81+
--arg IMAGE_NAME "$IMAGE_NAME" \
82+
--arg TARGET_TAG "$TARGET_TAG" \
83+
'
84+
map(select(.RepoTags[] | contains($TARGET_TAG)))[0]
85+
| .RepoDigests[]
86+
| select(. | startswith($IMAGE_NAME))
87+
'
88+
)
89+
# Ensure REPO_DIGEST is not empty
90+
if [[ -z "$REPO_DIGEST" ]]; then
91+
>&2 echo "Repo Digest is empty, but is required for signing"
92+
exit 1
93+
fi
94+
95+
# Needed by future steps
96+
echo "REPO_DIGEST=$REPO_DIGEST" | tee -a $GITHUB_ENV
97+
7098
# Refer to image via its digest (docker.stackable.tech/stackable/airflow@sha256:0a1b2c...)
7199
# This generates a signature and publishes it to the registry, next to the image
72100
# Uses the keyless signing flow with Github Actions as identity provider
73-
cosign sign -y "$IMAGE_NAME@$DIGEST"
101+
cosign sign -y "${REPO_DIGEST}"
74102
75103
- name: Generate SBOM for the Nexus Image
76104
shell: bash
77105
run: |
78106
set -euo pipefail
79-
syft scan --output cyclonedx-json=sbom.json --select-catalogers "-cargo-auditable-binary-cataloger" --scope all-layers --source-name "${{ inputs.product }}" --source-version "$TAG_NAME" "$IMAGE_NAME@$DIGEST";
107+
syft scan --output cyclonedx-json=sbom.json --select-catalogers "-cargo-auditable-binary-cataloger" --scope all-layers --source-name "${{ inputs.product }}" --source-version "$TAG_NAME" "${REPO_DIGEST}";
108+
# The DIGEST is the right side of `REPO_DIGEST` (split by '@')
109+
DIGEST=${REPO_DIGEST#*@}
80110
# Determine the PURL for the image
81111
PURL="pkg:docker/stackable/${{ inputs.product }}@$DIGEST?repository_url=docker.stackable.tech";
82112
# Get metadata from the image
83-
IMAGE_METADATA_DESCRIPTION=$(docker inspect --format='{{.Config.Labels.description}}' "$IMAGE_NAME@$DIGEST");
84-
IMAGE_METADATA_NAME=$(docker inspect --format='{{.Config.Labels.name}}' "$IMAGE_NAME@$DIGEST");
113+
IMAGE_METADATA_DESCRIPTION=$(docker inspect --format='{{.Config.Labels.description}}' "${REPO_DIGEST}");
114+
IMAGE_METADATA_NAME=$(docker inspect --format='{{.Config.Labels.name}}' "${REPO_DIGEST}");
85115
# Merge the SBOM with the metadata for the image
86116
jq -s '{"metadata":{"component":{"description":"'"$IMAGE_METADATA_NAME. $IMAGE_METADATA_DESCRIPTION"'","supplier":{"name":"Stackable GmbH","url":["https://stackable.tech/"]},"author":"Stackable GmbH","purl":"'"$PURL"'","publisher":"Stackable GmbH"}}} * .[0]' sbom.json > sbom.merged.json;
87117
# Attest the SBOM to the image
88-
cosign attest -y --predicate sbom.merged.json --type cyclonedx "$IMAGE_NAME@$DIGEST"
118+
cosign attest -y --predicate sbom.merged.json --type cyclonedx "${REPO_DIGEST}"
89119
120+
# TODO (@NickLarsenNZ): Make this a reusable action in .github/actions/push-and-sign/action.yml
90121
- name: Push Image to oci.stackable.tech and sign via cosign
91122
shell: bash
92123
run: |
93124
set -euo pipefail
125+
# Update the registry
94126
IMAGE_NAME=oci.stackable.tech/sdp/${{ inputs.product }}
95-
echo "image: $IMAGE_NAME"
96-
docker tag "$(< bake-target-tags)" "$IMAGE_NAME:$TAG_NAME"
97-
docker image push "$(< bake-target-tags)"
127+
# IMAGE_NAME is needed by future steps
128+
echo "IMAGE_NAME=$IMAGE_NAME" | tee -a $GITHUB_ENV
129+
OLD_TARGET_TAG="$(< bake-target-tags)"
130+
TARGET_TAG="$IMAGE_NAME:$TAG_NAME"
131+
docker tag "$OLD_TARGET_TAG" "$TARGET_TAG"
132+
docker image push "$TARGET_TAG"
98133
# Obtain the digest of the image, because signing by tag is deprecated and will be removed from cosign in the future
99-
DIGEST=$(docker images --digests "$(< bake-target-tags)" --format '{{.Digest}}')
100-
echo "DIGEST=$DIGEST" >> $GITHUB_ENV
134+
# This must be done after a push, otherwise the repo digest is empty.
135+
# Find the digest for the image. Notice the architecture at the end of TAG
136+
# IMAGE_VERSION does not contain the architecture (otherwise we could have used ${IMAGE_NAME}:${IMAGE_VERSION})
137+
# Given:
138+
# IMAGE_NAME: oci.stackable.tech/sdp/hello-world
139+
# TARGET_TAG: oci.stackable.tech/sdp/hello-world:0.0.1-SNAPSHOT-stackable0.0.0-dev-arm64
140+
# Expect:
141+
# STDOUT: oci.stackable.tech/sdp/hello-world@sha256:917f800259ef4915f976e93987b752fd64debf347568610d7f685d20220fc88a
142+
REPO_DIGEST=$(
143+
docker inspect "$TARGET_TAG" --format json | \
144+
jq -r \
145+
--arg IMAGE_NAME "$IMAGE_NAME" \
146+
--arg TARGET_TAG "$TARGET_TAG" \
147+
'
148+
map(select(.RepoTags[] | contains($TARGET_TAG)))[0]
149+
| .RepoDigests[]
150+
| select(. | startswith($IMAGE_NAME))
151+
'
152+
)
153+
# Ensure REPO_DIGEST is not empty
154+
if [[ -z "$REPO_DIGEST" ]]; then
155+
>&2 echo "Repo Digest is empty, but is required for signing"
156+
exit 1
157+
fi
158+
159+
# REPO_DIGEST is needed by future steps
160+
echo "REPO_DIGEST=$REPO_DIGEST" | tee -a $GITHUB_ENV
161+
101162
# Refer to image via its digest (oci.stackable.tech/sdp/airflow@sha256:0a1b2c...)
102163
# This generates a signature and publishes it to the registry, next to the image
103164
# Uses the keyless signing flow with Github Actions as identity provider
104-
cosign sign -y "$IMAGE_NAME@$DIGEST"
165+
cosign sign -y "${REPO_DIGEST}"
105166
106167
- name: Generate SBOM for the Harbor Image
107168
shell: bash
108169
run: |
109170
set -euo pipefail
110-
IMAGE_NAME=oci.stackable.tech/sdp/${{ inputs.product }}
111-
syft scan --output cyclonedx-json=sbom.json --select-catalogers "-cargo-auditable-binary-cataloger" --scope all-layers --source-name "${{ inputs.product }}" --source-version "$TAG_NAME" "$IMAGE_NAME@$DIGEST";
171+
syft scan --output cyclonedx-json=sbom.json --select-catalogers "-cargo-auditable-binary-cataloger" --scope all-layers --source-name "${{ inputs.product }}" --source-version "$TAG_NAME" "${REPO_DIGEST}";
172+
# The DIGEST is the right side of `REPO_DIGEST` (split by '@')
173+
DIGEST=${REPO_DIGEST#*@}
112174
# Determine the PURL for the image
113175
PURL="pkg:docker/sdp/${{ inputs.product }}@$DIGEST?repository_url=oci.stackable.tech";
114176
# Get metadata from the image
115-
IMAGE_METADATA_DESCRIPTION=$(docker inspect --format='{{.Config.Labels.description}}' "$IMAGE_NAME@$DIGEST");
116-
IMAGE_METADATA_NAME=$(docker inspect --format='{{.Config.Labels.name}}' "$IMAGE_NAME@$DIGEST");
177+
IMAGE_METADATA_DESCRIPTION=$(docker inspect --format='{{.Config.Labels.description}}' "${REPO_DIGEST}");
178+
IMAGE_METADATA_NAME=$(docker inspect --format='{{.Config.Labels.name}}' "${REPO_DIGEST}");
117179
# Merge the SBOM with the metadata for the image
118180
jq -s '{"metadata":{"component":{"description":"'"$IMAGE_METADATA_NAME. $IMAGE_METADATA_DESCRIPTION"'","supplier":{"name":"Stackable GmbH","url":["https://stackable.tech/"]},"author":"Stackable GmbH","purl":"'"$PURL"'","publisher":"Stackable GmbH"}}} * .[0]' sbom.json > sbom.merged.json;
119181
# Attest the SBOM to the image
120-
cosign attest -y --predicate sbom.merged.json --type cyclonedx "$IMAGE_NAME@$DIGEST"
182+
cosign attest -y --predicate sbom.merged.json --type cyclonedx "${REPO_DIGEST}"

‎.github/actions/publish-manifest/action.yml‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,14 +45,25 @@ runs:
4545
shell: bash
4646
run: |
4747
set -euo pipefail
48+
49+
# Get the image index digest
50+
# Eg: index_digest docker.stackable.tech/stackable/hello-world:0.0.1-SNAPSHOT-stackable0.0.0-dev
51+
#
52+
# Note: `docker manifest push` currently outputs the same hash, but `manifest` is experimental and the
53+
# STDOUT is more likely to change than the structured output.
54+
function index_digest {
55+
docker buildx imagetools inspect --format '{{println .Manifest.Digest}}' "$1"
56+
}
57+
4858
MANIFEST_NAME="docker.stackable.tech/stackable/${{ inputs.product }}:$IMAGE_VERSION"
4959
# Create and push to Stackable Nexus
5060
# `docker manifest push` directly returns the digest of the manifest list
5161
# As it is an experimental feature, this might change in the future
5262
# Further reading: https://docs.docker.com/reference/cli/docker/manifest/push/
5363
# --amend because the manifest list would be updated since we use the same tag: 0.0.0-dev
5464
docker manifest create "$MANIFEST_NAME" --amend "${MANIFEST_NAME}-amd64" --amend "${MANIFEST_NAME}-arm64"
55-
DIGEST=$(docker manifest push $MANIFEST_NAME)
65+
docker manifest push "$MANIFEST_NAME"
66+
DIGEST=$(index_digest "$MANIFEST_NAME")
5667
5768
# Refer to image via its digest (oci.stackable.tech/sdp/airflow@sha256:0a1b2c...)
5869
# This generates a signature and publishes it to the registry, next to the image
@@ -62,5 +73,6 @@ runs:
6273
# Push to oci.stackable.tech as well
6374
MANIFEST_NAME="oci.stackable.tech/sdp/${{ inputs.product }}:$IMAGE_VERSION"
6475
docker manifest create "$MANIFEST_NAME" --amend "${MANIFEST_NAME}-amd64" --amend "${MANIFEST_NAME}-arm64"
65-
DIGEST=$(docker manifest push $MANIFEST_NAME)
76+
docker manifest push $MANIFEST_NAME
77+
DIGEST=$(index_digest "$MANIFEST_NAME")
6678
cosign sign -y "$MANIFEST_NAME@$DIGEST"

‎.github/workflows/dev_airflow.yaml‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -55,12 +55,6 @@ jobs:
5555
product-name: ${{ env.PRODUCT_NAME }}
5656
product-version: ${{ matrix.versions }}
5757
build-cache-nexus-password: ${{ secrets.BUILD_CACHE_NEXUS_PASSWORD }}
58-
- name: debug
59-
run: |
60-
set -x
61-
cat bake-target-tags
62-
DIGEST=$(docker images --digests "$(< bake-target-tags)" --format '{{.Digest}}')
63-
echo "DIGEST=$DIGEST"
6458
- name: Publish Product Image
6559
uses: ./.github/actions/publish-image
6660
with:

‎.github/workflows/release.yml‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -113,9 +113,11 @@ jobs:
113113
TAG_NAME=$(cut -d ":" -f 2 < bake-target-tags)
114114
echo "image: $IMAGE_NAME"
115115
echo "tag: $TAG_NAME"
116-
docker image push "$(< bake-target-tags)"
117-
# Obtain the digest of the image, because signing by tag is deprecated and will be removed from cosign in the future
118-
DIGEST=$(docker images --digests "$(< bake-target-tags)" --format '{{.Digest}}')
116+
# Store the output of `docker image push` into a variable, so we can parse it for the digest
117+
PUSH_OUTPUT=$(docker image push "$(< bake-target-tags)" 2>&1)
118+
echo "$PUSH_OUTPUT"
119+
# Obtain the digest of the pushed image from the output of `docker image push`, because signing by tag is deprecated and will be removed from cosign in the future
120+
DIGEST=$(echo "$PUSH_OUTPUT" | awk "/: digest: sha256:[a-f0-9]{64} size: [0-9]+$/ { print \$3 }")
119121
# Refer to image via its digest (docker.stackable.tech/stackable/airflow@sha256:0a1b2c...)
120122
# This generates a signature and publishes it to the registry, next to the image
121123
# Uses the keyless signing flow with Github Actions as identity provider
@@ -137,9 +139,11 @@ jobs:
137139
IMAGE_NAME=oci.stackable.tech/sdp/${{ matrix.product }}
138140
echo "image: $IMAGE_NAME"
139141
docker tag "$(< bake-target-tags)" "$IMAGE_NAME:$TAG_NAME"
140-
docker image push "$IMAGE_NAME:$TAG_NAME"
141-
# Obtain the digest of the image, because signing by tag is deprecated and will be removed from cosign in the future
142-
DIGEST=$(docker images --digests "$(< bake-target-tags)" --format '{{.Digest}}')
142+
# Store the output of `docker image push` into a variable, so we can parse it for the digest
143+
PUSH_OUTPUT=$(docker image push "$IMAGE_NAME:$TAG_NAME" 2>&1)
144+
echo "$PUSH_OUTPUT"
145+
# Obtain the digest of the pushed image from the output of `docker image push`, because signing by tag is deprecated and will be removed from cosign in the future
146+
DIGEST=$(echo "$PUSH_OUTPUT" | awk "/: digest: sha256:[a-f0-9]{64} size: [0-9]+$/ { print \$3 }")
143147
# Refer to image via its digest (oci.stackable.tech/sdp/airflow@sha256:0a1b2c...)
144148
# This generates a signature and publishes it to the registry, next to the image
145149
# Uses the keyless signing flow with Github Actions as identity provider

0 commit comments

Comments
 (0)