Skip to content

Commit 6961371

Browse files
bgentrybgentry
committed
fix(ci) harden offline sigstore bundle verification
The offline bundle verification step would fail when oras discover returned a referrers index with manifests=null. To resolve this, I implemented a separate script which: - Extracts referrers prefetch + offline verification into scripts/ci - Force-fetch subject + referrers via curl (custom header) before oras - Retry discovery briefly and handle empty referrer sets safely
1 parent 4e58b55 commit 6961371

3 files changed

Lines changed: 207 additions & 60 deletions

File tree

‎.github/workflows/docker-riverproui.yaml‎

Lines changed: 17 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -506,6 +506,12 @@ jobs:
506506
# If neither MANIFEST_DIGEST (from merge job) nor ecr_manifest_digest (manual input)
507507
# is available, we skip index referrer prefetch rather than fail.
508508
- name: Prefetch all referrers content
509+
env:
510+
AUTH_USER: river
511+
AUTH_PASSWORD: ${{ secrets.RIVERPRO_GO_MOD_CREDENTIAL }}
512+
FORCE_FETCH_SECRET: ${{ secrets.FORCE_FETCH_SECRET }}
513+
REGISTRY_MANIFEST_URL: https://riverqueue.com/v2/riverproui/manifests
514+
REGISTRY_REFERRERS_URL: https://riverqueue.com/v2/riverproui/referrers
509515
run: |
510516
# Include both the ECR-derived index digest (attestations live here)
511517
# and the live registry's index digest (may differ but could gain referrers).
@@ -518,24 +524,7 @@ jobs:
518524
echo "Note: no index subject digest provided; skipping index referrers prefetch."
519525
fi
520526
subjects+=("${{ steps.resolve-digest.outputs.digest }}" "${{ steps.platform-digests.outputs.amd64_digest }}" "${{ steps.platform-digests.outputs.arm64_digest }}")
521-
for subject in "${subjects[@]}"; do
522-
oras discover --format json "$IMAGE_NAME@$subject" | tee /tmp/referrers.json
523-
digests=$(jq -r '.manifests[]?.digest // empty' /tmp/referrers.json)
524-
for d in $digests; do
525-
tmpfile=$(mktemp)
526-
# Fetch the full referrer manifest to a file so we can parse layers + config
527-
oras manifest fetch --output "$tmpfile" "$IMAGE_NAME@$d"
528-
# Prefetch config blob (often empty but warms cache)
529-
cfg=$(jq -r '.config.digest // empty' "$tmpfile")
530-
if [ -n "$cfg" ]; then
531-
oras blob fetch --output /dev/null "$IMAGE_NAME@$cfg"
532-
fi
533-
# Prefetch any layer blobs (DSSE envelope, bundle, etc.)
534-
jq -r '.layers[]?.digest // empty' "$tmpfile" | while read -r ld; do
535-
[ -n "$ld" ] && oras blob fetch --output /dev/null "$IMAGE_NAME@$ld"
536-
done
537-
done
538-
done
527+
bash scripts/ci/prefetch-referrers-content.sh "${subjects[@]}"
539528
540529
# Verify index (if digest is available) and per-arch attestations against live registry.
541530
# Index-level attestation is bound to the ECR index digest; provide it via MANIFEST_DIGEST
@@ -576,55 +565,23 @@ jobs:
576565
577566
# Offline verification of Sigstore bundle: only attempt for index if digest provided.
578567
- name: Verify Sigstore bundle against subject bytes (offline crypto)
568+
env:
569+
AUTH_USER: river
570+
AUTH_PASSWORD: ${{ secrets.RIVERPRO_GO_MOD_CREDENTIAL }}
571+
FORCE_FETCH_SECRET: ${{ secrets.FORCE_FETCH_SECRET }}
572+
REGISTRY_MANIFEST_URL: https://riverqueue.com/v2/riverproui/manifests
573+
REGISTRY_REFERRERS_URL: https://riverqueue.com/v2/riverproui/referrers
579574
run: |
580575
FALLBACK="${{ inputs.ecr_manifest_digest }}"
581576
INDEX_SUBJECT="${MANIFEST_DIGEST:-$FALLBACK}"
582-
verify_bundle () {
583-
subject="$1"
584-
# Exact subject bytes
585-
oras manifest fetch --output /tmp/subject.json "$IMAGE_NAME@$subject"
586-
587-
# Find *all* bundle candidates for this subject (both media-type spellings)
588-
refjson=$(mktemp)
589-
oras discover --format json "$IMAGE_NAME@$subject" > "$refjson"
590-
mapfile -t bundle_mfs < <(jq -r '.manifests[]
591-
| select(.artifactType=="application/vnd.dev.sigstore.bundle.v0.3+json"
592-
or .artifactType=="application/vnd.dev.sigstore.bundle+json;version=0.3")
593-
| .digest' "$refjson")
594-
595-
ok=0
596-
for bmf in "${bundle_mfs[@]}"; do
597-
[ -z "$bmf" ] && continue
598-
mf=$(mktemp)
599-
oras manifest fetch --output "$mf" "$IMAGE_NAME@$bmf"
600-
bundle_blob=$(jq -r '.layers[0].digest' "$mf")
601-
oras blob fetch --output /tmp/bundle.json "$IMAGE_NAME@$bundle_blob"
602-
603-
if cosign verify-blob-attestation \
604-
--bundle /tmp/bundle.json \
605-
--new-bundle-format \
606-
--certificate-oidc-issuer "https://token.actions.githubusercontent.com" \
607-
--certificate-identity-regexp '^https://github.com/riverqueue/riverui/.*' \
608-
/tmp/subject.json > /tmp/verify.out 2>&1; then
609-
echo "Verified OK with bundle $bmf"
610-
ok=1
611-
break
612-
fi
613-
done
614-
615-
if [ "$ok" -ne 1 ]; then
616-
echo "No matching bundle verified for subject $subject"
617-
cat /tmp/verify.out || true
618-
exit 1
619-
fi
620-
}
621577
if [ -n "$INDEX_SUBJECT" ]; then
622-
verify_bundle "$INDEX_SUBJECT"
578+
bash scripts/ci/verify-sigstore-bundles-offline.sh "$INDEX_SUBJECT"
623579
else
624580
echo "Skipping offline index bundle verification: no MANIFEST_DIGEST or ecr_manifest_digest provided."
625581
fi
626-
verify_bundle "${{ steps.platform-digests.outputs.amd64_digest }}"
627-
verify_bundle "${{ steps.platform-digests.outputs.arm64_digest }}"
582+
bash scripts/ci/verify-sigstore-bundles-offline.sh \
583+
"${{ steps.platform-digests.outputs.amd64_digest }}" \
584+
"${{ steps.platform-digests.outputs.arm64_digest }}"
628585
629586
- name: Upload debug artifacts
630587
if: failure() || cancelled()

‎scripts/ci/prefetch-referrers-content.sh‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
# Prefetch referrers for one or more OCI subjects (digests), and warm their blobs.
5+
#
6+
# Required env:
7+
# - IMAGE_NAME: e.g. "riverqueue.com/riverproui"
8+
# - AUTH_USER: basic auth username
9+
# - AUTH_PASSWORD: basic auth password
10+
# - FORCE_FETCH_SECRET: value for X-Force-Fetch-From-Upstream (best-effort)
11+
# - REGISTRY_MANIFEST_URL: e.g. "https://riverqueue.com/v2/riverproui/manifests"
12+
# - REGISTRY_REFERRERS_URL: e.g. "https://riverqueue.com/v2/riverproui/referrers"
13+
#
14+
# Usage:
15+
# bash scripts/ci/prefetch-referrers-content.sh sha256:... sha256:...
16+
17+
: "${IMAGE_NAME:?IMAGE_NAME is required}"
18+
: "${AUTH_USER:?AUTH_USER is required}"
19+
: "${AUTH_PASSWORD:?AUTH_PASSWORD is required}"
20+
: "${FORCE_FETCH_SECRET:?FORCE_FETCH_SECRET is required}"
21+
: "${REGISTRY_MANIFEST_URL:?REGISTRY_MANIFEST_URL is required}"
22+
: "${REGISTRY_REFERRERS_URL:?REGISTRY_REFERRERS_URL is required}"
23+
24+
ACCEPT_MANIFESTS="application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.manifest.v1+json"
25+
26+
force_fetch_manifest_and_referrers() {
27+
subject="$1"
28+
29+
# Best-effort: warm the pull-through cache for the subject bytes and its referrers index.
30+
curl -fsS -u "${AUTH_USER}:${AUTH_PASSWORD}" \
31+
-H "X-Force-Fetch-From-Upstream: ${FORCE_FETCH_SECRET}" \
32+
-H "Accept: ${ACCEPT_MANIFESTS}" \
33+
"${REGISTRY_MANIFEST_URL}/${subject}" -o /dev/null || true
34+
35+
curl -fsS -u "${AUTH_USER}:${AUTH_PASSWORD}" \
36+
-H "X-Force-Fetch-From-Upstream: ${FORCE_FETCH_SECRET}" \
37+
-H "Accept: application/vnd.oci.image.index.v1+json" \
38+
"${REGISTRY_REFERRERS_URL}/${subject}" -o /dev/null || true
39+
}
40+
41+
if [ "$#" -lt 1 ]; then
42+
echo "usage: $0 <subject-digest> [<subject-digest> ...]" >&2
43+
exit 2
44+
fi
45+
46+
for subject in "$@"; do
47+
[ -z "$subject" ] && continue
48+
49+
force_fetch_manifest_and_referrers "$subject"
50+
51+
# Discover and warm each referrer manifest and its blobs.
52+
oras discover --format json "${IMAGE_NAME}@${subject}" | tee /tmp/referrers.json
53+
digests="$(jq -r '.manifests[]?.digest // empty' /tmp/referrers.json)"
54+
for d in $digests; do
55+
[ -z "$d" ] && continue
56+
57+
tmpfile="$(mktemp)"
58+
oras manifest fetch --output "$tmpfile" "${IMAGE_NAME}@${d}"
59+
60+
cfg="$(jq -r '.config.digest // empty' "$tmpfile")"
61+
if [ -n "$cfg" ]; then
62+
oras blob fetch --output /dev/null "${IMAGE_NAME}@${cfg}"
63+
fi
64+
65+
jq -r '.layers[]?.digest // empty' "$tmpfile" | while read -r ld; do
66+
[ -n "$ld" ] && oras blob fetch --output /dev/null "${IMAGE_NAME}@${ld}"
67+
done
68+
done
69+
done
70+
71+

‎scripts/ci/verify-sigstore-bundles-offline.sh‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
# Offline verification of Sigstore bundles against exact subject bytes.
5+
#
6+
# Required env:
7+
# - IMAGE_NAME: e.g. "riverqueue.com/riverproui"
8+
# - AUTH_USER: basic auth username
9+
# - AUTH_PASSWORD: basic auth password
10+
# - FORCE_FETCH_SECRET: value for X-Force-Fetch-From-Upstream (best-effort)
11+
# - REGISTRY_MANIFEST_URL: e.g. "https://riverqueue.com/v2/riverproui/manifests"
12+
# - REGISTRY_REFERRERS_URL: e.g. "https://riverqueue.com/v2/riverproui/referrers"
13+
#
14+
# Usage:
15+
# bash scripts/ci/verify-sigstore-bundles-offline.sh sha256:... sha256:... sha256:...
16+
#
17+
# Notes:
18+
# - We "force fetch" the subject + referrers index first to avoid ORAS limitations around custom headers.
19+
# - We retry discovery briefly to avoid eventual-consistency/race flakes in pull-through caches.
20+
21+
: "${IMAGE_NAME:?IMAGE_NAME is required}"
22+
: "${AUTH_USER:?AUTH_USER is required}"
23+
: "${AUTH_PASSWORD:?AUTH_PASSWORD is required}"
24+
: "${FORCE_FETCH_SECRET:?FORCE_FETCH_SECRET is required}"
25+
: "${REGISTRY_MANIFEST_URL:?REGISTRY_MANIFEST_URL is required}"
26+
: "${REGISTRY_REFERRERS_URL:?REGISTRY_REFERRERS_URL is required}"
27+
28+
ACCEPT_MANIFESTS="application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.manifest.v1+json"
29+
30+
echo "oras: $(oras version 2>/dev/null || echo unknown)"
31+
echo "cosign: $(cosign version 2>/dev/null || echo unknown)"
32+
echo "jq: $(jq --version 2>/dev/null || echo unknown)"
33+
34+
force_fetch_manifest_and_referrers() {
35+
subject="$1"
36+
37+
curl -fsS -u "${AUTH_USER}:${AUTH_PASSWORD}" \
38+
-H "X-Force-Fetch-From-Upstream: ${FORCE_FETCH_SECRET}" \
39+
-H "Accept: ${ACCEPT_MANIFESTS}" \
40+
"${REGISTRY_MANIFEST_URL}/${subject}" -o /dev/null || true
41+
42+
curl -fsS -u "${AUTH_USER}:${AUTH_PASSWORD}" \
43+
-H "X-Force-Fetch-From-Upstream: ${FORCE_FETCH_SECRET}" \
44+
-H "Accept: application/vnd.oci.image.index.v1+json" \
45+
"${REGISTRY_REFERRERS_URL}/${subject}" -o /dev/null || true
46+
}
47+
48+
verify_bundle_for_subject() {
49+
subject="$1"
50+
: > /tmp/verify.out
51+
52+
# Exact subject bytes (do not go through tag indirections).
53+
oras manifest fetch --output /tmp/subject.json "${IMAGE_NAME}@${subject}"
54+
55+
bundle_mfs=()
56+
refjson=""
57+
for attempt in 1 2 3 4 5; do
58+
refjson="$(mktemp)"
59+
force_fetch_manifest_and_referrers "$subject"
60+
oras discover --format json "${IMAGE_NAME}@${subject}" > "$refjson"
61+
62+
mapfile -t bundle_mfs < <(jq -r '(.manifests // [])[]
63+
| select(.artifactType=="application/vnd.dev.sigstore.bundle.v0.3+json"
64+
or .artifactType=="application/vnd.dev.sigstore.bundle+json;version=0.3")
65+
| .digest' "$refjson")
66+
67+
if [ ${#bundle_mfs[@]} -gt 0 ]; then
68+
break
69+
fi
70+
71+
if [ "$attempt" -lt 5 ]; then
72+
echo "No bundle referrers discovered yet for ${subject} (attempt ${attempt}/5); retrying soon..."
73+
sleep $((attempt * 2))
74+
fi
75+
done
76+
77+
ok=0
78+
for bmf in "${bundle_mfs[@]}"; do
79+
[ -z "$bmf" ] && continue
80+
81+
mf="$(mktemp)"
82+
oras manifest fetch --output "$mf" "${IMAGE_NAME}@${bmf}"
83+
bundle_blob="$(jq -r '.layers[0].digest' "$mf")"
84+
oras blob fetch --output /tmp/bundle.json "${IMAGE_NAME}@${bundle_blob}"
85+
86+
if cosign verify-blob-attestation \
87+
--bundle /tmp/bundle.json \
88+
--new-bundle-format \
89+
--certificate-oidc-issuer "https://token.actions.githubusercontent.com" \
90+
--certificate-identity-regexp '^https://github.com/riverqueue/riverui/.*' \
91+
/tmp/subject.json > /tmp/verify.out 2>&1; then
92+
echo "Verified OK with bundle ${bmf} for subject ${subject}"
93+
ok=1
94+
break
95+
fi
96+
done
97+
98+
if [ "$ok" -ne 1 ]; then
99+
echo "No matching bundle verified for subject ${subject}"
100+
if [ ${#bundle_mfs[@]} -eq 0 ]; then
101+
echo "Note: no bundle referrers were discovered for this subject. Referrers index JSON:"
102+
cat "$refjson" || true
103+
fi
104+
cat /tmp/verify.out || true
105+
exit 1
106+
fi
107+
}
108+
109+
if [ "$#" -lt 1 ]; then
110+
echo "usage: $0 <subject-digest> [<subject-digest> ...]" >&2
111+
exit 2
112+
fi
113+
114+
for subject in "$@"; do
115+
[ -z "$subject" ] && continue
116+
verify_bundle_for_subject "$subject"
117+
done
118+
119+

0 commit comments

Comments
 (0)