Skip to content

Commit 72d2f0c

Browse files
pennypacker-hepennypacker-he
committed
[minor] Speed up package repository publication
1 parent a2d18f3 commit 72d2f0c

3 files changed

Lines changed: 181 additions & 20 deletions

File tree

‎.github/workflows/package-lock-test.yaml‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,15 @@ on:
44
pull_request:
55
paths:
66
- "scripts/publish-package-repo.sh"
7+
- "scripts/test-publish-cache-control.sh"
78
- "scripts/test-publish-lock.sh"
89
- ".github/workflows/package-lock-test.yaml"
910
push:
1011
branches:
1112
- main
1213
paths:
1314
- "scripts/publish-package-repo.sh"
15+
- "scripts/test-publish-cache-control.sh"
1416
- "scripts/test-publish-lock.sh"
1517
- ".github/workflows/package-lock-test.yaml"
1618

@@ -29,7 +31,14 @@ jobs:
2931
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
3032

3133
- name: Check shell syntax
32-
run: bash -n scripts/publish-package-repo.sh scripts/test-publish-lock.sh
34+
run: >-
35+
bash -n
36+
scripts/publish-package-repo.sh
37+
scripts/test-publish-cache-control.sh
38+
scripts/test-publish-lock.sh
39+
40+
- name: Test cache-control upload contract
41+
run: bash scripts/test-publish-cache-control.sh
3342

3443
- name: Test lock handoff race
3544
run: bash scripts/test-publish-lock.sh

‎scripts/publish-package-repo.sh‎

Lines changed: 13 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -241,18 +241,24 @@ PY
241241
done
242242
}
243243

244-
is_deferred_repository_metadata_path() {
244+
is_package_asset_path() {
245245
local relative_path="$1"
246246

247247
case "$relative_path" in
248-
dists/*|Packages|Packages.*|Release|Release.gpg|InRelease|*/repodata/*)
248+
*.deb|*.rpm|*.apk|by-hash/*|*/by-hash/*)
249249
return 0
250250
;;
251251
esac
252252

253253
return 1
254254
}
255255

256+
is_deferred_repository_metadata_path() {
257+
local relative_path="$1"
258+
259+
[ "$relative_path" != ".publish.lock" ] && ! is_package_asset_path "$relative_path"
260+
}
261+
256262
metadata_upload_order() {
257263
local relative_path="$1"
258264

@@ -293,14 +299,11 @@ metadata_upload_order() {
293299
cache_control_for_path() {
294300
local relative_path="$1"
295301

296-
case "$relative_path" in
297-
*.deb|*.rpm|*.apk|by-hash/*|*/by-hash/*)
298-
printf '%s\n' "$PACKAGE_ASSET_CACHE_CONTROL"
299-
;;
300-
*)
301-
printf '%s\n' "$PACKAGE_METADATA_CACHE_CONTROL"
302-
;;
303-
esac
302+
if is_package_asset_path "$relative_path"; then
303+
printf '%s\n' "$PACKAGE_ASSET_CACHE_CONTROL"
304+
else
305+
printf '%s\n' "$PACKAGE_METADATA_CACHE_CONTROL"
306+
fi
304307
}
305308

306309
upload_staged_file() {
@@ -721,15 +724,6 @@ gcloud storage rsync \
721724
log_step "Uploading repository metadata to $destination"
722725
upload_deferred_repository_metadata
723726

724-
log_step "Updating cache-control metadata"
725-
while IFS= read -r -d '' staged_file; do
726-
relative_path="${staged_file#"$PACKAGE_REPO_STAGE_DIR"/}"
727-
object_path="${destination%/}/${relative_path}"
728-
cache_control="$(cache_control_for_path "$relative_path")"
729-
730-
gcloud storage objects update "$object_path" --cache-control="$cache_control" >/dev/null
731-
done < <(find "$PACKAGE_REPO_STAGE_DIR" -type f -print0)
732-
733727
invalidate_cdn_cache
734728

735729
printf 'Published package repository to %s\n' "$destination"

‎scripts/test-publish-cache-control.sh‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
#!/usr/bin/env bash
2+
3+
set -Eeuo pipefail
4+
5+
repo_root="$(cd "$(dirname "$0")/.." && pwd)"
6+
tmp="$(mktemp -d "${TMPDIR:-/tmp}/package-cache-control-test.XXXXXX")"
7+
trap 'rm -rf "$tmp"' EXIT
8+
9+
mkdir -p "$tmp/bin" "$tmp/dist" "$tmp/stage"
10+
printf 'current package\n' >"$tmp/dist/widget-1.0.0-1.x86_64.rpm"
11+
12+
cat >"$tmp/bin/gcloud" <<'MOCK'
13+
#!/usr/bin/env bash
14+
set -Eeuo pipefail
15+
16+
log_file="${MOCK_GCLOUD_LOG:?}"
17+
printf '%q ' "$@" >>"$log_file"
18+
printf '\n' >>"$log_file"
19+
20+
case "$*" in
21+
"storage cp - gs://test-bucket/widget/.publish.lock --if-generation-match=0")
22+
cat >/dev/null
23+
;;
24+
"storage objects describe gs://test-bucket/widget/.publish.lock --format=value(generation,update_time)")
25+
printf '12345\t2026-01-01T00:00:00+0000\n'
26+
;;
27+
"storage rsync --recursive gs://test-bucket/widget "*)
28+
stage_dir="$5"
29+
mkdir -p \
30+
"$stage_dir/by-hash/SHA256" \
31+
"$stage_dir/dists/bookworm" \
32+
"$stage_dir/legacy"
33+
printf 'existing dependency\n' >"$stage_dir/dependency-2.0.0-1.x86_64.rpm"
34+
printf 'immutable index\n' >"$stage_dir/by-hash/SHA256/existing"
35+
printf 'mutable release metadata\n' >"$stage_dir/dists/bookworm/Release"
36+
printf 'mutable extension metadata\n' >"$stage_dir/legacy/channel.json"
37+
;;
38+
"storage rsync --recursive --cache-control=asset-contract "*)
39+
asset_dir="$5"
40+
while IFS= read -r -d '' asset_file; do
41+
relative_path="${asset_file#"$asset_dir"/}"
42+
printf 'ASSET %s\n' "$relative_path" >>"$log_file"
43+
case "$relative_path" in
44+
*.deb|*.rpm|*.apk|by-hash/*|*/by-hash/*)
45+
;;
46+
*)
47+
printf 'Non-asset path was included in the asset rsync: %s\n' "$relative_path" >&2
48+
exit 1
49+
;;
50+
esac
51+
done < <(find "$asset_dir" -type f -print0)
52+
;;
53+
"storage cp --cache-control=metadata-contract "*)
54+
test -f "$4"
55+
;;
56+
"secrets versions access latest "*)
57+
printf 'test secret\n'
58+
;;
59+
"storage rm gs://test-bucket/widget/.publish.lock --if-generation-match=12345")
60+
;;
61+
*)
62+
printf 'Unexpected gcloud invocation: %s\n' "$*" >&2
63+
exit 1
64+
;;
65+
esac
66+
MOCK
67+
68+
cat >"$tmp/bin/rpm" <<'MOCK'
69+
#!/usr/bin/env bash
70+
set -Eeuo pipefail
71+
72+
case "${*: -1}" in
73+
*dependency-*)
74+
printf 'dependency\n'
75+
;;
76+
*)
77+
printf 'widget\n'
78+
;;
79+
esac
80+
MOCK
81+
82+
cat >"$tmp/bin/gpg" <<'MOCK'
83+
#!/usr/bin/env bash
84+
set -Eeuo pipefail
85+
86+
output=""
87+
while [ "$#" -gt 0 ]; do
88+
if [ "$1" = "--output" ]; then
89+
output="$2"
90+
shift 2
91+
continue
92+
fi
93+
shift
94+
done
95+
96+
if [ -n "$output" ]; then
97+
printf 'generated by mock gpg\n' >"$output"
98+
else
99+
printf 'mock public key\n'
100+
fi
101+
MOCK
102+
103+
cat >"$tmp/bin/createrepo_c" <<'MOCK'
104+
#!/usr/bin/env bash
105+
set -Eeuo pipefail
106+
107+
repository_dir="${*: -1}"
108+
mkdir -p "$repository_dir/repodata"
109+
printf 'mutable RPM metadata\n' >"$repository_dir/repodata/repomd.xml"
110+
printf 'mutable RPM metadata payload\n' >"$repository_dir/repodata/primary.xml.gz"
111+
MOCK
112+
113+
chmod +x "$tmp/bin/gcloud" "$tmp/bin/rpm" "$tmp/bin/gpg" "$tmp/bin/createrepo_c"
114+
115+
gcloud_log="$tmp/gcloud.log"
116+
publisher_log="$tmp/publisher.log"
117+
PATH="$tmp/bin:$PATH" \
118+
MOCK_GCLOUD_LOG="$gcloud_log" \
119+
DIST_DIR="$tmp/dist" \
120+
GCLOUD_PROJECT=test-project \
121+
GCS_BUCKET=test-bucket \
122+
GCS_BUCKET_PREFIX=widget \
123+
PACKAGE_NAME=widget \
124+
PACKAGE_REPO_STAGE_DIR="$tmp/stage" \
125+
APTLY_GPG_KEY_ID=test-key \
126+
PACKAGE_ASSET_CACHE_CONTROL=asset-contract \
127+
PACKAGE_METADATA_CACHE_CONTROL=metadata-contract \
128+
CDN_INVALIDATE_CACHE=false \
129+
LOCK_HEARTBEAT_SECONDS=0 \
130+
bash "$repo_root/scripts/publish-package-repo.sh" >"$publisher_log" 2>&1
131+
132+
grep -Fq "storage rsync --recursive --cache-control=asset-contract " "$gcloud_log"
133+
grep -Fq "ASSET dependency-2.0.0-1.x86_64.rpm" "$gcloud_log"
134+
grep -Fq "ASSET widget-1.0.0-1.x86_64.rpm" "$gcloud_log"
135+
grep -Fq "ASSET by-hash/SHA256/existing" "$gcloud_log"
136+
137+
for metadata_path in \
138+
"dists/bookworm/Release" \
139+
"legacy/channel.json" \
140+
"rpm/repodata/primary.xml.gz" \
141+
"rpm/repodata/repomd.xml" \
142+
"rpm/repodata/repomd.xml.asc" \
143+
"widget-archive-keyring.asc" \
144+
"widget-archive-keyring.gpg"; do
145+
grep -Fq \
146+
"storage cp --cache-control=metadata-contract $tmp/stage/$metadata_path gs://test-bucket/widget/$metadata_path" \
147+
"$gcloud_log"
148+
done
149+
150+
if grep -Fq "storage objects update" "$gcloud_log"; then
151+
printf 'Publisher performed redundant per-object cache-control updates\n' >&2
152+
exit 1
153+
fi
154+
155+
if grep -Fq "Updating cache-control metadata" "$publisher_log"; then
156+
printf 'Publisher retained the redundant cache-control reconciliation phase\n' >&2
157+
exit 1
158+
fi

0 commit comments

Comments
 (0)