Skip to content

Commit 757a33e

Browse files
pantherman594pantherman594
authored
Pre fetch devcontainer features to get around ghcr.io rate limiting (#400)
* Pre fetch devcontainer features to get around ghcr.io rate limiting * Shellcheck, fix extract
1 parent 7f42a40 commit 757a33e

4 files changed

Lines changed: 200 additions & 0 deletions

File tree

‎feature-versions/state.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,5 +88,10 @@
8888
"tag": "4",
8989
"installed": "sha256:0cb4a78491b6e62ee8a9bf4fbeacbd15b5013d19bc420591b05383a696315e60",
9090
"filter": "startupscript\\/butane\\/050-parse-devcontainer\\.sh"
91+
},
92+
"gcr.io/go-containerregistry/crane": {
93+
"tag": "latest",
94+
"installed": "sha256:d3a706262093746258f20107ab4e95536f9d6d45c8c3f3acf6b02b1801b440d6",
95+
"filter": "startupscript\\/butane\\/prefetch-oci-features\\.sh"
9196
}
9297
}

‎startupscript/butane/050-parse-devcontainer.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,9 @@ if [[ -d "${DEVCONTAINER_FEATURES_PATH}" ]]; then
7878
rsync -a --ignore-existing "${DEVCONTAINER_FEATURES_PATH}/" "${DEVCONTAINER_PATH}/.devcontainer/features"
7979
fi
8080

81+
/home/core/prefetch-oci-features.sh "${DEVCONTAINER_CONFIG_PATH}" || \
82+
echo "WARNING: prefetch-oci-features.sh failed, continuing with remote features" >&2
83+
8184
replace_template_options() {
8285
local TEMPLATE_PATH="$1"
8386

‎startupscript/butane/prefetch-oci-features.sh‎

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
#!/bin/bash
2+
3+
# prefetch-oci-features.sh pre-downloads OCI devcontainer features to local
4+
# directories, then rewrites devcontainer.json to use local paths. This avoids
5+
# ghcr.io rate limits during devcontainer up.
6+
7+
set -o errexit
8+
set -o nounset
9+
set -o pipefail
10+
set -o xtrace
11+
12+
readonly CRANE_IMAGE="gcr.io/go-containerregistry/crane@sha256:d3a706262093746258f20107ab4e95536f9d6d45c8c3f3acf6b02b1801b440d6"
13+
14+
crane() {
15+
docker run --rm "${CRANE_IMAGE}" "$@"
16+
}
17+
18+
readonly MAX_RETRIES=3
19+
readonly RETRY_DELAY=5
20+
21+
retry() {
22+
local attempt
23+
for attempt in $(seq 1 "${MAX_RETRIES}"); do
24+
if "$@"; then
25+
return 0
26+
fi
27+
if [[ "${attempt}" -lt "${MAX_RETRIES}" ]]; then
28+
echo "Attempt ${attempt}/${MAX_RETRIES} failed, retrying in ${RETRY_DELAY}s..." >&2
29+
sleep "${RETRY_DELAY}"
30+
fi
31+
done
32+
return 1
33+
}
34+
35+
if [[ $# -lt 1 ]]; then
36+
echo "Usage: $0 <path/to/devcontainer.json>" >&2
37+
exit 1
38+
fi
39+
40+
readonly CONFIG_PATH="$1"
41+
42+
# Derive the project root from the config path.
43+
# .devcontainer.json is at the project root; .devcontainer/devcontainer.json is one level down.
44+
CONFIG_DIR="$(cd "$(dirname "${CONFIG_PATH}")" && pwd)"
45+
if [[ "$(basename "${CONFIG_DIR}")" == ".devcontainer" ]]; then
46+
PROJECT_DIR="$(dirname "${CONFIG_DIR}")"
47+
else
48+
PROJECT_DIR="${CONFIG_DIR}"
49+
fi
50+
readonly PROJECT_DIR
51+
readonly FEATURES_DIR="${PROJECT_DIR}/.devcontainer/features"
52+
53+
# ref_to_dir_name converts an OCI feature reference to a namespaced directory
54+
# name, avoiding collisions with local features (e.g., features/src/java).
55+
# ghcr.io/devcontainers/features/java@sha256:... -> devcontainers-features-java
56+
# ghcr.io/rocker-org/devcontainer-features/r-packages:1 -> rocker-org-devcontainer-features-r-packages
57+
ref_to_dir_name() {
58+
local ref="$1"
59+
ref_to_resource "${ref}" \
60+
| sed 's|^[^/]*/||' \
61+
| tr '/' '-'
62+
}
63+
64+
# ref_to_resource extracts the registry/path portion without the version or digest.
65+
# ghcr.io/devcontainers/features/java@sha256:... -> ghcr.io/devcontainers/features/java
66+
# ghcr.io/devcontainers/features/java:1 -> ghcr.io/devcontainers/features/java
67+
ref_to_resource() {
68+
local ref="$1"
69+
echo "${ref}" | sed 's/@sha256:.*//' | sed 's/:[^/]*$//'
70+
}
71+
72+
# Extract ghcr.io feature references that are JSON keys (followed by ":").
73+
FEATURE_REFS=()
74+
while IFS= read -r ref; do
75+
FEATURE_REFS+=("${ref}")
76+
done < <(grep -o '"ghcr\.io/[^"]*"[[:space:]]*:' "${CONFIG_PATH}" | sed 's/[[:space:]]*:$//' | tr -d '"')
77+
78+
if [[ ${#FEATURE_REFS[@]} -eq 0 ]]; then
79+
echo "No OCI feature references found in ${CONFIG_PATH}"
80+
exit 0
81+
fi
82+
83+
mkdir -p "${FEATURES_DIR}"
84+
85+
# Track which OCI refs were successfully prefetched, for installsAfter rewriting.
86+
declare -A PREFETCHED_REFS # OCI resource -> local path
87+
88+
for FEATURE_REF in "${FEATURE_REFS[@]}"; do
89+
DIR_NAME="$(ref_to_dir_name "${FEATURE_REF}")"
90+
LOCAL_DIR="${FEATURES_DIR}/${DIR_NAME}"
91+
RESOURCE="$(ref_to_resource "${FEATURE_REF}")"
92+
93+
if [[ -d "${LOCAL_DIR}" ]] && [[ -f "${LOCAL_DIR}/devcontainer-feature.json" ]]; then
94+
echo "Already exists: ${LOCAL_DIR}, skipping download"
95+
PREFETCHED_REFS["${RESOURCE}"]="./.devcontainer/features/${DIR_NAME}"
96+
continue
97+
fi
98+
99+
echo "Prefetching ${FEATURE_REF} -> ${LOCAL_DIR}"
100+
101+
# For tag-based refs (no @sha256:), resolve to a digest first.
102+
RESOLVED_REF="${FEATURE_REF}"
103+
if [[ "${FEATURE_REF}" != *"@sha256:"* ]]; then
104+
DIGEST="$(retry crane digest "${FEATURE_REF}")" || {
105+
echo "WARNING: Failed to resolve digest for ${FEATURE_REF}, skipping" >&2
106+
continue
107+
}
108+
RESOLVED_REF="${RESOURCE}@${DIGEST}"
109+
fi
110+
111+
MANIFEST="$(retry crane manifest "${RESOLVED_REF}")" || {
112+
echo "WARNING: Failed to fetch manifest for ${RESOLVED_REF}, skipping" >&2
113+
continue
114+
}
115+
116+
LAYER_DIGEST="$(echo "${MANIFEST}" | jq -r '.layers[0].digest')"
117+
if [[ -z "${LAYER_DIGEST}" ]] || [[ "${LAYER_DIGEST}" == "null" ]]; then
118+
echo "WARNING: No layer found in manifest for ${RESOLVED_REF}, skipping" >&2
119+
continue
120+
fi
121+
122+
BLOB_TAR="$(mktemp)"
123+
download_blob() {
124+
crane blob "${RESOURCE}@${LAYER_DIGEST}" > "${BLOB_TAR}"
125+
}
126+
retry download_blob || {
127+
echo "WARNING: Failed to download blob for ${RESOLVED_REF}, skipping" >&2
128+
rm -f "${BLOB_TAR}"
129+
continue
130+
}
131+
132+
mkdir -p "${LOCAL_DIR}"
133+
tar -xf "${BLOB_TAR}" -C "${LOCAL_DIR}" || {
134+
echo "WARNING: Failed to extract blob for ${RESOLVED_REF}, skipping" >&2
135+
rm -f "${BLOB_TAR}"
136+
rm -rf "${LOCAL_DIR}"
137+
continue
138+
}
139+
rm -f "${BLOB_TAR}"
140+
141+
if [[ ! -f "${LOCAL_DIR}/devcontainer-feature.json" ]]; then
142+
echo "WARNING: Downloaded feature missing devcontainer-feature.json, skipping" >&2
143+
rm -rf "${LOCAL_DIR}"
144+
continue
145+
fi
146+
147+
PREFETCHED_REFS["${RESOURCE}"]="./.devcontainer/features/${DIR_NAME}"
148+
echo "Prefetched ${FEATURE_REF} -> ${LOCAL_DIR}"
149+
done
150+
151+
if [[ ${#PREFETCHED_REFS[@]} -eq 0 ]]; then
152+
echo "No features were prefetched"
153+
exit 0
154+
fi
155+
156+
# Rewrite devcontainer.json: replace OCI refs with local paths.
157+
for FEATURE_REF in "${FEATURE_REFS[@]}"; do
158+
RESOURCE="$(ref_to_resource "${FEATURE_REF}")"
159+
LOCAL_PATH="${PREFETCHED_REFS[${RESOURCE}]:-}"
160+
if [[ -z "${LOCAL_PATH}" ]]; then
161+
continue
162+
fi
163+
164+
# shellcheck disable=SC2016 # backslash-ampersand is intentional sed replacement syntax
165+
ESCAPED_REF="$(printf '%s' "${FEATURE_REF}" | sed 's/[[\.*^$()+?{|]/\\&/g')"
166+
sed -i "s|\"${ESCAPED_REF}\"|\"${LOCAL_PATH}\"|g" "${CONFIG_PATH}"
167+
done
168+
169+
# Rewrite installsAfter in all features, not just prefetched ones. Local features
170+
# like workbench-tools may reference OCI features we've prefetched.
171+
for FEATURE_JSON in "${FEATURES_DIR}"/*/devcontainer-feature.json; do
172+
if [[ ! -f "${FEATURE_JSON}" ]]; then
173+
continue
174+
fi
175+
176+
for INSTALL_AFTER_RESOURCE in "${!PREFETCHED_REFS[@]}"; do
177+
INSTALL_AFTER_LOCAL="${PREFETCHED_REFS[${INSTALL_AFTER_RESOURCE}]}"
178+
sed -i "s|\"${INSTALL_AFTER_RESOURCE}\"|\"${INSTALL_AFTER_LOCAL}\"|g" "${FEATURE_JSON}"
179+
done
180+
done
181+
182+
echo "Prefetched ${#PREFETCHED_REFS[@]} of ${#FEATURE_REFS[@]} OCI features"

‎tests/common/build.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,16 @@ popd
5858
mkdir -p "${SRC_DIR}/.devcontainer/features"
5959
rsync -a --ignore-existing "features/src/" "${SRC_DIR}/.devcontainer/features"
6060

61+
############################
62+
# Prefetch OCI features
63+
############################
64+
PREFETCH_SCRIPT="./startupscript/butane/prefetch-oci-features.sh"
65+
if [[ -f "${SRC_DIR}/.devcontainer.json" ]]; then
66+
"${PREFETCH_SCRIPT}" "${SRC_DIR}/.devcontainer.json"
67+
elif [[ -f "${SRC_DIR}/.devcontainer/devcontainer.json" ]]; then
68+
"${PREFETCH_SCRIPT}" "${SRC_DIR}/.devcontainer/devcontainer.json"
69+
fi
70+
6171
############################
6272
# Install Devcontainer CLI
6373
############################

0 commit comments

Comments
 (0)