Skip to content

Commit 4b27335

Browse files
committed
feat(workflows): pre-pull Docker images for Spark matrix tests and disable Testcontainers Ryuk
- Added a step to pre-pull essential Docker images to enhance test reliability and reduce potential runtime delays. - Disabled the Testcontainers Ryuk sidecar to optimize resource usage during tests.
1 parent 1ee2e9a commit 4b27335

1 file changed

Lines changed: 75 additions & 0 deletions

File tree

.github/workflows/release.yml

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,9 @@ jobs:
5454
EXTRA_GRADLE_ARGS: ${{ github.event.inputs.gradle_args || '' }}
5555
SKIP_TESTS: ${{ github.event.inputs.skip_tests || 'false' }}
5656
PUBLISH_GITHUB_PACKAGES: ${{ github.event.inputs.publish_github_packages || 'false' }}
57+
TESTCONTAINERS_RYUK_DISABLED: "true"
58+
DOCKER_IMAGE_CACHE_DIR: /tmp/bigdata-test-docker-image-cache
59+
DOCKER_IMAGE_CACHE_FILE: /tmp/bigdata-test-docker-image-cache/spark-matrix-images.tar
5760

5861
steps:
5962
- name: Checkout
@@ -84,6 +87,78 @@ jobs:
8487
docker version
8588
docker info
8689
90+
- name: Cache Spark matrix Docker images
91+
if: env.SKIP_TESTS != 'true'
92+
id: docker-image-cache
93+
uses: actions/cache@v4
94+
with:
95+
path: ${{ env.DOCKER_IMAGE_CACHE_DIR }}
96+
key: ${{ runner.os }}-spark-matrix-docker-images-v1-${{ hashFiles('example/spark/src/test/resources/spark-bigdata-test-common.toml') }}
97+
restore-keys: |
98+
${{ runner.os }}-spark-matrix-docker-images-v1-
99+
100+
- name: Load cached Spark matrix Docker images
101+
if: env.SKIP_TESTS != 'true'
102+
run: |
103+
if [ -s "$DOCKER_IMAGE_CACHE_FILE" ]; then
104+
docker load --input "$DOCKER_IMAGE_CACHE_FILE"
105+
docker images
106+
else
107+
echo "No cached Docker image archive found at $DOCKER_IMAGE_CACHE_FILE"
108+
fi
109+
110+
- name: Pull Spark matrix Docker images
111+
if: env.SKIP_TESTS != 'true'
112+
run: |
113+
set -euo pipefail
114+
115+
pull_with_retry() {
116+
local image="$1"
117+
local attempt
118+
for attempt in 1 2 3 4 5; do
119+
if docker pull "$image"; then
120+
return 0
121+
fi
122+
local sleep_seconds=$((attempt * 15))
123+
echo "Pull failed for ${image}; retrying in ${sleep_seconds}s (${attempt}/5)"
124+
sleep "$sleep_seconds"
125+
done
126+
docker pull "$image"
127+
}
128+
129+
images=(
130+
"apache/hadoop:3.5.0"
131+
"apache/kafka:4.1.2"
132+
"confluentinc/cp-schema-registry:7.8.0"
133+
"fsouza/fake-gcs-server:1.54"
134+
"ghcr.io/openprojectx/cloudera-hms:0.1.16"
135+
"ghcr.io/openprojectx/directory-kerby/kerby-kdc:latest"
136+
"ghcr.io/openprojectx/hive:3.1.3-hadoop-3.4.2-gcs-4.0.4-jdk17-0.1.4"
137+
"localstack/localstack:4.14.0"
138+
"postgres:16-alpine"
139+
)
140+
141+
for image in "${images[@]}"; do
142+
pull_with_retry "$image"
143+
done
144+
145+
- name: Save Spark matrix Docker images
146+
if: env.SKIP_TESTS != 'true'
147+
run: |
148+
set -euo pipefail
149+
mkdir -p "$DOCKER_IMAGE_CACHE_DIR"
150+
docker save --output "$DOCKER_IMAGE_CACHE_FILE" \
151+
"apache/hadoop:3.5.0" \
152+
"apache/kafka:4.1.2" \
153+
"confluentinc/cp-schema-registry:7.8.0" \
154+
"fsouza/fake-gcs-server:1.54" \
155+
"ghcr.io/openprojectx/cloudera-hms:0.1.16" \
156+
"ghcr.io/openprojectx/directory-kerby/kerby-kdc:latest" \
157+
"ghcr.io/openprojectx/hive:3.1.3-hadoop-3.4.2-gcs-4.0.4-jdk17-0.1.4" \
158+
"localstack/localstack:4.14.0" \
159+
"postgres:16-alpine"
160+
ls -lh "$DOCKER_IMAGE_CACHE_FILE"
161+
87162
- name: Run Spark big data matrix tests
88163
if: env.SKIP_TESTS != 'true'
89164
run: |

0 commit comments

Comments
 (0)