|
54 | 54 | EXTRA_GRADLE_ARGS: ${{ github.event.inputs.gradle_args || '' }} |
55 | 55 | SKIP_TESTS: ${{ github.event.inputs.skip_tests || 'false' }} |
56 | 56 | PUBLISH_GITHUB_PACKAGES: ${{ github.event.inputs.publish_github_packages || 'false' }} |
| 57 | + TESTCONTAINERS_RYUK_DISABLED: "true" |
| 58 | + DOCKER_IMAGE_CACHE_DIR: /tmp/bigdata-test-docker-image-cache |
| 59 | + DOCKER_IMAGE_CACHE_FILE: /tmp/bigdata-test-docker-image-cache/spark-matrix-images.tar |
57 | 60 |
|
58 | 61 | steps: |
59 | 62 | - name: Checkout |
|
84 | 87 | docker version |
85 | 88 | docker info |
86 | 89 |
|
| 90 | + - name: Cache Spark matrix Docker images |
| 91 | + if: env.SKIP_TESTS != 'true' |
| 92 | + id: docker-image-cache |
| 93 | + uses: actions/cache@v4 |
| 94 | + with: |
| 95 | + path: ${{ env.DOCKER_IMAGE_CACHE_DIR }} |
| 96 | + key: ${{ runner.os }}-spark-matrix-docker-images-v1-${{ hashFiles('example/spark/src/test/resources/spark-bigdata-test-common.toml') }} |
| 97 | + restore-keys: | |
| 98 | + ${{ runner.os }}-spark-matrix-docker-images-v1- |
| 99 | +
|
| 100 | + - name: Load cached Spark matrix Docker images |
| 101 | + if: env.SKIP_TESTS != 'true' |
| 102 | + run: | |
| 103 | + if [ -s "$DOCKER_IMAGE_CACHE_FILE" ]; then |
| 104 | + docker load --input "$DOCKER_IMAGE_CACHE_FILE" |
| 105 | + docker images |
| 106 | + else |
| 107 | + echo "No cached Docker image archive found at $DOCKER_IMAGE_CACHE_FILE" |
| 108 | + fi |
| 109 | +
|
| 110 | + - name: Pull Spark matrix Docker images |
| 111 | + if: env.SKIP_TESTS != 'true' |
| 112 | + run: | |
| 113 | + set -euo pipefail |
| 114 | +
|
| 115 | + pull_with_retry() { |
| 116 | + local image="$1" |
| 117 | + local attempt |
| 118 | + for attempt in 1 2 3 4 5; do |
| 119 | + if docker pull "$image"; then |
| 120 | + return 0 |
| 121 | + fi |
| 122 | + local sleep_seconds=$((attempt * 15)) |
| 123 | + echo "Pull failed for ${image}; retrying in ${sleep_seconds}s (${attempt}/5)" |
| 124 | + sleep "$sleep_seconds" |
| 125 | + done |
| 126 | + docker pull "$image" |
| 127 | + } |
| 128 | +
|
| 129 | + images=( |
| 130 | + "apache/hadoop:3.5.0" |
| 131 | + "apache/kafka:4.1.2" |
| 132 | + "confluentinc/cp-schema-registry:7.8.0" |
| 133 | + "fsouza/fake-gcs-server:1.54" |
| 134 | + "ghcr.io/openprojectx/cloudera-hms:0.1.16" |
| 135 | + "ghcr.io/openprojectx/directory-kerby/kerby-kdc:latest" |
| 136 | + "ghcr.io/openprojectx/hive:3.1.3-hadoop-3.4.2-gcs-4.0.4-jdk17-0.1.4" |
| 137 | + "localstack/localstack:4.14.0" |
| 138 | + "postgres:16-alpine" |
| 139 | + ) |
| 140 | +
|
| 141 | + for image in "${images[@]}"; do |
| 142 | + pull_with_retry "$image" |
| 143 | + done |
| 144 | +
|
| 145 | + - name: Save Spark matrix Docker images |
| 146 | + if: env.SKIP_TESTS != 'true' |
| 147 | + run: | |
| 148 | + set -euo pipefail |
| 149 | + mkdir -p "$DOCKER_IMAGE_CACHE_DIR" |
| 150 | + docker save --output "$DOCKER_IMAGE_CACHE_FILE" \ |
| 151 | + "apache/hadoop:3.5.0" \ |
| 152 | + "apache/kafka:4.1.2" \ |
| 153 | + "confluentinc/cp-schema-registry:7.8.0" \ |
| 154 | + "fsouza/fake-gcs-server:1.54" \ |
| 155 | + "ghcr.io/openprojectx/cloudera-hms:0.1.16" \ |
| 156 | + "ghcr.io/openprojectx/directory-kerby/kerby-kdc:latest" \ |
| 157 | + "ghcr.io/openprojectx/hive:3.1.3-hadoop-3.4.2-gcs-4.0.4-jdk17-0.1.4" \ |
| 158 | + "localstack/localstack:4.14.0" \ |
| 159 | + "postgres:16-alpine" |
| 160 | + ls -lh "$DOCKER_IMAGE_CACHE_FILE" |
| 161 | +
|
87 | 162 | - name: Run Spark big data matrix tests |
88 | 163 | if: env.SKIP_TESTS != 'true' |
89 | 164 | run: | |
|
0 commit comments