feat(extensions): add support for custom JCEKS aliases in S3JceksExte… #122
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Release | |
| on: | |
| push: | |
| branches: | |
| - master | |
| workflow_dispatch: | |
| inputs: | |
| gradle_args: | |
| description: "Extra Gradle args for the release run, for example: --stacktrace" | |
| required: false | |
| default: "" | |
| type: string | |
| gradle_log_level: | |
| description: "Gradle log level used by workflow Gradle invocations" | |
| required: false | |
| default: info | |
| type: choice | |
| options: | |
| - quiet | |
| - warn | |
| - lifecycle | |
| - info | |
| - debug | |
| skip_tests: | |
| description: "Skip Docker verification and all test tasks, and run the release directly" | |
| required: false | |
| default: false | |
| type: boolean | |
| run_spark_matrix_tests: | |
| description: "Run Spark big data matrix tests" | |
| required: false | |
| default: true | |
| type: boolean | |
| run_spring_gradle_plugin_smoke: | |
| description: "Run Spring Gradle plugin bootRunLocal API smoke test" | |
| required: false | |
| default: true | |
| type: boolean | |
| run_release_task_tests: | |
| description: "Allow tests to run inside the Gradle release task" | |
| required: false | |
| default: true | |
| type: boolean | |
| publish_github_packages: | |
| description: "Also publish artifacts to GitHub Packages. Disabled by default because GitHub Packages can return checksum upload conflicts." | |
| required: false | |
| default: false | |
| type: boolean | |
| enable_docker_image_cache: | |
| description: "Cache Testcontainers Docker images as a tar archive. Disabled by default because save/load can be slower than pulling." | |
| required: false | |
| default: false | |
| type: boolean | |
| permissions: | |
| contents: write | |
| packages: write | |
| concurrency: | |
| group: release-master | |
| cancel-in-progress: false | |
| jobs: | |
| release: | |
| if: >- | |
| github.event_name == 'workflow_dispatch' || | |
| ( | |
| !contains(github.event.head_commit.message, '[Gradle Release Plugin]') && | |
| !contains(github.event.head_commit.message, '[skip ci]') && | |
| !contains(github.event.head_commit.message, '[ci skip]') && | |
| !contains(github.event.head_commit.message, '[no ci]') && | |
| !contains(github.event.head_commit.message, '[skip actions]') && | |
| !contains(github.event.head_commit.message, '[actions skip]') && | |
| !contains(github.event.head_commit.message, 'skip-checks:true') | |
| ) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| env: | |
| GRADLE_OPTS: >- | |
| -Dorg.gradle.daemon=false | |
| -Dorg.gradle.internal.http.connectionTimeout=120000 | |
| -Dorg.gradle.internal.http.socketTimeout=300000 | |
| EXTRA_GRADLE_ARGS: ${{ github.event.inputs.gradle_args || '' }} | |
| GRADLE_LOG_LEVEL: ${{ github.event.inputs.gradle_log_level || 'info' }} | |
| SKIP_TESTS: ${{ github.event.inputs.skip_tests || 'false' }} | |
| RUN_SPARK_MATRIX_TESTS: ${{ github.event_name != 'workflow_dispatch' || inputs.run_spark_matrix_tests }} | |
| RUN_SPRING_GRADLE_PLUGIN_SMOKE: ${{ github.event_name != 'workflow_dispatch' || inputs.run_spring_gradle_plugin_smoke }} | |
| RUN_RELEASE_TASK_TESTS: ${{ github.event_name != 'workflow_dispatch' || inputs.run_release_task_tests }} | |
| PUBLISH_GITHUB_PACKAGES: ${{ github.event.inputs.publish_github_packages || 'false' }} | |
| ENABLE_DOCKER_IMAGE_CACHE: ${{ github.event.inputs.enable_docker_image_cache || 'false' }} | |
| TESTCONTAINERS_RYUK_DISABLED: "true" | |
| DOCKER_IMAGE_CACHE_DIR: /tmp/bigdata-test-docker-image-cache | |
| DOCKER_IMAGE_CACHE_FILE: /tmp/bigdata-test-docker-image-cache/spark-matrix-images.tar | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| token: ${{ secrets.RELEASE_GITHUB_TOKEN }} | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: '17' | |
| - name: Set up Gradle | |
| uses: gradle/actions/setup-gradle@v4 | |
| with: | |
| cache-read-only: false | |
| - name: Configure git | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| - name: Verify Docker environment | |
| if: >- | |
| env.SKIP_TESTS != 'true' && | |
| ( | |
| env.RUN_SPARK_MATRIX_TESTS == 'true' || | |
| env.RUN_SPRING_GRADLE_PLUGIN_SMOKE == 'true' || | |
| env.RUN_RELEASE_TASK_TESTS == 'true' | |
| ) | |
| run: | | |
| docker version | |
| docker info | |
| - name: Cache Spark matrix Docker images | |
| if: env.SKIP_TESTS != 'true' && env.RUN_SPARK_MATRIX_TESTS == 'true' && env.ENABLE_DOCKER_IMAGE_CACHE == 'true' | |
| id: docker-image-cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.DOCKER_IMAGE_CACHE_DIR }} | |
| key: ${{ runner.os }}-spark-matrix-docker-images-v1-${{ hashFiles('example/spark/src/test/resources/spark-bigdata-test-common.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-spark-matrix-docker-images-v1- | |
| - name: Load cached Spark matrix Docker images | |
| if: env.SKIP_TESTS != 'true' && env.RUN_SPARK_MATRIX_TESTS == 'true' && env.ENABLE_DOCKER_IMAGE_CACHE == 'true' | |
| run: | | |
| if [ -s "$DOCKER_IMAGE_CACHE_FILE" ]; then | |
| docker load --input "$DOCKER_IMAGE_CACHE_FILE" | |
| docker images | |
| else | |
| echo "No cached Docker image archive found at $DOCKER_IMAGE_CACHE_FILE" | |
| fi | |
| - name: Pull Spark matrix Docker images | |
| if: env.SKIP_TESTS != 'true' && env.RUN_SPARK_MATRIX_TESTS == 'true' | |
| run: | | |
| set -euo pipefail | |
| pull_with_retry() { | |
| local image="$1" | |
| local attempt | |
| for attempt in 1 2 3 4 5; do | |
| if docker pull "$image"; then | |
| return 0 | |
| fi | |
| local sleep_seconds=$((attempt * 15)) | |
| echo "Pull failed for ${image}; retrying in ${sleep_seconds}s (${attempt}/5)" | |
| sleep "$sleep_seconds" | |
| done | |
| docker pull "$image" | |
| } | |
| images=( | |
| "apache/hadoop:3.5.0" | |
| "apache/kafka:4.1.2" | |
| "confluentinc/cp-schema-registry:7.8.0" | |
| "fsouza/fake-gcs-server:1.54" | |
| "ghcr.io/openprojectx/cloudera-hms:0.1.16" | |
| "ghcr.io/openprojectx/directory-kerby/kerby-kdc:latest" | |
| "ghcr.io/openprojectx/hive:3.1.3-hadoop-3.4.2-gcs-4.0.4-jdk17-0.1.4" | |
| "localstack/localstack:4.14.0" | |
| "postgres:16-alpine" | |
| ) | |
| for image in "${images[@]}"; do | |
| pull_with_retry "$image" | |
| done | |
| - name: Save Spark matrix Docker images | |
| if: env.SKIP_TESTS != 'true' && env.RUN_SPARK_MATRIX_TESTS == 'true' && env.ENABLE_DOCKER_IMAGE_CACHE == 'true' | |
| run: | | |
| set -euo pipefail | |
| mkdir -p "$DOCKER_IMAGE_CACHE_DIR" | |
| docker save --output "$DOCKER_IMAGE_CACHE_FILE" \ | |
| "apache/hadoop:3.5.0" \ | |
| "apache/kafka:4.1.2" \ | |
| "confluentinc/cp-schema-registry:7.8.0" \ | |
| "fsouza/fake-gcs-server:1.54" \ | |
| "ghcr.io/openprojectx/cloudera-hms:0.1.16" \ | |
| "ghcr.io/openprojectx/directory-kerby/kerby-kdc:latest" \ | |
| "ghcr.io/openprojectx/hive:3.1.3-hadoop-3.4.2-gcs-4.0.4-jdk17-0.1.4" \ | |
| "localstack/localstack:4.14.0" \ | |
| "postgres:16-alpine" | |
| ls -lh "$DOCKER_IMAGE_CACHE_FILE" | |
| - name: Run Spark big data matrix tests | |
| if: env.SKIP_TESTS != 'true' && env.RUN_SPARK_MATRIX_TESTS == 'true' | |
| run: | | |
| ./gradlew --no-configuration-cache :example:spark:sparkBigDataMatrixTest --stacktrace "--$GRADLE_LOG_LEVEL" | |
| - name: Publish local artifacts for standalone examples | |
| if: env.SKIP_TESTS != 'true' && env.RUN_SPRING_GRADLE_PLUGIN_SMOKE == 'true' | |
| run: | | |
| ./gradlew --no-configuration-cache publishToMavenLocal -x test -x sparkBigDataMatrixTest --stacktrace "--$GRADLE_LOG_LEVEL" | |
| - name: Run Spring Gradle plugin API smoke test | |
| if: env.SKIP_TESTS != 'true' && env.RUN_SPRING_GRADLE_PLUGIN_SMOKE == 'true' | |
| run: | | |
| set -euo pipefail | |
| release_version="$(awk -F= '/^[[:space:]]*version[[:space:]]*=/{gsub(/^[[:space:]]+|[[:space:]]+$/, "", $2); print $2; exit}' gradle.properties)" | |
| log_dir="$PWD/build/spring-gradle-plugin-ci" | |
| log_file="${log_dir}/bootRunLocal.log" | |
| mkdir -p "$log_dir" | |
| cleanup() { | |
| if [ -n "${boot_pid:-}" ] && kill -0 "$boot_pid" 2>/dev/null; then | |
| kill "$boot_pid" 2>/dev/null || true | |
| wait "$boot_pid" 2>/dev/null || true | |
| fi | |
| if [ -f "$log_file" ]; then | |
| echo "::group::Spring Gradle plugin bootRunLocal log" | |
| tail -n 400 "$log_file" | |
| echo "::endgroup::" | |
| fi | |
| } | |
| trap cleanup EXIT | |
| ( | |
| cd example/spring-gradle-plugin | |
| BIGDATA_TEST_PLUGIN_VERSION="$release_version" ../../gradlew bootRunLocal --stacktrace "--$GRADLE_LOG_LEVEL" | |
| ) > "$log_file" 2>&1 & | |
| boot_pid=$! | |
| for attempt in $(seq 1 120); do | |
| if ! kill -0 "$boot_pid" 2>/dev/null; then | |
| echo "bootRunLocal exited before the API became ready" >&2 | |
| exit 1 | |
| fi | |
| if curl -fsS http://127.0.0.1:8080/api/s3/objects >/tmp/spring-gradle-plugin-list.json 2>/dev/null; then | |
| break | |
| fi | |
| sleep 2 | |
| done | |
| curl -fsS -X PUT \ | |
| -H 'Content-Type: text/plain' \ | |
| --data 'hello from spring gradle plugin ci' \ | |
| http://127.0.0.1:8080/api/s3/objects/ci-smoke.txt \ | |
| | tee /tmp/spring-gradle-plugin-put.json | |
| curl -fsS http://127.0.0.1:8080/api/s3/objects/ci-smoke.txt \ | |
| | tee /tmp/spring-gradle-plugin-get.txt | |
| grep -F 'hello from spring gradle plugin ci' /tmp/spring-gradle-plugin-get.txt | |
| curl -fsS http://127.0.0.1:8080/api/s3/objects \ | |
| | tee /tmp/spring-gradle-plugin-list-after-put.json | |
| grep -F 'ci-smoke.txt' /tmp/spring-gradle-plugin-list-after-put.json | |
| curl -fsS -X DELETE http://127.0.0.1:8080/api/s3/objects/ci-smoke.txt \ | |
| | tee /tmp/spring-gradle-plugin-delete.json | |
| grep -F '"deleted":true' /tmp/spring-gradle-plugin-delete.json | |
| - name: Collect and print Spark diagnostics on failure | |
| if: failure() && env.SKIP_TESTS != 'true' && env.RUN_SPARK_MATRIX_TESTS == 'true' | |
| run: | | |
| set +e | |
| diagnostics_dir="build/github-diagnostics" | |
| mkdir -p "${diagnostics_dir}/docker" "${diagnostics_dir}/spark" | |
| echo "::group::Docker containers" | |
| docker ps -a | tee "${diagnostics_dir}/docker/containers.txt" | |
| echo "::endgroup::" | |
| echo "::group::Docker images" | |
| docker images | tee "${diagnostics_dir}/docker/images.txt" | |
| echo "::endgroup::" | |
| mapfile -t container_ids < <(docker ps -aq) | |
| for container_id in "${container_ids[@]}"; do | |
| container_name="$(docker inspect --format '{{.Name}}' "${container_id}" 2>/dev/null | sed 's#^/##')" | |
| if [ -z "${container_name}" ]; then | |
| container_name="${container_id}" | |
| fi | |
| safe_name="$(printf '%s' "${container_name}" | tr -c 'A-Za-z0-9_.-' '_')" | |
| docker inspect "${container_id}" > "${diagnostics_dir}/docker/${safe_name}.inspect.json" 2>&1 | |
| docker logs "${container_id}" > "${diagnostics_dir}/docker/${safe_name}.stdout-stderr.log" 2>&1 | |
| done | |
| echo "::group::Spark container log files" | |
| if [ -d example/spark/build/bigdata-test-container-logs ]; then | |
| find example/spark/build/bigdata-test-container-logs -type f -print | sort | |
| else | |
| echo "No Spark container log directory found." | |
| fi | |
| echo "::endgroup::" | |
| if [ -d example/spark/build/bigdata-test-container-logs ]; then | |
| while IFS= read -r log_file; do | |
| echo "::group::${log_file}" | |
| sed -n '1,400p' "${log_file}" | |
| line_count="$(wc -l < "${log_file}")" | |
| if [ "${line_count}" -gt 400 ]; then | |
| echo "... truncated first 400 lines from ${line_count} total lines ..." | |
| echo "--- tail ---" | |
| tail -n 400 "${log_file}" | |
| fi | |
| echo "::endgroup::" | |
| done < <(find example/spark/build/bigdata-test-container-logs -type f -print | sort) | |
| cp -a example/spark/build/bigdata-test-container-logs "${diagnostics_dir}/spark/container-logs" | |
| fi | |
| echo "::group::Spark test XML results" | |
| if [ -d example/spark/build/test-results ]; then | |
| while IFS= read -r result_file; do | |
| echo "--- ${result_file} ---" | |
| sed -n '1,240p' "${result_file}" | |
| done < <(find example/spark/build/test-results -type f -name '*.xml' -print | sort) | |
| else | |
| echo "No Spark test result directory found." | |
| fi | |
| echo "::endgroup::" | |
| cp -a example/spark/build/test-results "${diagnostics_dir}/spark/test-results" 2>/dev/null || true | |
| cp -a example/spark/build/reports/tests "${diagnostics_dir}/spark/test-reports" 2>/dev/null || true | |
| - name: Upload test reports and container logs | |
| if: >- | |
| failure() && | |
| env.SKIP_TESTS != 'true' && | |
| ( | |
| env.RUN_SPARK_MATRIX_TESTS == 'true' || | |
| env.RUN_SPRING_GRADLE_PLUGIN_SMOKE == 'true' || | |
| env.RUN_RELEASE_TASK_TESTS == 'true' | |
| ) | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-diagnostics-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: | | |
| **/build/reports/tests/ | |
| **/build/test-results/ | |
| **/build/reports/problems/ | |
| **/build/bigdata-test-container-logs/ | |
| build/spring-gradle-plugin-ci/ | |
| build/github-diagnostics/ | |
| if-no-files-found: ignore | |
| retention-days: 14 | |
| - name: Write signing key | |
| env: | |
| SIGNING_KEY_ASC: ${{ secrets.SIGNING_KEY_ASC }} | |
| run: | | |
| printf '%s' "$SIGNING_KEY_ASC" > "$RUNNER_TEMP/signing-key.asc" | |
| - name: Capture release metadata | |
| run: | | |
| release_version="$(awk -F= '/^[[:space:]]*version[[:space:]]*=/{gsub(/^[[:space:]]+|[[:space:]]+$/, "", $2); print $2; exit}' gradle.properties)" | |
| if [ -z "$release_version" ]; then | |
| echo "Could not read version from gradle.properties" >&2 | |
| exit 1 | |
| fi | |
| release_version="${release_version%-SNAPSHOT}" | |
| echo "RELEASE_VERSION=${release_version}" >> "$GITHUB_ENV" | |
| echo "GITHUB_MAVEN_REPOSITORY=https://maven.pkg.github.com/OpenProjectX/bigdata-test" >> "$GITHUB_ENV" | |
| - name: Run release | |
| id: run_release | |
| timeout-minutes: 90 | |
| env: | |
| OSSRH_USERNAME: ${{ secrets.OSSRH_USERNAME }} | |
| OSSRH_PASSWORD: ${{ secrets.OSSRH_PASSWORD }} | |
| GITHUB_PACKAGES_USERNAME: ${{ github.actor }} | |
| GITHUB_PACKAGES_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| SIGNING_KEY_FILE: ${{ runner.temp }}/signing-key.asc | |
| SIGNING_KEY_PASSWORD: ${{ secrets.SIGNING_KEY_PASSWORD }} | |
| run: | | |
| skip_test_args=() | |
| spark_test_excludes=( | |
| -x sparkBigDataMatrixTest | |
| -x sparkApacheDepsApacheHmsTest | |
| -x sparkApacheDepsApacheHmsKerberosTest | |
| -x sparkApacheDepsClouderaHmsTest | |
| -x sparkApacheDepsClouderaHmsKerberosTest | |
| -x sparkClouderaDepsApacheHmsTest | |
| -x sparkClouderaDepsApacheHmsKerberosTest | |
| -x sparkClouderaDepsClouderaHmsTest | |
| -x sparkClouderaDepsClouderaHmsKerberosTest | |
| ) | |
| if [ "$SKIP_TESTS" = "true" ] || [ "$RUN_RELEASE_TASK_TESTS" != "true" ]; then | |
| skip_test_args=(-x test "${spark_test_excludes[@]}") | |
| elif [ "$RUN_SPARK_MATRIX_TESTS" != "true" ]; then | |
| skip_test_args=("${spark_test_excludes[@]}") | |
| fi | |
| ./gradlew --no-configuration-cache release --stacktrace "--$GRADLE_LOG_LEVEL" "${skip_test_args[@]}" $EXTRA_GRADLE_ARGS | |
| - name: Write release publish summary | |
| if: always() && env.RELEASE_VERSION != '' | |
| env: | |
| RELEASE_OUTCOME: ${{ steps.run_release.outcome }} | |
| run: | | |
| group_id="org.openprojectx.bigdata.test.core" | |
| version="${RELEASE_VERSION}" | |
| github_repo_url="${GITHUB_MAVEN_REPOSITORY}" | |
| artifacts=( | |
| "core" | |
| "junit5" | |
| "extensions" | |
| "bigdata-test-spring-boot-autoconfigure" | |
| "bigdata-test-spring-boot-starter" | |
| ) | |
| { | |
| if [ "$RELEASE_OUTCOME" = "success" ]; then | |
| echo "## Release publish summary" | |
| else | |
| echo "## Release publish attempt" | |
| fi | |
| echo | |
| echo "- Version: \`${version}\`" | |
| echo "- Release result: \`${RELEASE_OUTCOME}\`" | |
| echo "- Maven group: \`${group_id}\`" | |
| echo "- GitHub Maven repository: \`${github_repo_url}\`" | |
| echo "- GitHub Packages index: https://github.com/orgs/OpenProjectX/packages?ecosystem=maven" | |
| echo | |
| if [ "$RELEASE_OUTCOME" != "success" ]; then | |
| echo "> The release step did not complete successfully. The artifacts below are the planned publication coordinates; check the Gradle log to confirm whether any partial uploads happened before the failure." | |
| echo | |
| fi | |
| echo "| Artifact | Maven coordinate | GitHub Packages | Maven Central |" | |
| echo "|---|---|---|---|" | |
| for artifact in "${artifacts[@]}"; do | |
| package_name="${group_id}.${artifact}" | |
| coordinate="${group_id}:${artifact}:${version}" | |
| github_package_url="https://github.com/orgs/OpenProjectX/packages/maven/${package_name}" | |
| central_url="https://central.sonatype.com/artifact/${group_id}/${artifact}/${version}" | |
| echo "| \`${artifact}\` | \`${coordinate}\` | [package](${github_package_url}) | [artifact](${central_url}) |" | |
| done | |
| echo | |
| echo "> Maven Central search can lag for a short time after Sonatype release completes." | |
| echo | |
| echo "### GitHub Packages repository" | |
| echo | |
| echo '```kotlin' | |
| echo 'maven {' | |
| echo " url = uri(\"${github_repo_url}\")" | |
| echo ' credentials {' | |
| echo ' username = System.getenv("GITHUB_ACTOR")' | |
| echo ' password = System.getenv("GITHUB_TOKEN")' | |
| echo ' }' | |
| echo '}' | |
| echo '```' | |
| } >> "$GITHUB_STEP_SUMMARY" |