Skip to content

Commit dae5e41

Browse files
CoderYellowCoderYellow
committed
Add production Jib image and release workflow
1 parent ac45a2f commit dae5e41

8 files changed

Lines changed: 456 additions & 4 deletions

File tree

‎.github/workflows/release.yml‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
name: Release
2+
3+
on:
4+
push:
5+
branches:
6+
- master
7+
workflow_dispatch:
8+
inputs:
9+
gradle_args:
10+
description: "Extra Gradle args for the release run, for example: --stacktrace"
11+
required: false
12+
default: ""
13+
type: string
14+
gradle_log_level:
15+
description: "Gradle log level used by workflow Gradle invocations"
16+
required: false
17+
default: info
18+
type: choice
19+
options:
20+
- quiet
21+
- warn
22+
- lifecycle
23+
- info
24+
- debug
25+
skip_tests:
26+
description: "Skip verification tasks before release"
27+
required: false
28+
default: false
29+
type: boolean
30+
31+
permissions:
32+
contents: write
33+
packages: write
34+
35+
concurrency:
36+
group: release-master
37+
cancel-in-progress: false
38+
39+
jobs:
40+
release:
41+
if: >-
42+
github.event_name == 'workflow_dispatch' ||
43+
(
44+
!contains(github.event.head_commit.message, '[Gradle Release Plugin]') &&
45+
!contains(github.event.head_commit.message, '[skip ci]') &&
46+
!contains(github.event.head_commit.message, '[ci skip]') &&
47+
!contains(github.event.head_commit.message, '[no ci]') &&
48+
!contains(github.event.head_commit.message, '[skip actions]') &&
49+
!contains(github.event.head_commit.message, '[actions skip]') &&
50+
!contains(github.event.head_commit.message, 'skip-checks:true')
51+
)
52+
runs-on: ubuntu-latest
53+
timeout-minutes: 180
54+
55+
env:
56+
GRADLE_OPTS: >-
57+
-Dorg.gradle.daemon=false
58+
-Dorg.gradle.internal.http.connectionTimeout=120000
59+
-Dorg.gradle.internal.http.socketTimeout=300000
60+
EXTRA_GRADLE_ARGS: ${{ github.event.inputs.gradle_args || '' }}
61+
GRADLE_LOG_LEVEL: ${{ github.event.inputs.gradle_log_level || 'info' }}
62+
SKIP_TESTS: ${{ github.event.inputs.skip_tests || 'false' }}
63+
64+
steps:
65+
- name: Checkout
66+
uses: actions/checkout@v4
67+
with:
68+
fetch-depth: 0
69+
token: ${{ secrets.RELEASE_GITHUB_TOKEN }}
70+
71+
- name: Set up JDK 17
72+
uses: actions/setup-java@v4
73+
with:
74+
distribution: temurin
75+
java-version: '17'
76+
77+
- name: Set up Gradle
78+
uses: gradle/actions/setup-gradle@v4
79+
with:
80+
cache-read-only: false
81+
82+
- name: Cache Jib
83+
uses: actions/cache@v4
84+
with:
85+
path: |
86+
~/.cache/google-cloud-tools-java/jib
87+
~/.gradle/caches/jib
88+
key: jib-${{ runner.os }}-${{ hashFiles('app/build.gradle.kts', 'build.gradle.kts', 'gradle.properties') }}
89+
restore-keys: |
90+
jib-${{ runner.os }}-
91+
92+
- name: Configure git
93+
run: |
94+
git config user.name "github-actions[bot]"
95+
git config user.email "github-actions[bot]@users.noreply.github.com"
96+
97+
- name: Verify build
98+
if: env.SKIP_TESTS != 'true'
99+
run: |
100+
./gradlew --no-configuration-cache check --stacktrace "--$GRADLE_LOG_LEVEL"
101+
102+
- name: Write signing key
103+
env:
104+
SIGNING_KEY_ASC: ${{ secrets.SIGNING_KEY_ASC }}
105+
run: |
106+
printf '%s' "$SIGNING_KEY_ASC" > "$RUNNER_TEMP/signing-key.asc"
107+
108+
- name: Capture release metadata
109+
run: |
110+
release_version="$(awk -F= '/^[[:space:]]*version[[:space:]]*=/{gsub(/^[[:space:]]+|[[:space:]]+$/, "", $2); print $2; exit}' gradle.properties)"
111+
if [ -z "$release_version" ]; then
112+
echo "Could not read version from gradle.properties" >&2
113+
exit 1
114+
fi
115+
release_version="${release_version%-SNAPSHOT}"
116+
echo "RELEASE_VERSION=${release_version}" >> "$GITHUB_ENV"
117+
118+
- name: Run release
119+
id: run_release
120+
timeout-minutes: 150
121+
env:
122+
OSSRH_USERNAME: ${{ secrets.OSSRH_USERNAME }}
123+
OSSRH_PASSWORD: ${{ secrets.OSSRH_PASSWORD }}
124+
SIGNING_KEY_FILE: ${{ runner.temp }}/signing-key.asc
125+
SIGNING_KEY_PASSWORD: ${{ secrets.SIGNING_KEY_PASSWORD }}
126+
JIB_TO_IMAGE: ghcr.io/openprojectx/yarn-log-api
127+
JIB_TO_USERNAME: ${{ github.actor }}
128+
JIB_TO_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
129+
run: |
130+
skip_test_args=()
131+
if [ "$SKIP_TESTS" = "true" ]; then
132+
skip_test_args=(-x test)
133+
fi
134+
./gradlew --no-configuration-cache release --stacktrace "--$GRADLE_LOG_LEVEL" "${skip_test_args[@]}" $EXTRA_GRADLE_ARGS
135+
136+
- name: Upload diagnostics
137+
if: failure()
138+
uses: actions/upload-artifact@v4
139+
with:
140+
name: release-diagnostics-${{ github.run_id }}-${{ github.run_attempt }}
141+
path: |
142+
**/build/reports/
143+
**/build/test-results/
144+
if-no-files-found: ignore
145+
retention-days: 14
146+
147+
- name: Write release publish summary
148+
if: always() && env.RELEASE_VERSION != ''
149+
env:
150+
RELEASE_OUTCOME: ${{ steps.run_release.outcome }}
151+
run: |
152+
group_id="org.openprojectx.hadoop.yarn.log.api"
153+
version="${RELEASE_VERSION}"
154+
artifacts=(
155+
"core"
156+
"yarn-log-api-spring-boot-autoconfigure"
157+
"yarn-log-api-spring-boot-starter"
158+
)
159+
160+
{
161+
if [ "$RELEASE_OUTCOME" = "success" ]; then
162+
echo "## Release publish summary"
163+
else
164+
echo "## Release publish attempt"
165+
fi
166+
echo
167+
echo "- Version: \`${version}\`"
168+
echo "- Release result: \`${RELEASE_OUTCOME}\`"
169+
echo "- Maven group: \`${group_id}\`"
170+
echo "- Container image: \`ghcr.io/openprojectx/yarn-log-api:${version}\`"
171+
echo
172+
if [ "$RELEASE_OUTCOME" != "success" ]; then
173+
echo "> The release step did not complete successfully. The artifacts below are the planned Maven Central coordinates; check the Gradle log to confirm whether any partial uploads happened before the failure."
174+
echo
175+
fi
176+
echo "| Artifact | Maven coordinate | Maven Central |"
177+
echo "|---|---|---|"
178+
for artifact in "${artifacts[@]}"; do
179+
coordinate="${group_id}:${artifact}:${version}"
180+
central_url="https://central.sonatype.com/artifact/${group_id}/${artifact}/${version}"
181+
echo "| \`${artifact}\` | \`${coordinate}\` | [artifact](${central_url}) |"
182+
done
183+
echo
184+
echo "> Maven Central search can lag for a short time after Sonatype release completes."
185+
} >> "$GITHUB_STEP_SUMMARY"

‎README.md‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,3 +35,59 @@ Connect to `/api/v1/yarn/logs` and send:
3535

3636
Provide a `YarnLogAuthorizer` bean to enforce the embedding application's
3737
authorization policy.
38+
39+
## Container image
40+
41+
The `app` module is published as `ghcr.io/openprojectx/yarn-log-api`. It runs as
42+
UID/GID `10001`, listens on port `8080`, uses container-aware JVM memory limits,
43+
and exposes Spring Boot liveness and readiness endpoints.
44+
45+
Run against a non-Kerberos cluster by mounting the Hadoop client configuration:
46+
47+
```bash
48+
docker run --rm -p 8080:8080 \
49+
-e HADOOP_CONF_DIR=/etc/hadoop/conf \
50+
-v /path/to/hadoop-conf:/etc/hadoop/conf:ro \
51+
ghcr.io/openprojectx/yarn-log-api:0.1.0
52+
```
53+
54+
For a Kerberos cluster, also mount a service keytab and provide its principal:
55+
56+
```bash
57+
docker run --rm -p 8080:8080 \
58+
-e HADOOP_CONF_DIR=/etc/hadoop/conf \
59+
-e YARN_LOG_API_KERBEROS_PRINCIPAL=yarn-log-api/host.example.com@EXAMPLE.COM \
60+
-e YARN_LOG_API_KERBEROS_KEYTAB=/etc/security/keytabs/yarn-log-api.keytab \
61+
-v /path/to/hadoop-conf:/etc/hadoop/conf:ro \
62+
-v /path/to/yarn-log-api.keytab:/etc/security/keytabs/yarn-log-api.keytab:ro \
63+
ghcr.io/openprojectx/yarn-log-api:0.1.0
64+
```
65+
66+
The mounted Hadoop directory should contain at least `core-site.xml` and
67+
`yarn-site.xml`; include `hdfs-site.xml` when aggregated logs are stored in
68+
HDFS. The service account needs permission to query YARN and read the configured
69+
remote log directory. Put the API behind your authenticated gateway and provide
70+
a `YarnLogAuthorizer` implementation before exposing application logs to users.
71+
72+
Container probes:
73+
74+
```text
75+
GET /actuator/health/liveness
76+
GET /actuator/health/readiness
77+
```
78+
79+
Build the image into the local Docker daemon:
80+
81+
```bash
82+
./gradlew :app:jibDockerBuild
83+
docker run --rm -p 8080:8080 ghcr.io/openprojectx/yarn-log-api:0.1.0-SNAPSHOT
84+
```
85+
86+
Publish to a registry explicitly:
87+
88+
```bash
89+
JIB_TO_IMAGE=registry.example.com/platform/yarn-log-api \
90+
JIB_TO_USERNAME="$REGISTRY_USER" \
91+
JIB_TO_PASSWORD="$REGISTRY_PASSWORD" \
92+
./gradlew :app:jib
93+
```

‎app/build.gradle.kts‎

Lines changed: 67 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,76 @@
11
plugins {
22
id("buildsrc.convention.kotlin-jvm")
33
id("buildsrc.convention.spring-kotlin")
4+
id("com.google.cloud.tools.jib") version "3.5.4"
45
}
56

67

78
dependencies {
8-
99
implementation(project(":yarn-log-api-spring-boot-starter"))
10+
implementation("org.springframework.boot:spring-boot-starter-actuator")
11+
implementation(libs.hadoopYarnCommon) {
12+
exclude(group = "com.sun.jersey")
13+
exclude(group = "org.slf4j", module = "slf4j-reload4j")
14+
}
15+
implementation(libs.hadoopCommon) {
16+
exclude(group = "org.slf4j", module = "slf4j-reload4j")
17+
}
18+
}
19+
20+
jib {
21+
from {
22+
image = providers.gradleProperty("jibFromImage")
23+
.orElse("eclipse-temurin:17-jre-jammy")
24+
.get()
25+
}
26+
to {
27+
image = providers.gradleProperty("jibToImage")
28+
.orElse(providers.environmentVariable("JIB_TO_IMAGE"))
29+
.orElse("ghcr.io/openprojectx/yarn-log-api")
30+
.get()
31+
tags = setOf(project.version.toString())
1032

11-
}
33+
val username = providers.gradleProperty("jibToUsername")
34+
.orElse(providers.environmentVariable("JIB_TO_USERNAME"))
35+
.orNull
36+
val password = providers.gradleProperty("jibToPassword")
37+
.orElse(providers.environmentVariable("JIB_TO_PASSWORD"))
38+
.orNull
39+
40+
if (!username.isNullOrBlank() && !password.isNullOrBlank()) {
41+
auth {
42+
this.username = username
43+
this.password = password
44+
}
45+
}
46+
}
47+
container {
48+
mainClass = "org.openprojectx.hadoop.yarn.log.api.app.YarnLogApiApplicationKt"
49+
ports = listOf("8080")
50+
user = "10001:10001"
51+
workingDirectory = "/app"
52+
extraClasspath = listOf("/etc/hadoop/conf")
53+
volumes = listOf("/etc/hadoop/conf", "/etc/security/keytabs")
54+
jvmFlags = listOf(
55+
"-XX:InitialRAMPercentage=25.0",
56+
"-XX:MaxRAMPercentage=75.0",
57+
"-XX:+ExitOnOutOfMemoryError",
58+
"-Djava.security.egd=file:/dev/urandom",
59+
)
60+
labels = mapOf(
61+
"org.opencontainers.image.title" to "YARN Log API",
62+
"org.opencontainers.image.description" to "Reactive SSE and WebSocket API for YARN application logs",
63+
"org.opencontainers.image.source" to "https://github.com/OpenProjectX/yarn-log-api",
64+
"org.opencontainers.image.version" to project.version.toString(),
65+
"org.opencontainers.image.licenses" to "Apache-2.0",
66+
)
67+
}
68+
extraDirectories {
69+
paths {
70+
path {
71+
setFrom(file("src/main/jib"))
72+
into = "/"
73+
}
74+
}
75+
}
76+
}

‎app/src/main/jib/etc/group‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
root:x:0:
2+
daemon:x:1:
3+
bin:x:2:
4+
sys:x:3:
5+
adm:x:4:
6+
tty:x:5:
7+
disk:x:6:
8+
lp:x:7:
9+
mail:x:8:
10+
news:x:9:
11+
uucp:x:10:
12+
man:x:12:
13+
proxy:x:13:
14+
kmem:x:15:
15+
dialout:x:20:
16+
fax:x:21:
17+
voice:x:22:
18+
cdrom:x:24:
19+
floppy:x:25:
20+
tape:x:26:
21+
sudo:x:27:
22+
audio:x:29:
23+
dip:x:30:
24+
www-data:x:33:
25+
backup:x:34:
26+
operator:x:37:
27+
list:x:38:
28+
irc:x:39:
29+
src:x:40:
30+
gnats:x:41:
31+
shadow:x:42:
32+
utmp:x:43:
33+
video:x:44:
34+
sasl:x:45:
35+
plugdev:x:46:
36+
staff:x:50:
37+
games:x:60:
38+
users:x:100:
39+
yarn-log-api:x:10001:
40+
nogroup:x:65534:

‎app/src/main/jib/etc/passwd‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
root:x:0:0:root:/root:/bin/bash
2+
daemon:x:1:1:daemon:/usr/sbin:/usr/sbin/nologin
3+
bin:x:2:2:bin:/bin:/usr/sbin/nologin
4+
sys:x:3:3:sys:/dev:/usr/sbin/nologin
5+
sync:x:4:65534:sync:/bin:/bin/sync
6+
games:x:5:60:games:/usr/games:/usr/sbin/nologin
7+
man:x:6:12:man:/var/cache/man:/usr/sbin/nologin
8+
lp:x:7:7:lp:/var/spool/lpd:/usr/sbin/nologin
9+
mail:x:8:8:mail:/var/mail:/usr/sbin/nologin
10+
news:x:9:9:news:/var/spool/news:/usr/sbin/nologin
11+
uucp:x:10:10:uucp:/var/spool/uucp:/usr/sbin/nologin
12+
proxy:x:13:13:proxy:/bin:/usr/sbin/nologin
13+
www-data:x:33:33:www-data:/var/www:/usr/sbin/nologin
14+
backup:x:34:34:backup:/var/backups:/usr/sbin/nologin
15+
list:x:38:38:Mailing List Manager:/var/list:/usr/sbin/nologin
16+
irc:x:39:39:ircd:/run/ircd:/usr/sbin/nologin
17+
gnats:x:41:41:Gnats Bug-Reporting System (admin):/var/lib/gnats:/usr/sbin/nologin
18+
_apt:x:100:65534::/nonexistent:/usr/sbin/nologin
19+
yarn-log-api:x:10001:10001:YARN Log API:/app:/usr/sbin/nologin
20+
nobody:x:65534:65534:nobody:/nonexistent:/usr/sbin/nologin

0 commit comments

Comments
 (0)