Skip to content

Commit 22d0872

Browse files
CoderYellowCoderYellow
committed
Add spark-test-bom module to centralize org.openprojectx dependency management and introduce slim Docker image
1 parent 30d94c9 commit 22d0872

9 files changed

Lines changed: 177 additions & 21 deletions

File tree

‎.github/workflows/build.yml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,3 +64,30 @@ jobs:
6464
labels: ${{ steps.meta.outputs.labels }}
6565
cache-from: type=gha
6666
cache-to: type=gha,mode=max
67+
68+
# Same image repo, tags suffixed with -slim (e.g. latest-slim, <shortsha>-slim).
69+
- name: Docker metadata (slim)
70+
id: meta-slim
71+
uses: docker/metadata-action@v5
72+
with:
73+
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
74+
flavor: |
75+
suffix=-slim,onlatest=true
76+
tags: |
77+
type=ref,event=branch
78+
type=ref,event=tag
79+
type=sha
80+
type=raw,value=${{ steps.vars.outputs.short_sha }}
81+
type=raw,value=latest,enable={{is_default_branch}}
82+
83+
# The slim image only populates the Maven local repo with the org.openprojectx.* artifacts.
84+
- name: Build and push slim image
85+
uses: docker/build-push-action@v6
86+
with:
87+
context: .
88+
file: Dockerfile.slim
89+
push: ${{ github.event_name != 'pull_request' }}
90+
tags: ${{ steps.meta-slim.outputs.tags }}
91+
labels: ${{ steps.meta-slim.outputs.labels }}
92+
cache-from: type=gha,scope=slim
93+
cache-to: type=gha,mode=max,scope=slim

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,3 +135,5 @@ buildNumber.properties
135135

136136
# End of https://www.toptal.com/developers/gitignore/api/maven,jetbrains
137137
/.idea/
138+
/spark-apache/build/
139+
/spark-cloudera/build/

‎Dockerfile.slim‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# Slim image: populate the Maven local repository with ONLY the org.openprojectx.* artifacts this
2+
# project uses (the bigdata-test framework + its transitive openprojectx modules, the java-dns
3+
# agent, and the hadoop-native-loader Maven plugin). No Spark/Hadoop/third-party jars are
4+
# downloaded -- every artifact is fetched with -Dtransitive=false, so this stays small.
5+
#
6+
# The set is discovered dynamically from the project's dependency graph, so it tracks version
7+
# changes automatically (e.g. bumping bigdata-test.version) instead of a hard-coded list.
8+
FROM maven:3.9-eclipse-temurin-17
9+
10+
ARG HTTP_PROXY=""
11+
ARG HTTPS_PROXY=""
12+
ARG NO_PROXY=""
13+
ENV HTTP_PROXY=${HTTP_PROXY} HTTPS_PROXY=${HTTPS_PROXY} NO_PROXY=${NO_PROXY} \
14+
http_proxy=${HTTP_PROXY} https_proxy=${HTTPS_PROXY} no_proxy=${NO_PROXY}
15+
16+
WORKDIR /workspace/spark-test
17+
18+
# The poms are what's needed to resolve the dependency graph (.dockerignore filters ignored files).
19+
COPY . .
20+
21+
RUN set -eux; \
22+
REPOS="central::default::https://repo1.maven.org/maven2,cloudera::default::https://repository.cloudera.com/repository/cloudera-repos/,confluent::default::https://packages.confluent.io/maven/"; \
23+
# 1) Every org.openprojectx.* DEPENDENCY in the graph (direct + transitive openprojectx),
24+
# excluding this project's own modules. Jars only -- no transitives, so no Spark.
25+
mvn -B dependency:tree -Dincludes='org.openprojectx*' \
26+
| grep -oE 'org\.openprojectx[^: ]*:[^: ]+:jar:[0-9][^: ]*' \
27+
| grep -v '^org\.openprojectx\.bigdata\.test\.example:' \
28+
| sort -u > /tmp/op-gavs.txt; \
29+
cat /tmp/op-gavs.txt; \
30+
while IFS=: read -r g a t v; do \
31+
mvn -B dependency:get -Dtransitive=false -DremoteRepositories="$REPOS" -Dartifact="$g:$a:$v:$t"; \
32+
done < /tmp/op-gavs.txt; \
33+
# 2) The org.openprojectx.* Maven PLUGIN (and its native-libs core), version read from the pom.
34+
HNL="$(mvn -B -q help:evaluate -Dexpression=hadoop-native-loader.version -DforceStdout)"; \
35+
mvn -B dependency:get -Dtransitive=false -DremoteRepositories="$REPOS" -Dartifact="org.openprojectx.hadoop.native.loader.core:maven-plugin:$HNL"; \
36+
mvn -B dependency:get -Dtransitive=false -DremoteRepositories="$REPOS" -Dartifact="org.openprojectx.hadoop.native.loader.core:core:$HNL"; \
37+
echo "=== org.openprojectx artifacts in local repo ==="; \
38+
find /root/.m2/repository/org/openprojectx -name '*.jar' | sort
39+
40+
CMD ["bash"]

‎README.md‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ the default test in the original Gradle example.
2525

2626
| Module | Purpose |
2727
|----------------------|-------------------------------------------------------------------------------------------|
28+
| `spark-test-bom` | BOM that manages the `org.openprojectx.*` dependency versions (bigdata-test + java-dns) in one place; the other modules import it. |
2829
| `spark-test-common` | The shared scenario, the `SparkBigDataTestExample` test class, and the test resources (TOML/Avro/log4j2). Holds the test code exactly once. |
2930
| `spark-apache` | Runs the shared test against the **Apache** line (Spark 3.5.7 / Hadoop 3.4.2 / Iceberg 1.11.0). |
3031
| `spark-cloudera` | Runs the shared test against the **Cloudera** line (Spark 3.3.2.3.3.7190.9-1 / Hadoop 3.1.1.7.1.9.14-2 / Iceberg 1.8.1). |
@@ -118,13 +119,28 @@ docker build -t spark-test .
118119
docker build --build-arg HTTPS_PROXY=http://host.docker.internal:10809 -t spark-test .
119120
```
120121

122+
### Slim image ([`Dockerfile.slim`](Dockerfile.slim))
123+
124+
A lightweight image that pre-populates the Maven local repo with **only the `org.openprojectx.*`
125+
artifacts** the project uses — the bigdata-test framework and its transitive openprojectx modules
126+
(`junit5`, `extensions`, `core`, `hive-docker-testcontainers`), the `java-dns` agent, and the
127+
`hadoop-native-loader` Maven plugin (+ its native-libs `core`). It does **not** download Spark,
128+
Hadoop, or any other third-party jars: the set is discovered from the dependency graph and each
129+
artifact is fetched with `-Dtransitive=false`. The list is dynamic, so version bumps (e.g.
130+
`bigdata-test.version`) are picked up automatically.
131+
132+
```bash
133+
docker build -f Dockerfile.slim -t spark-test-openprojectx .
134+
```
135+
121136
## CI / GitHub Container Registry
122137

123138
[`.github/workflows/build.yml`](.github/workflows/build.yml) builds that image (installing the deps
124139
and building the jars in the image build) and publishes it to **GitHub Container Registry**
125-
(`ghcr.io/<owner>/spark-test`) on pushes to the default branch and on `v*` tags. It needs no extra
126-
secrets — it authenticates with the built-in `GITHUB_TOKEN` (`packages: write`). Pull requests build
127-
the image but do not push.
140+
(`ghcr.io/<owner>/spark-test`) on pushes to the default branch and on `v*` tags. It also builds and
141+
pushes the [slim image](#slim-image-dockerfileslim) to the same repo with `-slim`-suffixed tags
142+
(`latest-slim`, `<short-sha>-slim`, …). It needs no extra secrets — it authenticates with the
143+
built-in `GITHUB_TOKEN` (`packages: write`). Pull requests build both images but do not push.
128144

129145
[`.github/workflows/windows.yml`](.github/workflows/windows.yml) runs on `windows-latest` to check
130146
Windows compatibility: it builds every module with Maven and verifies the `hadoop-native-loader`

‎pom.xml‎

Lines changed: 5 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
</description>
1414

1515
<modules>
16+
<module>spark-test-bom</module>
1617
<module>spark-test-common</module>
1718
<module>spark-apache</module>
1819
<module>spark-cloudera</module>
@@ -92,24 +93,10 @@
9293
<version>2.15.2</version>
9394
</dependency>
9495

95-
<!-- bigdata-test framework (local 0.1.1-SNAPSHOT) -->
96-
<dependency>
97-
<groupId>org.openprojectx.bigdata.test.core</groupId>
98-
<artifactId>junit5</artifactId>
99-
<version>${bigdata-test.version}</version>
100-
</dependency>
101-
<dependency>
102-
<groupId>org.openprojectx.bigdata.test.core</groupId>
103-
<artifactId>extensions</artifactId>
104-
<version>${bigdata-test.version}</version>
105-
</dependency>
106-
107-
<!-- java-dns agent jar, attached to the Surefire JVM to redirect GCS DNS -->
108-
<dependency>
109-
<groupId>org.openprojectx.java.dns</groupId>
110-
<artifactId>core</artifactId>
111-
<version>${java-dns.version}</version>
112-
</dependency>
96+
<!-- The org.openprojectx.* dependencies (bigdata-test framework + java-dns agent) are
97+
managed by the spark-test-bom module; the consuming modules import that BOM.
98+
(The parent cannot import it here: the BOM inherits this parent, so a self-import
99+
cycle would result.) -->
113100

114101
<!-- JUnit 5 -->
115102
<dependency>

‎spark-apache/pom.xml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,19 @@
2323
<log4j.slf4j.version>2.20.0</log4j.slf4j.version>
2424
</properties>
2525

26+
<dependencyManagement>
27+
<dependencies>
28+
<!-- Manages the org.openprojectx.* dependency versions. -->
29+
<dependency>
30+
<groupId>org.openprojectx.bigdata.test.example</groupId>
31+
<artifactId>spark-test-bom</artifactId>
32+
<version>${project.version}</version>
33+
<type>pom</type>
34+
<scope>import</scope>
35+
</dependency>
36+
</dependencies>
37+
</dependencyManagement>
38+
2639
<dependencies>
2740
<!-- shared scenario + test class + resources -->
2841
<dependency>

‎spark-cloudera/pom.xml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,19 @@
2323
<log4j.slf4j.version>2.18.0</log4j.slf4j.version>
2424
</properties>
2525

26+
<dependencyManagement>
27+
<dependencies>
28+
<!-- Manages the org.openprojectx.* dependency versions. -->
29+
<dependency>
30+
<groupId>org.openprojectx.bigdata.test.example</groupId>
31+
<artifactId>spark-test-bom</artifactId>
32+
<version>${project.version}</version>
33+
<type>pom</type>
34+
<scope>import</scope>
35+
</dependency>
36+
</dependencies>
37+
</dependencyManagement>
38+
2639
<dependencies>
2740
<!-- shared scenario + test class + resources -->
2841
<dependency>

‎spark-test-bom/pom.xml‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<modelVersion>4.0.0</modelVersion>
6+
7+
<parent>
8+
<groupId>org.openprojectx.bigdata.test.example</groupId>
9+
<artifactId>spark-test-parent</artifactId>
10+
<version>0.1.1-SNAPSHOT</version>
11+
</parent>
12+
13+
<artifactId>spark-test-bom</artifactId>
14+
<packaging>pom</packaging>
15+
<name>spark-test-bom</name>
16+
<description>
17+
BOM that manages the org.openprojectx.* dependencies used by this project (the bigdata-test
18+
framework and the java-dns agent). Modules import this BOM so the openprojectx versions are
19+
declared in exactly one place. Versions come from the parent's bigdata-test.version /
20+
java-dns.version properties.
21+
</description>
22+
23+
<dependencyManagement>
24+
<dependencies>
25+
<!-- bigdata-test framework -->
26+
<dependency>
27+
<groupId>org.openprojectx.bigdata.test.core</groupId>
28+
<artifactId>junit5</artifactId>
29+
<version>${bigdata-test.version}</version>
30+
</dependency>
31+
<dependency>
32+
<groupId>org.openprojectx.bigdata.test.core</groupId>
33+
<artifactId>extensions</artifactId>
34+
<version>${bigdata-test.version}</version>
35+
</dependency>
36+
37+
<!-- java-dns agent jar, attached to the Surefire JVM to redirect GCS DNS -->
38+
<dependency>
39+
<groupId>org.openprojectx.java.dns</groupId>
40+
<artifactId>core</artifactId>
41+
<version>${java-dns.version}</version>
42+
</dependency>
43+
</dependencies>
44+
</dependencyManagement>
45+
</project>

‎spark-test-common/pom.xml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,19 @@
2929
<compile.hadoop.version>3.4.2</compile.hadoop.version>
3030
</properties>
3131

32+
<dependencyManagement>
33+
<dependencies>
34+
<!-- Manages the org.openprojectx.* dependency versions. -->
35+
<dependency>
36+
<groupId>org.openprojectx.bigdata.test.example</groupId>
37+
<artifactId>spark-test-bom</artifactId>
38+
<version>${project.version}</version>
39+
<type>pom</type>
40+
<scope>import</scope>
41+
</dependency>
42+
</dependencies>
43+
</dependencyManagement>
44+
3245
<dependencies>
3346
<dependency>
3447
<groupId>org.openprojectx.bigdata.test.core</groupId>

0 commit comments

Comments
 (0)