Skip to content

Commit b2cb855

Browse files
CoderYellowCoderYellow
committed
Upgrade bigdata-test to 0.1.10; enhance Dockerfile.slim for automated dependency discovery and improved artifact resolution.
1 parent 45a69db commit b2cb855

3 files changed

Lines changed: 93 additions & 31 deletions

File tree

‎Dockerfile.slim‎

Lines changed: 68 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,15 @@
33
# project uses (the bigdata-test framework + its transitive openprojectx modules, the java-dns
44
# agent, and the hadoop-native-loader Maven plugin), each with its -sources jar.
55
#
6+
# CONFIG-DRIVEN: the artifact list is NOT hardcoded here. It is discovered by scanning every project
7+
# POM (parent + all modules) for org.openprojectx.* <dependency>/<plugin> entries, with their
8+
# ${...} version placeholders resolved from the parent <properties>. So adding an openprojectx
9+
# dependency to any module pom (e.g. spark-test-bom/pom.xml) or bumping a *.version property flows
10+
# into this image automatically -- no edits to this Dockerfile needed.
11+
#
612
# It does a small breadth-first walk over the openprojectx POMs only (each fetched with
713
# dependency:get -Dtransitive=false), so it never resolves the huge/slow Spark/Hadoop/Cloudera
8-
# graph -- only the openprojectx closure is downloaded. Versions are read from the project pom, so
9-
# version bumps (e.g. bigdata-test.version) are picked up automatically.
14+
# graph -- only the openprojectx closure is downloaded.
1015
FROM maven:3.9-eclipse-temurin-17
1116

1217
ARG HTTP_PROXY=""
@@ -17,52 +22,85 @@ ENV HTTP_PROXY=${HTTP_PROXY} HTTPS_PROXY=${HTTPS_PROXY} NO_PROXY=${NO_PROXY} \
1722

1823
WORKDIR /workspace/spark-test
1924

20-
# Only the project pom is needed -- it holds the org.openprojectx.* versions. Copy it OUTSIDE the
21-
# working directory so Maven does not auto-discover it as a (broken) reactor: its <modules> point at
22-
# child dirs that are not in this build context, which would make every dependency:get fail.
23-
COPY pom.xml /tmp/project-pom.xml
25+
# Copy the project POMs (parent + every module pom) OUTSIDE the working directory, so Maven never
26+
# auto-discovers them as a (broken) reactor -- the modules' sources are not in this build context,
27+
# which would make every dependency:get fail. --parents preserves each pom's path, and the ./*/pom.xml
28+
# glob picks up any module automatically, so new modules need no change here either.
29+
COPY --parents ./pom.xml ./*/pom.xml /tmp/poms/
2430

2531
RUN <<'SCRIPT'
2632
set -eux
2733
REPOS="central::default::https://repo1.maven.org/maven2,cloudera::default::https://repository.cloudera.com/repository/cloudera-repos/,confluent::default::https://packages.confluent.io/maven/"
28-
POM=/tmp/project-pom.xml
29-
BDT="$(grep -oE '<bigdata-test.version>[^<]+' "$POM" | head -1 | sed 's/.*>//')"
30-
JDNS="$(grep -oE '<java-dns.version>[^<]+' "$POM" | head -1 | sed 's/.*>//')"
31-
HNL="$(grep -oE '<hadoop-native-loader.version>[^<]+' "$POM" | head -1 | sed 's/.*>//')"
34+
POMS=/tmp/poms
35+
PARENT="$POMS/pom.xml"
36+
37+
# Build a sed script that resolves ${prop} version placeholders from the parent <properties>
38+
# (e.g. ${bigdata-test.version} -> 0.1.9). Only single-line <key>value</key> properties are taken.
39+
awk '
40+
/<properties>/ { p=1; next }
41+
/<\/properties>/ { p=0 }
42+
p && match($0, /<[A-Za-z0-9._-]+>[^<]+<\//) {
43+
line=$0; sub(/^[ \t]*</,"",line)
44+
key=line; sub(/>.*/,"",key)
45+
val=line; sub(/^[^>]*>/,"",val); sub(/<.*/,"",val)
46+
gsub(/[&|\\]/,"\\\\&",val)
47+
print "s|\\${" key "}|" val "|g"
48+
}
49+
' "$PARENT" > /tmp/props.sed
3250

33-
# Extracts org.openprojectx.* dependencies (with literal versions) from a POM.
51+
# Extracts org.openprojectx.* coordinates from <dependency> and <plugin> blocks of a POM, skipping
52+
# this project's own reactor modules (groupId org.openprojectx.bigdata.test.example), which are
53+
# built locally rather than fetched. Versions may still contain ${...}; they are resolved downstream.
3454
cat > /tmp/op-deps.awk <<'AWK'
35-
/<dependency>/ { g=ar=v=""; ind=1; next }
36-
/<\/dependency>/ { if (ind && g ~ /^org\.openprojectx/ && v ~ /^[0-9]/) print g":"ar":"v; ind=0; next }
37-
ind && /<groupId>/ { t=$0; gsub(/.*<groupId>|<\/groupId>.*/,"",t); g=t }
38-
ind && /<artifactId>/{ t=$0; gsub(/.*<artifactId>|<\/artifactId>.*/,"",t); ar=t }
39-
ind && /<version>/ { t=$0; gsub(/.*<version>|<\/version>.*/,"",t); v=t }
55+
function flush() {
56+
if (depth && g ~ /^org\.openprojectx/ && g !~ /\.example$/ && a != "" && v != "")
57+
print g ":" a ":" v
58+
depth=0; g=a=v=""
59+
}
60+
/<dependency>|<plugin>/ { g=a=v=""; depth=1; next }
61+
/<\/dependency>|<\/plugin>/ { flush(); next }
62+
depth && /<groupId>/ { t=$0; gsub(/.*<groupId>|<\/groupId>.*/,"",t); g=t }
63+
depth && /<artifactId>/ { t=$0; gsub(/.*<artifactId>|<\/artifactId>.*/,"",t); a=t }
64+
depth && /<version>/ { t=$0; gsub(/.*<version>|<\/version>.*/,"",t); v=t }
4065
AWK
4166

42-
# Seed the queue with the openprojectx entry points (deps + the Maven plugin and its core).
43-
printf '%s\n' \
44-
"org.openprojectx.bigdata.test.core:junit5:$BDT" \
45-
"org.openprojectx.bigdata.test.core:extensions:$BDT" \
46-
"org.openprojectx.java.dns:core:$JDNS" \
47-
"org.openprojectx.hadoop.native.loader.core:maven-plugin:$HNL" \
48-
"org.openprojectx.hadoop.native.loader.core:core:$HNL" > /tmp/queue.txt
67+
# resolve = extract openprojectx coords from the given POM(s), resolve ${...} versions, and keep
68+
# only concrete (numeric) versions -- drops managed refs that carry no version of their own.
69+
resolve() { awk -f /tmp/op-deps.awk "$@" | sed -f /tmp/props.sed | awk -F: '$3 ~ /^[0-9]/'; }
70+
71+
# Seed the queue from every project POM -- this is the config-driven entry point.
72+
resolve $(find "$POMS" -name pom.xml | sort) | sort -u > /tmp/queue.txt
4973
: > /tmp/done.txt
74+
: > /tmp/failed.txt
5075

51-
# Breadth-first over the openprojectx closure: fetch each jar + -sources, then enqueue the
52-
# openprojectx dependencies found in its POM.
76+
# Breadth-first over the openprojectx closure: fetch each jar (+ -sources), then enqueue the
77+
# openprojectx dependencies found in its own POM. The Spark/Hadoop/Cloudera graph is never walked.
78+
# Fetches are tolerant: the BOM legitimately lists gradle-only / unreleased openprojectx artifacts
79+
# that have no resolvable jar for this version line; those are recorded in failed.txt and skipped
80+
# rather than failing the build.
5381
while [ -s /tmp/queue.txt ]; do
5482
gav="$(head -1 /tmp/queue.txt)"; sed -i 1d /tmp/queue.txt
5583
grep -qxF "$gav" /tmp/done.txt && continue
5684
echo "$gav" >> /tmp/done.txt
5785
g="${gav%%:*}"; rest="${gav#*:}"; a="${rest%%:*}"; v="${rest##*:}"
58-
mvn -B dependency:get -Dtransitive=false -DremoteRepositories="$REPOS" -Dartifact="$g:$a:$v"
59-
mvn -B dependency:get -Dtransitive=false -DremoteRepositories="$REPOS" -Dartifact="$g:$a:$v:jar:sources" || true
60-
pom="/root/.m2/repository/$(echo "$g" | tr . /)/$a/$v/$a-$v.pom"
61-
[ -f "$pom" ] && awk -f /tmp/op-deps.awk "$pom" >> /tmp/queue.txt || true
86+
if mvn -B dependency:get -Dtransitive=false -DremoteRepositories="$REPOS" -Dartifact="$g:$a:$v"; then
87+
mvn -B dependency:get -Dtransitive=false -DremoteRepositories="$REPOS" -Dartifact="$g:$a:$v:jar:sources" || true
88+
pom="/root/.m2/repository/$(echo "$g" | tr . /)/$a/$v/$a-$v.pom"
89+
[ -f "$pom" ] && resolve "$pom" >> /tmp/queue.txt || true
90+
else
91+
echo "$gav" >> /tmp/failed.txt
92+
fi
6293
done
6394

64-
echo "=== org.openprojectx artifacts in local repo ==="
95+
echo "=== org.openprojectx artifacts fetched ==="
6596
find /root/.m2/repository/org/openprojectx -name '*.jar' | sort
97+
if [ -s /tmp/failed.txt ]; then
98+
echo "=== openprojectx artifacts with no resolvable jar (skipped) ==="
99+
sort -u /tmp/failed.txt
100+
fi
101+
# Sanity check: a config or network problem that fetches nothing must fail the build loudly.
102+
find /root/.m2/repository/org/openprojectx -name '*.jar' | grep -q . \
103+
|| { echo "ERROR: no org.openprojectx jars were fetched"; exit 1; }
66104
SCRIPT
67105

68106
CMD ["bash"]

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
<maven.compiler.release>17</maven.compiler.release>
2525

2626
<!-- bigdata-test framework, published locally as 0.1.1-SNAPSHOT -->
27-
<bigdata-test.version>0.1.9</bigdata-test.version>
27+
<bigdata-test.version>0.1.10</bigdata-test.version>
2828

2929
<!-- shared, version-line independent dependencies -->
3030
<junit.version>5.11.4</junit.version>

‎spark-test-bom/pom.xml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,30 @@
3434
<version>${bigdata-test.version}</version>
3535
</dependency>
3636

37+
<dependency>
38+
<groupId>org.openprojectx.bigdata.test.core</groupId>
39+
<artifactId>org.openprojectx.bigdata-test.gradle.plugin</artifactId>
40+
<version>${bigdata-test.version}</version>
41+
</dependency>
42+
43+
<dependency>
44+
<groupId>org.openprojectx.bigdata.test.core</groupId>
45+
<artifactId>gradle-plugin</artifactId>
46+
<version>${bigdata-test.version}</version>
47+
</dependency>
48+
49+
<dependency>
50+
<groupId>org.openprojectx.bigdata.test.core</groupId>
51+
<artifactId>bigdata-test-spring-boot-autoconfigure</artifactId>
52+
<version>${bigdata-test.version}</version>
53+
</dependency>
54+
55+
<dependency>
56+
<groupId>org.openprojectx.bigdata.test.core</groupId>
57+
<artifactId>bigdata-test-spring-boot-starter</artifactId>
58+
<version>${bigdata-test.version}</version>
59+
</dependency>
60+
3761
<!-- java-dns agent jar, attached to the Surefire JVM to redirect GCS DNS -->
3862
<dependency>
3963
<groupId>org.openprojectx.java.dns</groupId>

0 commit comments

Comments
 (0)