Skip to content

Commit 938f2ca

Browse files
jinseopkim0jinseopkim0
committed
chore: add missing client library check scripts for downstream CI
1 parent 4d4d90f commit 938f2ca

3 files changed

Lines changed: 355 additions & 0 deletions

File tree

‎.kokoro/client-library-check-doclet.sh‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
#!/bin/bash
2+
# Copyright 2024 Google LLC
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
# Presubmit to ensure the dependencies of the Google Libraries BOM, with the modification of change
17+
# in the PR, pick up the highest versions among transitive dependencies.
18+
# https://maven.apache.org/enforcer/enforcer-rules/requireUpperBoundDeps.html
19+
20+
set -eo pipefail
21+
# Display commands being run.
22+
set -x
23+
24+
function get_current_version_from_versions_txt() {
25+
versions=$1
26+
key=$2
27+
version=$(grep "$key:" "${versions}" | cut -d: -f3) # 3rd field is current
28+
echo "${version}"
29+
}
30+
31+
function get_released_version_from_versions_txt() {
32+
versions=$1
33+
key=$2
34+
version=$(grep "$key:" "${versions}" | cut -d: -f2) # 2nd field is release
35+
echo "${version}"
36+
}
37+
38+
function replace_java_shared_config_version() {
39+
version=$1
40+
# replace version
41+
xmllint --shell <(cat pom.xml) << EOF
42+
setns x=http://maven.apache.org/POM/4.0.0
43+
cd .//x:artifactId[text()="google-cloud-shared-config"]
44+
cd ../x:version
45+
set ${version}
46+
save pom.xml
47+
EOF
48+
}
49+
50+
function replace_java_shared_dependencies_version() {
51+
version=$1
52+
# replace version
53+
xmllint --shell <(cat pom.xml) << EOF
54+
setns x=http://maven.apache.org/POM/4.0.0
55+
cd .//x:properties/x:google-cloud-shared-dependencies.version
56+
set ${version}
57+
save pom.xml
58+
EOF
59+
}
60+
61+
function replace_sdk_platform_java_config_version() {
62+
version=$1
63+
# replace version
64+
xmllint --shell <(cat pom.xml) << EOF
65+
setns x=http://maven.apache.org/POM/4.0.0
66+
cd .//x:artifactId[text()="sdk-platform-java-config"]
67+
cd ../x:version
68+
set ${version}
69+
save pom.xml
70+
EOF
71+
}
72+
REPO=$1
73+
# Get the directory of the build script
74+
scriptDir=$(realpath $(dirname "${BASH_SOURCE[0]}"))
75+
## cd to the parent directory, i.e. the root of the git repo
76+
cd ${scriptDir}/..
77+
78+
# Make artifacts available for 'mvn validate' at the bottom
79+
mvn install -DskipTests=true -Dmaven.javadoc.skip=true -Dgcloud.download.skip=true -B -V -q --no-transfer-progress
80+
81+
# Get version of doclet used to generate Cloud RAD for javadoc testing with the doclet below
82+
rm -rf java-docfx-doclet
83+
git clone https://github.com/googleapis/java-docfx-doclet.git
84+
pushd java-docfx-doclet/third_party/docfx-doclet-143274
85+
git checkout 1.9.0
86+
mvn package -Dmaven.test.skip=true -B -V --no-transfer-progress
87+
88+
# work from the root directory
89+
popd
90+
docletPath=$(realpath "java-docfx-doclet/third_party/docfx-doclet-143274/target/docfx-doclet-1.0-SNAPSHOT-jar-with-dependencies.jar")
91+
echo "This is the doclet path: ${docletPath}"
92+
93+
# Read the current version of this BOM in the POM. Example version: '0.116.1-alpha-SNAPSHOT'
94+
VERSION_POM=java-shared-config/pom.xml
95+
# Namespace (xmlns) prevents xmllint from specifying tag names in XPath
96+
JAVA_SHARED_CONFIG_VERSION=`sed -e 's/xmlns=".*"//' ${VERSION_POM} | xmllint --xpath '/project/version/text()' -`
97+
98+
if [ -z "${JAVA_SHARED_CONFIG_VERSION}" ]; then
99+
echo "Version is not found in ${VERSION_POM}"
100+
exit 1
101+
fi
102+
echo "Version: ${JAVA_SHARED_CONFIG_VERSION}"
103+
104+
# Update java-shared-config in sdk-platform-java-config
105+
rm -rf google-cloud-java
106+
# Find the latest tag matching v* and use it
107+
LATEST_TAG=$(git ls-remote --tags https://github.com/googleapis/google-cloud-java.git | grep 'refs/tags/v' | sort -k2,2 -V | tail -n 1 | awk '{print $2}' | sed 's|refs/tags/||')
108+
echo "Cloning google-cloud-java at tag: ${LATEST_TAG}"
109+
git clone "https://github.com/googleapis/google-cloud-java.git" -b "${LATEST_TAG}" --depth=1
110+
pushd google-cloud-java/sdk-platform-java
111+
SDK_PLATFORM_JAVA_CONFIG_VERSION=$(sed -e 's/xmlns=".*"//' sdk-platform-java-config/pom.xml | xmllint --xpath '/project/version/text()' -)
112+
113+
pushd sdk-platform-java-config
114+
# Use released version of google-cloud-shared-dependencies to avoid verifying SNAPSHOT changes.
115+
replace_java_shared_config_version "${JAVA_SHARED_CONFIG_VERSION}"
116+
echo "The diff in sdk-platform-java-config:"
117+
git --no-pager diff
118+
echo "--------"
119+
mvn install "-DskipTests=true" "-Dmaven.javadoc.skip=true" "-Dgcloud.download.skip=true" "-Dcheckstyle.skip=true" -B -V -q --no-transfer-progress
120+
popd
121+
122+
# Check javadoc generation with the doclet
123+
rm -rf "${REPO}"
124+
git clone "https://github.com/googleapis/${REPO}.git" --depth=1
125+
126+
pushd ${REPO}
127+
replace_sdk_platform_java_config_version "${SDK_PLATFORM_JAVA_CONFIG_VERSION}"
128+
129+
mvn clean -B --no-transfer-progress \
130+
-P docFX \
131+
-DdocletPath="${docletPath}" \
132+
-Dclirr.skip=true \
133+
-Denforcer.skip=true \
134+
-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
135+
-Dcheckstyle.skip=true \
136+
-Dflatten.skip=true \
137+
-Danimal.sniffer.skip=true \
138+
javadoc:aggregate
139+
140+
RETURN_CODE=$?
141+
if [ "${RETURN_CODE}" == 0 ]; then
142+
echo "Javadocs generated successfully with doclet"
143+
else
144+
echo "Javadoc generation FAILED with doclet"
145+
fi

‎.kokoro/client-library-check.sh‎

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
#!/bin/bash
2+
# Copyright 2021 Google LLC
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
# Presubmit to ensure the dependencies of the Google Libraries BOM, with the modification of change
17+
# in the PR, pick up the highest versions among transitive dependencies.
18+
# https://maven.apache.org/enforcer/enforcer-rules/requireUpperBoundDeps.html
19+
20+
set -eo pipefail
21+
# Display commands being run.
22+
set -x
23+
24+
function get_current_version_from_versions_txt() {
25+
versions=$1
26+
key=$2
27+
version=$(grep "$key:" "${versions}" | cut -d: -f3) # 3rd field is current
28+
echo "${version}"
29+
}
30+
31+
function get_released_version_from_versions_txt() {
32+
versions=$1
33+
key=$2
34+
version=$(grep "$key:" "${versions}" | cut -d: -f2) # 2nd field is release
35+
echo "${version}"
36+
}
37+
38+
function replace_java_shared_config_version() {
39+
version=$1
40+
# replace version
41+
xmllint --shell <(cat pom.xml) << EOF
42+
setns x=http://maven.apache.org/POM/4.0.0
43+
cd .//x:artifactId[text()="google-cloud-shared-config"]
44+
cd ../x:version
45+
set ${version}
46+
save pom.xml
47+
EOF
48+
}
49+
50+
function replace_java_shared_dependencies_version() {
51+
version=$1
52+
# replace version
53+
xmllint --shell <(cat pom.xml) << EOF
54+
setns x=http://maven.apache.org/POM/4.0.0
55+
cd .//x:properties/x:google-cloud-shared-dependencies.version
56+
set ${version}
57+
save pom.xml
58+
EOF
59+
}
60+
61+
function replace_sdk_platform_java_config_version() {
62+
version=$1
63+
# replace version
64+
xmllint --shell <(cat pom.xml) << EOF
65+
setns x=http://maven.apache.org/POM/4.0.0
66+
cd .//x:artifactId[text()="sdk-platform-java-config"]
67+
cd ../x:version
68+
set ${version}
69+
save pom.xml
70+
EOF
71+
}
72+
73+
if [[ $# -ne 2 ]];
74+
then
75+
echo "Usage: $0 <repo-name> <job-type>"
76+
echo "where repo-name is java-XXX and check-type is dependencies, lint, or clirr"
77+
exit 1
78+
fi
79+
REPO=$1
80+
# build.sh uses this environment variable
81+
export JOB_TYPE=$2
82+
83+
## Get the directory of the build script
84+
scriptDir=$(realpath $(dirname "${BASH_SOURCE[0]}"))
85+
## cd to the parent directory, i.e. the root of the git repo
86+
cd ${scriptDir}/..
87+
88+
# Make artifacts available for 'mvn validate' at the bottom
89+
mvn install -DskipTests=true -Dmaven.javadoc.skip=true -Dgcloud.download.skip=true -B -V -q
90+
91+
# Read the current version of this BOM in the POM. Example version: '0.116.1-alpha-SNAPSHOT'
92+
VERSION_POM=java-shared-config/pom.xml
93+
# Namespace (xmlns) prevents xmllint from specifying tag names in XPath
94+
JAVA_SHARED_CONFIG_VERSION=`sed -e 's/xmlns=".*"//' ${VERSION_POM} | xmllint --xpath '/project/version/text()' -`
95+
96+
if [ -z "${JAVA_SHARED_CONFIG_VERSION}" ]; then
97+
echo "Version is not found in ${VERSION_POM}"
98+
exit 1
99+
fi
100+
echo "Version: ${JAVA_SHARED_CONFIG_VERSION}"
101+
102+
# Update java-shared-config in sdk-platform-java-config
103+
rm -rf google-cloud-java
104+
# Find the latest tag matching v* and use it
105+
LATEST_TAG=$(git ls-remote --tags https://github.com/googleapis/google-cloud-java.git | grep 'refs/tags/v' | sort -k2,2 -V | tail -n 1 | awk '{print $2}' | sed 's|refs/tags/||')
106+
echo "Cloning google-cloud-java at tag: ${LATEST_TAG}"
107+
git clone "https://github.com/googleapis/google-cloud-java.git" -b "${LATEST_TAG}" --depth=1
108+
pushd google-cloud-java/sdk-platform-java
109+
SDK_PLATFORM_JAVA_CONFIG_VERSION=$(sed -e 's/xmlns=".*"//' sdk-platform-java-config/pom.xml | xmllint --xpath '/project/version/text()' -)
110+
111+
pushd sdk-platform-java-config
112+
# Use released version of google-cloud-shared-dependencies to avoid verifying SNAPSHOT changes.
113+
replace_java_shared_config_version "${JAVA_SHARED_CONFIG_VERSION}"
114+
echo "The diff in sdk-platform-java-config:"
115+
git --no-pager diff
116+
echo "--------"
117+
mvn install "-DskipTests=true" "-Dmaven.javadoc.skip=true" "-Dgcloud.download.skip=true" "-Dcheckstyle.skip=true" -B -V -q
118+
popd
119+
120+
popd
121+
122+
# Check this BOM against a few java client libraries
123+
# java-bigquery
124+
rm -rf "${REPO}"
125+
if [ -z "${REPO_TAG}" ]; then
126+
git clone "https://github.com/googleapis/${REPO}.git" --depth=1
127+
else
128+
git clone "https://github.com/googleapis/${REPO}.git" --depth=1 --branch "${REPO_TAG}"
129+
fi
130+
131+
pushd ${REPO}
132+
133+
# If using an older version of java-storage, continue replacing java-shared-config version otherwise replace
134+
# the version of sdk-platform-java-config.
135+
if [ "${REPO_TAG}" == "v2.9.3" ] && [ "${REPO}" == "java-storage" ]; then
136+
replace_java_shared_config_version "${JAVA_SHARED_CONFIG_VERSION}"
137+
else
138+
replace_sdk_platform_java_config_version "${SDK_PLATFORM_JAVA_CONFIG_VERSION}"
139+
fi
140+
141+
case ${JOB_TYPE} in
142+
dependencies)
143+
.kokoro/dependencies.sh
144+
RETURN_CODE=$?
145+
;;
146+
flatten-plugin)
147+
# This creates .flattened-pom.xml
148+
echo "Before running .kokoro/build.sh"
149+
.kokoro/build.sh
150+
echo "After running .kokoro/build.sh"
151+
pushd google-cloud-*
152+
mvn dependency:list -f .flattened-pom.xml -DincludeScope=runtime -Dsort=true \
153+
| grep '\[INFO] .*:.*:.*:.*:.*' |awk '{print $2}' > .actual-flattened-dependencies-list.txt
154+
echo "Diff from the expected file (${EXPECTED_DEPENDENCIES_LIST}):"
155+
diff "${scriptDir}/${EXPECTED_DEPENDENCIES_LIST}" .actual-flattened-dependencies-list.txt
156+
RETURN_CODE=$?
157+
if [ "${RETURN_CODE}" == 0 ]; then
158+
echo "No diff."
159+
else
160+
echo "There was a diff."
161+
fi
162+
popd
163+
;;
164+
*)
165+
# Here we replace the com.coveo fmt plugin with the spotify version.
166+
# This `sed` won't be needed once downstream repositories update
167+
# `.kokoro/build.sh` to use the `com.spotify.fmt` group ID.
168+
sed -i 's/com.coveo:fmt-maven-plugin/com.spotify.fmt:fmt-maven-plugin/' .kokoro/build.sh
169+
# This reads the JOB_TYPE environmental variable
170+
.kokoro/build.sh
171+
RETURN_CODE=$?
172+
;;
173+
esac
174+
175+
echo "exiting with ${RETURN_CODE}"
176+
exit ${RETURN_CODE}

‎.kokoro/java-storage-v2.9.3-expected-flattened-dependencies.txt‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
com.fasterxml.jackson.core:jackson-core:jar:2.13.3:compile
2+
com.google.api-client:google-api-client:jar:1.35.1:compile
3+
com.google.api.grpc:proto-google-common-protos:jar:2.9.0:compile
4+
com.google.api.grpc:proto-google-iam-v1:jar:1.4.1:compile
5+
com.google.api:api-common:jar:2.2.1:compile
6+
com.google.api:gax-httpjson:jar:0.103.2:compile
7+
com.google.api:gax:jar:2.18.2:compile
8+
com.google.apis:google-api-services-storage:jar:v1-rev20220705-1.32.1:compile
9+
com.google.auth:google-auth-library-credentials:jar:1.7.0:compile
10+
com.google.auth:google-auth-library-oauth2-http:jar:1.7.0:compile
11+
com.google.auto.value:auto-value-annotations:jar:1.11.0:compile
12+
com.google.cloud:google-cloud-core-http:jar:2.8.0:compile
13+
com.google.cloud:google-cloud-core:jar:2.8.0:compile
14+
com.google.code.findbugs:jsr305:jar:3.0.2:compile
15+
com.google.code.gson:gson:jar:2.9.0:compile
16+
com.google.errorprone:error_prone_annotations:jar:2.14.0:compile
17+
com.google.guava:failureaccess:jar:1.0.1:compile
18+
com.google.guava:guava:jar:31.1-jre:compile
19+
com.google.guava:listenablefuture:jar:9999.0-empty-to-avoid-conflict-with-guava:compile
20+
com.google.http-client:google-http-client-apache-v2:jar:1.42.0:compile
21+
com.google.http-client:google-http-client-appengine:jar:1.42.0:compile
22+
com.google.http-client:google-http-client-gson:jar:1.42.0:compile
23+
com.google.http-client:google-http-client-jackson2:jar:1.42.0:compile
24+
com.google.http-client:google-http-client:jar:1.42.0:compile
25+
com.google.j2objc:j2objc-annotations:jar:1.3:compile
26+
com.google.oauth-client:google-oauth-client:jar:1.34.1:compile
27+
com.google.protobuf:protobuf-java-util:jar:3.21.1:compile
28+
com.google.protobuf:protobuf-java:jar:3.21.1:compile
29+
io.grpc:grpc-context:jar:1.47.0:compile
30+
io.opencensus:opencensus-api:jar:0.31.1:compile
31+
io.opencensus:opencensus-contrib-http-util:jar:0.31.1:compile
32+
javax.annotation:javax.annotation-api:jar:1.3.2:compile
33+
org.checkerframework:checker-qual:jar:3.22.2:compile
34+
org.threeten:threetenbp:jar:1.6.0:compile

0 commit comments

Comments
 (0)