Skip to content

Commit 2cc1b8a

Browse files
xinyualxinyual
committed
merge to main
Signed-off-by: xinyual <xinyual@amazon.com>
2 parents 87b28b2 + 270aa0d commit 2cc1b8a

278 files changed

Lines changed: 13638 additions & 1138 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/get-sonatype-credentials.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/bash
2+
# Script to retrieve Sonatype credentials from AWS Secrets Manager
3+
4+
SONATYPE_USERNAME=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-username --query SecretString --output text)
5+
SONATYPE_PASSWORD=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-password --query SecretString --output text)
6+
echo "::add-mask::$SONATYPE_USERNAME"
7+
echo "::add-mask::$SONATYPE_PASSWORD"
8+
echo "SONATYPE_USERNAME=$SONATYPE_USERNAME" >> $GITHUB_ENV
9+
echo "SONATYPE_PASSWORD=$SONATYPE_PASSWORD" >> $GITHUB_ENV

‎.github/workflows/maven-publish.yml‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@ on:
88
- 1.*
99
- 2.*
1010

11+
env:
12+
SNAPSHOT_REPO_URL: https://aws.oss.sonatype.org/content/repositories/snapshots/
13+
1114
jobs:
1215
build-and-publish-snapshots:
1316
strategy:
@@ -29,10 +32,12 @@ jobs:
2932
with:
3033
role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }}
3134
aws-region: us-east-1
35+
36+
- name: get credentials
37+
run: |
38+
# Get credentials for publishing
39+
.github/get-sonatype-credentials.sh
40+
3241
- name: publish snapshots to maven
3342
run: |
34-
export SONATYPE_USERNAME=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-username --query SecretString --output text)
35-
export SONATYPE_PASSWORD=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-password --query SecretString --output text)
36-
echo "::add-mask::$SONATYPE_USERNAME"
37-
echo "::add-mask::$SONATYPE_PASSWORD"
38-
./gradlew publishPluginZipPublicationToSnapshotsRepository
43+
./gradlew publishPluginZipPublicationToSnapshotsRepository

‎.github/workflows/publish-snapshots-and-grammar-files.yml‎

Lines changed: 354 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,354 @@
1+
name: Publish snapshots and grammar files to maven
2+
3+
on:
4+
workflow_dispatch:
5+
push:
6+
branches:
7+
- main
8+
- 1.*
9+
- 2.*
10+
11+
env:
12+
SNAPSHOT_REPO_URL: https://aws.oss.sonatype.org/content/repositories/snapshots/
13+
14+
jobs:
15+
publish-grammar-files:
16+
strategy:
17+
fail-fast: false
18+
if: github.repository == 'opensearch-project/sql'
19+
runs-on: ubuntu-latest
20+
21+
permissions:
22+
id-token: write
23+
contents: write
24+
25+
env:
26+
TARGET_REPO_PATH: org/opensearch/language-grammar
27+
28+
steps:
29+
- uses: actions/setup-java@v3
30+
with:
31+
distribution: temurin # Temurin is a distribution of adoptium
32+
java-version: 21
33+
- uses: actions/checkout@v3
34+
- uses: aws-actions/configure-aws-credentials@v1.7.0
35+
with:
36+
role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }}
37+
aws-region: us-east-1
38+
39+
# Get Maven credentials
40+
- name: Setup publishing credentials
41+
id: creds
42+
run: |
43+
.github/get-sonatype-credentials.sh
44+
45+
# Extract version information directly from build.gradle
46+
- name: Set version
47+
id: set_version
48+
run: |
49+
VERSION=$(grep "version = " ./language-grammar/build.gradle | cut -d "'" -f 2)
50+
echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT
51+
echo "Using version: ${VERSION}"
52+
53+
# Capture commit ID
54+
- name: Set commit ID
55+
id: set_commit
56+
run: |
57+
COMMIT_ID=$(git log -1 --format='%H')
58+
echo "commit_id=${COMMIT_ID}" >> $GITHUB_OUTPUT
59+
echo "Using commit ID: ${COMMIT_ID}"
60+
61+
# Create ZIP of grammar files
62+
- name: Package grammar files
63+
run: |
64+
# Create directory for the zip content
65+
mkdir -p grammar_files
66+
67+
# Copy all .g4 files to the directory
68+
find ./language-grammar/src/main/antlr4 -name "*.g4" -type f -exec cp {} grammar_files/ \;
69+
70+
# List the files that will be included in the zip
71+
echo "Files to be included in the zip:"
72+
ls -la grammar_files/
73+
74+
# Create zip file
75+
cd grammar_files
76+
zip -r ../grammar.zip ./*
77+
cd ..
78+
79+
# Check the zip file
80+
ls -la grammar.zip
81+
82+
- name: Prepare for Maven publishing
83+
run: |
84+
# Define constants
85+
ARTIFACT_ID="language-grammar"
86+
GROUP_ID="org.opensearch"
87+
VERSION="${{ steps.set_version.outputs.VERSION }}"
88+
89+
# Create directory structure for Maven
90+
MAVEN_LOCAL_PATH="${HOME}/.m2/repository/${GROUP_ID//.//}/${ARTIFACT_ID}/${VERSION}"
91+
mkdir -p "${MAVEN_LOCAL_PATH}"
92+
93+
# Copy the zip file to Maven directory with proper naming
94+
MAVEN_ZIP_NAME="${ARTIFACT_ID}-${VERSION}.zip"
95+
cp grammar.zip "${MAVEN_LOCAL_PATH}/${MAVEN_ZIP_NAME}"
96+
97+
# Generate POM file
98+
cat > "${MAVEN_LOCAL_PATH}/${ARTIFACT_ID}-${VERSION}.pom" << EOF
99+
<?xml version="1.0" encoding="UTF-8"?>
100+
<project xmlns="http://maven.apache.org/POM/4.0.0"
101+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
102+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
103+
<modelVersion>4.0.0</modelVersion>
104+
<groupId>${GROUP_ID}</groupId>
105+
<artifactId>${ARTIFACT_ID}</artifactId>
106+
<version>${VERSION}</version>
107+
<packaging>zip</packaging>
108+
<description>OpenSearch Language Grammar Files</description>
109+
</project>
110+
EOF
111+
112+
echo "Grammar files prepared for Maven publishing as version ${VERSION}"
113+
114+
# Generate checksums for the Maven artifacts
115+
- name: Generate checksums
116+
run: |
117+
for i in `find ${HOME}/.m2/repository/org/opensearch/ -name "*.pom" -type f`; do sha512sum "$i" | awk '{print $1}' >> "$i.sha512"; done
118+
for i in `find ${HOME}/.m2/repository/org/opensearch/ -name "*.zip" -type f`; do sha512sum "$i" | awk '{print $1}' >> "$i.sha512"; done
119+
for i in `find ${HOME}/.m2/repository/org/opensearch/ -name "*.pom" -type f`; do sha256sum "$i" | awk '{print $1}' >> "$i.sha256"; done
120+
for i in `find ${HOME}/.m2/repository/org/opensearch/ -name "*.zip" -type f`; do sha256sum "$i" | awk '{print $1}' >> "$i.sha256"; done
121+
122+
# Checkout build libraries for publishing scripts
123+
- uses: actions/checkout@v4
124+
with:
125+
repository: 'opensearch-project/opensearch-build-libraries'
126+
path: 'build'
127+
128+
# Publish to Maven
129+
- name: Publish to Maven
130+
run: |
131+
# Copy local Maven repo to build directory
132+
cd build/resources/publish/
133+
cp -a $HOME/.m2/repository/* ./
134+
135+
# Run the publish script
136+
./publish-snapshot.sh ./
137+
138+
# Update metadata with commit ID
139+
- name: Add commit ID to metadata
140+
run: |
141+
COMMIT_ID="${{ steps.set_commit.outputs.commit_id }}"
142+
ARTIFACT_ID="language-grammar"
143+
VERSION="${{ steps.set_version.outputs.VERSION }}"
144+
145+
TEMP_DIR=$(mktemp -d)
146+
METADATA_FILE="${TEMP_DIR}/maven-metadata.xml"
147+
148+
# Download existing metadata
149+
META_URL="${SNAPSHOT_REPO_URL}org/opensearch/${ARTIFACT_ID}/${VERSION}/maven-metadata.xml"
150+
curl -s -u "${SONATYPE_USERNAME}:${SONATYPE_PASSWORD}" -o "${METADATA_FILE}" "${META_URL}"
151+
152+
if [ -s "${METADATA_FILE}" ]; then
153+
cp "${METADATA_FILE}" "${METADATA_FILE}.bak"
154+
155+
# Add commit ID to metadata
156+
awk -v commit="${COMMIT_ID}" '
157+
/<versioning>/ {
158+
print $0
159+
print " <commitId>" commit "</commitId>"
160+
next
161+
}
162+
{print}
163+
' "${METADATA_FILE}.bak" > "${METADATA_FILE}"
164+
165+
# Upload modified metadata
166+
curl -X PUT -u "${SONATYPE_USERNAME}:${SONATYPE_PASSWORD}" --upload-file "${METADATA_FILE}" "${META_URL}"
167+
168+
# Update checksums
169+
cd "${TEMP_DIR}"
170+
sha256sum "maven-metadata.xml" | awk '{print $1}' > "maven-metadata.xml.sha256"
171+
sha512sum "maven-metadata.xml" | awk '{print $1}' > "maven-metadata.xml.sha512"
172+
173+
curl -X PUT -u "${SONATYPE_USERNAME}:${SONATYPE_PASSWORD}" --upload-file "maven-metadata.xml.sha256" "${META_URL}.sha256"
174+
curl -X PUT -u "${SONATYPE_USERNAME}:${SONATYPE_PASSWORD}" --upload-file "maven-metadata.xml.sha512" "${META_URL}.sha512"
175+
176+
echo "Version metadata updated with commit ID"
177+
else
178+
echo "Failed to download metadata, skipping commit ID addition"
179+
fi
180+
181+
rm -rf "${TEMP_DIR}"
182+
183+
publish-async-query-core:
184+
strategy:
185+
fail-fast: false
186+
if: github.repository == 'opensearch-project/sql'
187+
runs-on: ubuntu-latest
188+
189+
permissions:
190+
id-token: write
191+
contents: write
192+
193+
steps:
194+
- uses: actions/setup-java@v3
195+
with:
196+
distribution: temurin
197+
java-version: 21
198+
- uses: actions/checkout@v3
199+
- uses: aws-actions/configure-aws-credentials@v1.7.0
200+
with:
201+
role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }}
202+
aws-region: us-east-1
203+
204+
# Get and mask credentials once in a dedicated step
205+
- name: Setup publishing credentials
206+
id: creds
207+
run: |
208+
.github/get-sonatype-credentials.sh
209+
210+
# Capture the commit ID for metadata purposes
211+
- name: Set commit ID
212+
id: set_commit
213+
run: |
214+
COMMIT_ID=$(git log -1 --format='%H')
215+
echo "commit_id=${COMMIT_ID}" >> $GITHUB_OUTPUT
216+
echo "Using commit ID: ${COMMIT_ID}"
217+
218+
# Replace the current "Extract version from build.gradle" step with this:
219+
- name: Extract version from build.gradle
220+
id: extract_version
221+
run: |
222+
# Extract the version directly from the build.gradle file
223+
VERSION=$(grep -m 1 "archiveVersion.set" ./async-query-core/build.gradle | sed -n "s/.*archiveVersion.set('\([^']*\)').*/\1/p")
224+
225+
# Add -SNAPSHOT suffix for snapshot repository if not already present
226+
if [[ ! $VERSION == *-SNAPSHOT ]]; then
227+
VERSION="${VERSION}-SNAPSHOT"
228+
fi
229+
230+
echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT
231+
echo "Version: ${VERSION}"
232+
233+
- name: Build and publish shadow JAR
234+
run: |
235+
# Build the shadow JAR
236+
./gradlew :async-query-core:shadowJar
237+
238+
# Define constants
239+
ARTIFACT_ID="async-query-core"
240+
GROUP_PATH="org/opensearch"
241+
VERSION="${{ steps.extract_version.outputs.VERSION }}"
242+
243+
# Find the generated shadow JAR
244+
SHADOW_JAR=$(find ./async-query-core/build/libs/ -name "*-all.jar" | head -n 1)
245+
246+
if [ -z "$SHADOW_JAR" ]; then
247+
echo "Error: Shadow JAR not found!"
248+
exit 1
249+
fi
250+
251+
# Create directory structure in local Maven repository
252+
MAVEN_LOCAL_PATH="${HOME}/.m2/repository/${GROUP_PATH}/${ARTIFACT_ID}/${VERSION}"
253+
mkdir -p "${MAVEN_LOCAL_PATH}"
254+
255+
# Copy the shadow JAR to the local Maven repository with proper naming
256+
MAVEN_JAR_NAME="${ARTIFACT_ID}-${VERSION}.jar"
257+
cp "${SHADOW_JAR}" "${MAVEN_LOCAL_PATH}/${MAVEN_JAR_NAME}"
258+
259+
# Generate a POM file
260+
cat > "${MAVEN_LOCAL_PATH}/${ARTIFACT_ID}-${VERSION}.pom" << EOF
261+
<?xml version="1.0" encoding="UTF-8"?>
262+
<project xmlns="http://maven.apache.org/POM/4.0.0"
263+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
264+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
265+
<modelVersion>4.0.0</modelVersion>
266+
<groupId>org.opensearch</groupId>
267+
<artifactId>${ARTIFACT_ID}</artifactId>
268+
<version>${VERSION}</version>
269+
</project>
270+
EOF
271+
272+
echo "Shadow JAR and POM published to local Maven repository for version ${VERSION}"
273+
274+
# Checkout opensearch-build-libraries repository for publishing scripts
275+
- uses: actions/checkout@v4
276+
with:
277+
repository: 'opensearch-project/opensearch-build-libraries'
278+
path: 'build'
279+
280+
- name: Generate SHA checksums for JAR and POM files
281+
run: |
282+
for i in `find ${HOME}/.m2/repository/org/opensearch/ -name "*.pom" -type f`; do sha512sum "$i" | awk '{print $1}' >> "$i.sha512"; done
283+
for i in `find ${HOME}/.m2/repository/org/opensearch/ -name "*.jar" -type f`; do sha512sum "$i" | awk '{print $1}' >> "$i.sha512"; done
284+
for i in `find ${HOME}/.m2/repository/org/opensearch/ -name "*.pom" -type f`; do sha256sum "$i" | awk '{print $1}' >> "$i.sha256"; done
285+
for i in `find ${HOME}/.m2/repository/org/opensearch/ -name "*.jar" -type f`; do sha256sum "$i" | awk '{print $1}' >> "$i.sha256"; done
286+
287+
- name: Install XML tools
288+
run: sudo apt-get update && sudo apt-get install -y xmlstarlet
289+
290+
- name: Publish snapshots to maven
291+
run: |
292+
# Publish snapshots to maven
293+
cd build/resources/publish/
294+
cp -a $HOME/.m2/repository/* ./
295+
./publish-snapshot.sh ./
296+
297+
- name: Update version metadata with commit ID
298+
run: |
299+
COMMIT_ID="${{ steps.set_commit.outputs.commit_id }}"
300+
ARTIFACT_ID="async-query-core"
301+
VERSION="${{ steps.extract_version.outputs.VERSION }}"
302+
303+
# Add commit ID to version-specific metadata file
304+
echo "Processing commit ID for version: ${VERSION}"
305+
306+
TEMP_DIR=$(mktemp -d)
307+
METADATA_FILE="${TEMP_DIR}/maven-metadata.xml"
308+
309+
# Download metadata from repository
310+
META_URL="${SNAPSHOT_REPO_URL}org/opensearch/${ARTIFACT_ID}/${VERSION}/maven-metadata.xml"
311+
echo "Downloading metadata from ${META_URL}"
312+
313+
# Try to download the metadata file
314+
curl -s -u "${SONATYPE_USERNAME}:${SONATYPE_PASSWORD}" -o "${METADATA_FILE}" "${META_URL}"
315+
316+
# If successful, modify and upload back
317+
if [ -s "${METADATA_FILE}" ]; then
318+
echo "Modifying metadata for ${VERSION}"
319+
cp "${METADATA_FILE}" "${METADATA_FILE}.bak"
320+
321+
# Apply same awk command from working example
322+
awk -v commit="${COMMIT_ID}" '
323+
/<versioning>/ {
324+
print $0
325+
print " <commitId>" commit "</commitId>"
326+
next
327+
}
328+
{print}
329+
' "${METADATA_FILE}.bak" > "${METADATA_FILE}"
330+
331+
# Upload modified file back
332+
echo "Uploading modified metadata to ${META_URL}"
333+
curl -X PUT -u "${SONATYPE_USERNAME}:${SONATYPE_PASSWORD}" --upload-file "${METADATA_FILE}" "${META_URL}"
334+
335+
# Update the SHA checksums
336+
cd "${TEMP_DIR}"
337+
sha256sum "maven-metadata.xml" | awk '{print $1}' > "maven-metadata.xml.sha256"
338+
sha512sum "maven-metadata.xml" | awk '{print $1}' > "maven-metadata.xml.sha512"
339+
340+
# Upload the checksums
341+
curl -X PUT -u "${SONATYPE_USERNAME}:${SONATYPE_PASSWORD}" --upload-file "maven-metadata.xml.sha256" "${META_URL}.sha256"
342+
curl -X PUT -u "${SONATYPE_USERNAME}:${SONATYPE_PASSWORD}" --upload-file "maven-metadata.xml.sha512" "${META_URL}.sha512"
343+
cd -
344+
345+
echo "Updated metadata and checksums for ${VERSION}"
346+
else
347+
echo "Failed to download metadata for ${VERSION} or file is empty"
348+
exit 1
349+
fi
350+
351+
# Clean up
352+
rm -rf "${TEMP_DIR}"
353+
354+
echo "Version metadata updated with commit ID"

0 commit comments

Comments
 (0)