Skip to content

Commit defdd24

Browse files
XiaoHongbo-HopeXiaoHongbo-Hope
authored
build: harden Rust crate publishing (#562)
1 parent 2db588a commit defdd24

15 files changed

Lines changed: 706 additions & 46 deletions

File tree

‎.github/workflows/release-rust.yml‎

Lines changed: 187 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,8 @@
1515
# specific language governing permissions and limitations
1616
# under the License.
1717

18-
# Publish paimon and paimon-datafusion crates to crates.io.
19-
#
20-
# Trigger: push a version tag (e.g. v0.1.0, v0.1.0-rc1).
21-
# Pre-release tags (containing '-') only run dry-run checks without publishing.
22-
#
23-
# Token auth: add secret CARGO_REGISTRY_TOKEN for crates.io publishing.
18+
# GA tags publish; RC tags validate only.
19+
# Secret: CARGO_REGISTRY_TOKEN.
2420

2521
name: Release Rust
2622

@@ -31,19 +27,15 @@ on:
3127
- "v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+"
3228
workflow_dispatch:
3329

30+
concurrency:
31+
group: ${{ github.workflow }}-${{ github.ref }}
32+
cancel-in-progress: false
33+
3434
jobs:
35-
publish:
35+
validate:
3636
runs-on: ubuntu-latest
3737
permissions:
3838
contents: read
39-
strategy:
40-
# Publish crates sequentially to respect dependency order
41-
max-parallel: 1
42-
matrix:
43-
# Order matters: paimon must be published before paimon-datafusion
44-
package:
45-
- paimon
46-
- paimon-datafusion
4739
steps:
4840
- uses: actions/checkout@v7
4941

@@ -52,24 +44,190 @@ jobs:
5244
rustup update stable
5345
rustup default stable
5446
55-
- name: Dry run
56-
# Only dry-run paimon since paimon-datafusion depends on it
57-
if: matrix.package == 'paimon'
58-
run: cargo publish -p ${{ matrix.package }} --all-features --dry-run
47+
- name: Validate release tag
48+
if: github.ref_type == 'tag'
49+
shell: bash
50+
run: |
51+
set -euo pipefail
52+
if [[ ! "${GITHUB_REF_NAME}" =~ ^v([0-9]+\.[0-9]+\.[0-9]+)(-rc[0-9]+)?$ ]]; then
53+
echo "::error::invalid release tag: ${GITHUB_REF_NAME}"
54+
exit 1
55+
fi
56+
TAG_VERSION="${BASH_REMATCH[1]}"
57+
WORKSPACE_VERSION=$(
58+
cargo metadata --locked --format-version 1 --no-deps |
59+
jq -er '
60+
[.packages[] |
61+
select(.name == "paimon" or .name == "paimon-datafusion") |
62+
.version] as $versions |
63+
if ($versions | length) == 2 and
64+
($versions | unique | length) == 1
65+
then $versions[0]
66+
else error("release crate versions differ")
67+
end
68+
'
69+
)
70+
if [ "${TAG_VERSION}" != "${WORKSPACE_VERSION}" ]; then
71+
echo "::error::tag ${TAG_VERSION} does not match workspace ${WORKSPACE_VERSION}"
72+
exit 1
73+
fi
74+
75+
- name: Check package inventory
76+
shell: bash
77+
run: |
78+
set -euo pipefail
79+
for crate_dir in crates/paimon crates/integrations/datafusion; do
80+
for legal_file in LICENSE NOTICE; do
81+
cmp --silent "${legal_file}" "${crate_dir}/${legal_file}" || {
82+
echo "::error::${crate_dir}/${legal_file} differs from ${legal_file}"
83+
exit 1
84+
}
85+
done
86+
done
87+
88+
for package in paimon paimon-datafusion; do
89+
files="/tmp/${package}-package-files.txt"
90+
cargo package --locked -p "${package}" --list > "${files}"
91+
for required in LICENSE NOTICE DEPENDENCIES.rust.tsv test_utils.rs; do
92+
grep --fixed-strings --line-regexp --quiet "${required}" "${files}" || {
93+
echo "::error::${package} package is missing ${required}"
94+
exit 1
95+
}
96+
done
97+
grep --quiet '^testdata/' "${files}" || {
98+
echo "::error::${package} package is missing testdata"
99+
exit 1
100+
}
101+
done
102+
103+
grep --fixed-strings --line-regexp --quiet \
104+
blob_test_utils.rs /tmp/paimon-package-files.txt || {
105+
echo "::error::paimon package is missing blob_test_utils.rs"
106+
exit 1
107+
}
108+
109+
- name: Package release crates
110+
# Cargo verifies workspace dependencies through its temporary registry.
111+
run: >
112+
cargo package --locked
113+
-p paimon
114+
-p paimon-datafusion
115+
--all-features
116+
117+
- name: Dry run paimon
118+
run: cargo publish --locked -p paimon --all-features --dry-run
119+
120+
publish-paimon:
121+
needs: validate
122+
if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref_name, '-')
123+
runs-on: ubuntu-latest
124+
permissions:
125+
contents: read
126+
steps:
127+
- uses: actions/checkout@v7
128+
129+
- name: Setup Rust toolchain
130+
run: |
131+
rustup update stable
132+
rustup default stable
59133
60-
- name: Publish ${{ matrix.package }} to crates.io
61-
if: startsWith(github.ref, 'refs/tags/') && !contains(github.ref, '-')
62-
run: cargo publish -p ${{ matrix.package }} --all-features
134+
- name: Publish paimon if absent
135+
shell: bash
63136
env:
64137
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
138+
run: |
139+
set -euo pipefail
140+
VERSION=$(
141+
cargo metadata --locked --format-version 1 --no-deps |
142+
jq -er '.packages[] | select(.name == "paimon") | .version'
143+
)
144+
USER_AGENT="paimon-rust-release/1.0 (${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY})"
145+
STATUS=$(
146+
curl --silent --show-error --retry 3 \
147+
--user-agent "${USER_AGENT}" \
148+
--output /tmp/paimon-version.json \
149+
--write-out '%{http_code}' \
150+
"https://crates.io/api/v1/crates/paimon/${VERSION}"
151+
)
152+
if [ "${STATUS}" = 200 ]; then
153+
jq -e --arg version "${VERSION}" '.version.num == $version' \
154+
/tmp/paimon-version.json >/dev/null
155+
echo "paimon ${VERSION} already exists; skipping"
156+
elif [ "${STATUS}" = 404 ]; then
157+
cargo publish --locked -p paimon --all-features
158+
else
159+
echo "::error::crates.io returned HTTP ${STATUS} for paimon ${VERSION}"
160+
exit 1
161+
fi
65162
66-
- name: Wait for crates.io to update
67-
if: startsWith(github.ref, 'refs/tags/') && !contains(github.ref, '-') && matrix.package == 'paimon'
163+
- name: Wait for paimon on crates.io
164+
shell: bash
68165
run: |
69-
VERSION=${GITHUB_REF_NAME#v}
70-
for i in {1..10}; do
71-
CURRENT=$(curl -s https://crates.io/api/v1/crates/paimon | jq -r '.crate.max_version')
72-
[ "$CURRENT" = "$VERSION" ] && echo "✅ paimon $VERSION available" && exit 0
73-
echo "Waiting for $VERSION (currently $CURRENT)... $i/10"
166+
set -euo pipefail
167+
VERSION=$(
168+
cargo metadata --locked --format-version 1 --no-deps |
169+
jq -er '.packages[] | select(.name == "paimon") | .version'
170+
)
171+
USER_AGENT="paimon-rust-release/1.0 (${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY})"
172+
for attempt in {1..20}; do
173+
CURRENT=$(
174+
curl --fail-with-body --silent --show-error --retry 3 \
175+
--user-agent "${USER_AGENT}" \
176+
"https://crates.io/api/v1/crates/paimon/${VERSION}" |
177+
jq -r '.version.num // empty' || true
178+
)
179+
if [ "${CURRENT}" = "${VERSION}" ]; then
180+
echo "paimon ${VERSION} is available"
181+
exit 0
182+
fi
183+
echo "Waiting for paimon ${VERSION}... ${attempt}/20"
74184
sleep 30
75185
done
186+
echo "::error::paimon ${VERSION} did not appear before timeout"
187+
exit 1
188+
189+
publish-paimon-datafusion:
190+
needs: publish-paimon
191+
if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref_name, '-')
192+
runs-on: ubuntu-latest
193+
permissions:
194+
contents: read
195+
steps:
196+
- uses: actions/checkout@v7
197+
198+
- name: Setup Rust toolchain
199+
run: |
200+
rustup update stable
201+
rustup default stable
202+
203+
- name: Dry run paimon-datafusion
204+
run: cargo publish --locked -p paimon-datafusion --all-features --dry-run
205+
206+
- name: Publish paimon-datafusion if absent
207+
shell: bash
208+
env:
209+
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
210+
run: |
211+
set -euo pipefail
212+
VERSION=$(
213+
cargo metadata --locked --format-version 1 --no-deps |
214+
jq -er '.packages[] | select(.name == "paimon-datafusion") | .version'
215+
)
216+
USER_AGENT="paimon-rust-release/1.0 (${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY})"
217+
STATUS=$(
218+
curl --silent --show-error --retry 3 \
219+
--user-agent "${USER_AGENT}" \
220+
--output /tmp/paimon-datafusion-version.json \
221+
--write-out '%{http_code}' \
222+
"https://crates.io/api/v1/crates/paimon-datafusion/${VERSION}"
223+
)
224+
if [ "${STATUS}" = 200 ]; then
225+
jq -e --arg version "${VERSION}" '.version.num == $version' \
226+
/tmp/paimon-datafusion-version.json >/dev/null
227+
echo "paimon-datafusion ${VERSION} already exists; skipping"
228+
elif [ "${STATUS}" = 404 ]; then
229+
cargo publish --locked -p paimon-datafusion --all-features
230+
else
231+
echo "::error::crates.io returned HTTP ${STATUS} for paimon-datafusion ${VERSION}"
232+
exit 1
233+
fi

‎crates/integrations/datafusion/Cargo.toml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ name = "paimon-datafusion"
2020
edition.workspace = true
2121
version.workspace = true
2222
publish = ["crates-io"]
23-
exclude = ["testdata/"]
2423
license.workspace = true
2524
rust-version.workspace = true
2625
homepage = "https://paimon.apache.org/docs/rust/datafusion/"

0 commit comments

Comments
 (0)