Skip to content

Commit 3cdf64c

Browse files
tae898tae898
committed
Optimize CI/CD workflows with reusable build workflow
- Create build-python-wheels.yml as reusable workflow - Downloads JARs once, filters using jar_exclusions.txt - Builds all 5 platform wheels in parallel - Can be called by other workflows or triggered manually - Accepts optional version input parameter - Update test-python-bindings.yml to use shared build - Add skip-build input to optionally use pre-built wheels - Eliminates redundant wheel building in release workflow - Standalone runs still build their own wheels - Update test-python-examples.yml to use shared build - Add skip-build input to optionally use pre-built wheels - Tests run on native runners (4 platforms) - Standalone runs still build their own wheels - Optimize release-python-packages.yml workflow - Build wheels once using build-python-wheels.yml - Tests run in parallel using pre-built wheels - Publish uses same wheels that passed tests - Reduces build time by ~20-30 minutes - Guarantees published wheels match tested wheels Benefits: - Build once, test in parallel, publish same wheels - Eliminates 3x redundant builds (test-bindings, test-examples, release) - Tests run in parallel after build completes - Single source of truth for build logic - Easier to maintain and update build process
1 parent d77961e commit 3cdf64c

4 files changed

Lines changed: 302 additions & 306 deletions

File tree

‎.github/workflows/build-python-wheels.yml‎

Lines changed: 240 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,240 @@
1+
name: Build Python Wheels
2+
3+
on:
4+
# Allow being called by other workflows
5+
workflow_call:
6+
inputs:
7+
version:
8+
description: 'Python package version (e.g., 25.10.1.dev0)'
9+
required: false
10+
type: string
11+
outputs:
12+
wheel-count:
13+
description: 'Number of wheels built'
14+
value: ${{ jobs.summary.outputs.wheel-count }}
15+
16+
# Allow manual trigger for testing
17+
workflow_dispatch:
18+
inputs:
19+
version:
20+
description: 'Python package version (leave empty to use pom.xml)'
21+
required: false
22+
type: string
23+
24+
jobs:
25+
# First job: Download ArcadeDB JARs once
26+
download-jars:
27+
name: Download and Filter JARs
28+
runs-on: ubuntu-latest
29+
steps:
30+
- name: Checkout code
31+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
32+
33+
- name: Set up Docker Buildx
34+
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
35+
36+
- name: Download JARs from ArcadeDB Docker image
37+
shell: bash
38+
run: |
39+
cd bindings/python
40+
41+
# Detect ArcadeDB version from pom.xml
42+
ARCADEDB_TAG=$(python3 extract_version.py --format=docker)
43+
echo "📌 ArcadeDB version: $ARCADEDB_TAG"
44+
45+
# Download JARs from official Docker image
46+
echo "📦 Downloading JARs from arcadedata/arcadedb:$ARCADEDB_TAG..."
47+
48+
mkdir -p src/arcadedb_embedded/jars
49+
50+
# Create temporary container and copy JARs
51+
CONTAINER_ID=$(docker create arcadedata/arcadedb:$ARCADEDB_TAG)
52+
docker cp $CONTAINER_ID:/home/arcadedb/lib/. src/arcadedb_embedded/jars/
53+
docker rm $CONTAINER_ID
54+
55+
ls -lh src/arcadedb_embedded/jars/
56+
JAR_COUNT=$(ls -1 src/arcadedb_embedded/jars/*.jar 2>/dev/null | wc -l)
57+
echo "✅ Downloaded $JAR_COUNT JAR files"
58+
59+
- name: Filter JARs based on exclusions
60+
shell: bash
61+
run: |
62+
cd bindings/python
63+
64+
JARS_DIR="src/arcadedb_embedded/jars"
65+
EXCLUSIONS_FILE="jar_exclusions.txt"
66+
67+
if [[ -f "$EXCLUSIONS_FILE" ]]; then
68+
echo "🗑️ Filtering JARs using $EXCLUSIONS_FILE..."
69+
EXCLUSION_COUNT=0
70+
71+
while IFS= read -r pattern || [[ -n "$pattern" ]]; do
72+
# Skip empty lines and comments
73+
if [[ -n "$pattern" ]] && [[ ! "$pattern" =~ ^# ]]; then
74+
echo " Pattern: $pattern"
75+
76+
# Remove matching JARs
77+
for jar in "$JARS_DIR"/$pattern; do
78+
if [[ -f "$jar" ]]; then
79+
rm -f "$jar"
80+
echo " - Removed: $(basename "$jar")"
81+
((EXCLUSION_COUNT++))
82+
fi
83+
done
84+
fi
85+
done < "$EXCLUSIONS_FILE"
86+
87+
JAR_COUNT_AFTER=$(ls -1 "$JARS_DIR"/*.jar 2>/dev/null | wc -l)
88+
echo "✅ Filtered: removed $EXCLUSION_COUNT JAR(s), $JAR_COUNT_AFTER remaining"
89+
else
90+
echo "⚠️ No jar_exclusions.txt found"
91+
fi
92+
93+
- name: Upload filtered JARs
94+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
95+
with:
96+
name: arcadedb-jars
97+
path: bindings/python/src/arcadedb_embedded/jars/*.jar
98+
retention-days: 1
99+
100+
# Build wheels for all platforms
101+
build:
102+
name: Build Wheel (${{ matrix.platform }})
103+
runs-on: ${{ matrix.runs-on }}
104+
needs: download-jars
105+
strategy:
106+
fail-fast: false
107+
matrix:
108+
include:
109+
- platform: linux/amd64
110+
runs-on: ubuntu-latest
111+
- platform: linux/arm64
112+
runs-on: ubuntu-latest
113+
- platform: darwin/amd64
114+
runs-on: macos-13
115+
- platform: darwin/arm64
116+
runs-on: macos-latest
117+
- platform: windows/amd64
118+
runs-on: windows-latest
119+
120+
steps:
121+
- name: Checkout code
122+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
123+
124+
- name: Set up Python
125+
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
126+
with:
127+
python-version: '3.11'
128+
129+
- name: Download filtered JARs
130+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
131+
with:
132+
name: arcadedb-jars
133+
path: bindings/python/src/arcadedb_embedded/jars
134+
135+
- name: Set up Java (for native builds)
136+
if: ${{ !startsWith(matrix.platform, 'linux/') }}
137+
uses: actions/setup-java@7a6d8a8234af8eb26422e24e3006232cccaa061b # v4.6.0
138+
with:
139+
distribution: 'temurin'
140+
java-version: '21'
141+
142+
- name: Set up QEMU (for linux/arm64)
143+
if: matrix.platform == 'linux/arm64'
144+
uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0
145+
with:
146+
platforms: linux/arm64
147+
148+
- name: Set up Docker Buildx (Linux only)
149+
if: startsWith(matrix.platform, 'linux/')
150+
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
151+
152+
- name: Install Python build tools
153+
shell: bash
154+
run: |
155+
python -m pip install --upgrade pip
156+
pip install build wheel setuptools
157+
158+
- name: Create python3 symlink (Windows)
159+
if: matrix.platform == 'windows/amd64'
160+
shell: bash
161+
run: |
162+
PYTHON_DIR=$(dirname "$(which python)")
163+
ln -s "$PYTHON_DIR/python.exe" "$PYTHON_DIR/python3.exe" || true
164+
python3 --version
165+
166+
- name: Build wheel
167+
shell: bash
168+
env:
169+
BUILD_VERSION: ${{ inputs.version }}
170+
run: |
171+
cd bindings/python
172+
echo "🔨 Building wheel for ${{ matrix.platform }}..."
173+
174+
if [ -n "$BUILD_VERSION" ]; then
175+
echo "📌 Using specified version: $BUILD_VERSION"
176+
fi
177+
178+
./build.sh ${{ matrix.platform }}
179+
180+
- name: Upload wheel
181+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
182+
with:
183+
name: wheel-${{ matrix.platform == 'linux/amd64' && 'linux-amd64' || matrix.platform == 'linux/arm64' && 'linux-arm64' || matrix.platform == 'darwin/amd64' && 'darwin-amd64' || matrix.platform == 'darwin/arm64' && 'darwin-arm64' || 'windows-amd64' }}
184+
path: bindings/python/dist/*.whl
185+
retention-days: 7
186+
187+
# Summary job
188+
summary:
189+
name: Build Summary
190+
needs: build
191+
runs-on: ubuntu-latest
192+
outputs:
193+
wheel-count: ${{ steps.count.outputs.count }}
194+
steps:
195+
- name: Download all wheels
196+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
197+
with:
198+
pattern: wheel-*
199+
path: wheels/
200+
merge-multiple: true
201+
202+
- name: Count wheels
203+
id: count
204+
run: |
205+
WHEEL_COUNT=$(ls -1 wheels/*.whl | wc -l)
206+
echo "count=$WHEEL_COUNT" >> $GITHUB_OUTPUT
207+
echo "📦 Built $WHEEL_COUNT wheels"
208+
209+
- name: Display wheels
210+
run: |
211+
echo "## 🎉 Build Complete!" >> $GITHUB_STEP_SUMMARY
212+
echo "" >> $GITHUB_STEP_SUMMARY
213+
echo "Built **$(ls -1 wheels/*.whl | wc -l)** platform-specific wheels:" >> $GITHUB_STEP_SUMMARY
214+
echo "" >> $GITHUB_STEP_SUMMARY
215+
echo "| Platform | Wheel | Size |" >> $GITHUB_STEP_SUMMARY
216+
echo "|----------|-------|------|" >> $GITHUB_STEP_SUMMARY
217+
218+
for wheel in wheels/*.whl; do
219+
if [ -f "$wheel" ]; then
220+
name=$(basename "$wheel")
221+
size=$(ls -lh "$wheel" | awk '{print $5}')
222+
223+
# Detect platform
224+
if [[ "$name" == *"manylinux"*"x86_64"* ]]; then
225+
platform="linux/amd64"
226+
elif [[ "$name" == *"manylinux"*"aarch64"* ]]; then
227+
platform="linux/arm64"
228+
elif [[ "$name" == *"macosx"*"x86_64"* ]]; then
229+
platform="darwin/amd64"
230+
elif [[ "$name" == *"macosx"*"arm64"* ]]; then
231+
platform="darwin/arm64"
232+
elif [[ "$name" == *"win_amd64"* ]]; then
233+
platform="windows/amd64"
234+
else
235+
platform="unknown"
236+
fi
237+
238+
echo "| $platform | \`$name\` | $size |" >> $GITHUB_STEP_SUMMARY
239+
fi
240+
done

0 commit comments

Comments
 (0)