Skip to content

Commit d8db8cb

Browse files
rbeucherrbeucher
committed
Enhance CI/CD workflow: Add version extraction and artifact handling for PyPI and conda builds
1 parent 85c9c08 commit d8db8cb

1 file changed

Lines changed: 103 additions & 27 deletions

File tree

‎.github/workflows/cd.yml‎

Lines changed: 103 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@ jobs:
1717
runs-on: ubuntu-latest
1818
permissions:
1919
id-token: write
20+
outputs:
21+
# Output the version for downstream jobs
22+
version: ${{ steps.get-version.outputs.version }}
2023
steps:
2124
- name: Checkout source
2225
uses: actions/checkout@v6.0.1
@@ -31,6 +34,19 @@ jobs:
3134
run: |
3235
git clean -xdf
3336
pyproject-build
37+
- name: Get version for artifact naming
38+
id: get-version
39+
run: |
40+
VERSION=$(python -c "import versioneer; print(versioneer.get_version())")
41+
VERSION=${VERSION#v}
42+
echo "version=${VERSION}" >> $GITHUB_OUTPUT
43+
echo "Building version: ${VERSION}"
44+
- name: Upload build artifacts
45+
uses: actions/upload-artifact@v4
46+
with:
47+
name: python-package-distributions-${{ steps.get-version.outputs.version }}
48+
path: dist/
49+
retention-days: 7
3450
- name: Publish package to PyPI
3551
uses: pypa/gh-action-pypi-publish@release/v1
3652
with:
@@ -40,36 +56,78 @@ jobs:
4056
name: build and deploy to conda
4157
needs: pypi
4258
runs-on: ubuntu-latest
59+
# This job uses artifacts from the PyPI job instead of waiting for PyPI propagation
60+
# This makes the build more reliable and faster
4361
steps:
4462
- name: Checkout source
4563
uses: actions/checkout@v6.0.1
46-
- name: Wait for PyPI propagation
64+
- name: Download build artifacts from PyPI job
65+
uses: actions/download-artifact@v4
66+
with:
67+
name: python-package-distributions-${{ needs.pypi.outputs.version }}
68+
path: dist/
69+
- name: Verify artifacts
4770
run: |
48-
echo "Waiting for PyPI package to be downloadable..."
49-
PACKAGE_NAME="access_moppy"
50-
VERSION=$(python -c "import versioneer; print(versioneer.get_version())")
51-
VERSION=${VERSION#v}
52-
echo "Looking for package: ${PACKAGE_NAME}==${VERSION}"
53-
for i in {1..90}; do
54-
echo "Attempt $i/90: Trying to download wheel and sdist..."
55-
rm -rf /tmp/pypi_check_download && mkdir -p /tmp/pypi_check_download
56-
pip download --no-deps --dest /tmp/pypi_check_download ${PACKAGE_NAME}==${VERSION} >/dev/null 2>&1
57-
pip download --no-deps --no-binary=:all: --dest /tmp/pypi_check_download ${PACKAGE_NAME}==${VERSION} >/dev/null 2>&1
58-
WHEEL=$(ls /tmp/pypi_check_download/*.whl 2>/dev/null | head -1)
59-
SDIST=$(ls /tmp/pypi_check_download/*.tar.gz 2>/dev/null | head -1)
60-
if [[ -n "$WHEEL" && -n "$SDIST" ]]; then
61-
echo "✅ Both wheel and sdist for ${PACKAGE_NAME}==${VERSION} are downloadable!"
62-
break
63-
fi
64-
if [ $i -eq 90 ]; then
65-
echo "❌ Could not download both wheel and sdist after 15 minutes"
66-
echo "Available files:"
67-
ls -l /tmp/pypi_check_download || echo "No files downloaded"
68-
exit 1
69-
fi
70-
echo "Files not yet downloadable, waiting 10 seconds..."
71-
sleep 10
72-
done
71+
echo "Downloaded artifacts:"
72+
ls -la dist/
73+
echo "Checking for wheel and sdist files..."
74+
WHEEL=$(ls dist/*.whl 2>/dev/null | head -1)
75+
SDIST=$(ls dist/*.tar.gz 2>/dev/null | head -1)
76+
if [[ -n "$WHEEL" && -n "$SDIST" ]]; then
77+
echo "✅ Both wheel and sdist found in artifacts!"
78+
echo "Wheel: $(basename $WHEEL)"
79+
echo "Sdist: $(basename $SDIST)"
80+
else
81+
echo "❌ Missing wheel or sdist in artifacts"
82+
exit 1
83+
fi
84+
- name: Prepare conda recipe for local build
85+
run: |
86+
echo "Preparing conda recipe to use local source..."
87+
# Create a modified meta.yaml that uses local source
88+
cp .conda/meta.yaml .conda/meta.yaml.backup
89+
cat > .conda/meta.yaml << 'EOF'
90+
{% set data = load_setup_py_data(setup_file='../setup.py', from_recipe_dir=True) %}
91+
{% set version = data.get('version') %}
92+
93+
package:
94+
name: access-moppy
95+
version: "{{ version }}"
96+
97+
source:
98+
path: ../
99+
100+
build:
101+
noarch: python
102+
number: 0
103+
script: "{{ PYTHON }} -m pip install . -vv"
104+
105+
requirements:
106+
host:
107+
- python
108+
- pip
109+
- versioneer
110+
run:
111+
- python >=3.11
112+
- xarray
113+
- numpy
114+
- dask
115+
- pyyaml
116+
- cftime
117+
118+
about:
119+
home: https://github.com/ACCESS-NRI/ACCESS-MOPPy
120+
summary: A Python package for CMORisation of ACCESS model output
121+
license: Apache-2.0
122+
license_file: LICENCE.txt
123+
doc_url: https://access-moppy.readthedocs.io/
124+
dev_url: https://github.com/ACCESS-NRI/ACCESS-MOPPy
125+
126+
extra:
127+
recipe-maintainers:
128+
- ACCESS-NRI
129+
EOF
130+
echo "Modified conda recipe to use local source (path: ../)"
73131
- name: Setup conda environment
74132
uses: conda-incubator/setup-miniconda@v3
75133
with:
@@ -87,10 +145,23 @@ jobs:
87145
user: accessnri
88146
label: main
89147
token: ${{ secrets.anaconda_token }}
148+
- name: Verify conda package build
149+
run: |
150+
echo "Conda build completed successfully!"
151+
echo "Version built: ${{ needs.pypi.outputs.version }}"
152+
echo "✅ Conda package should now be available in the accessnri channel"
153+
- name: Restore original conda recipe
154+
if: always() # Run even if build fails
155+
run: |
156+
if [ -f .conda/meta.yaml.backup ]; then
157+
echo "Restoring original conda recipe..."
158+
mv .conda/meta.yaml.backup .conda/meta.yaml
159+
echo "✅ Original conda recipe restored"
160+
fi
90161
91162
update_analysis3:
92163
name: Update ACCESS-MOPPy on Analysis3 Conda Environment
93-
needs: conda
164+
needs: [pypi, conda]
94165
runs-on: ubuntu-latest
95166
permissions:
96167
contents: read
@@ -110,12 +181,17 @@ jobs:
110181
run: |
111182
if [ -n "${INPUT_VERSION}" ]; then
112183
ver="${INPUT_VERSION}"
184+
elif [ -n "${{ needs.pypi.outputs.version }}" ]; then
185+
# Use version from PyPI job output (more reliable)
186+
ver="${{ needs.pypi.outputs.version }}"
113187
else
188+
# Fallback to parsing tag
114189
tag="${RELEASE_TAG}"
115190
ver="${tag#moppy-}"
116191
ver="${ver#v}"
117192
fi
118193
echo "version=$ver" >> $GITHUB_OUTPUT
194+
echo "Using version: $ver"
119195
- name: Install tooling (jq, gh, Python deps)
120196
run: |
121197
sudo apt-get update -y

0 commit comments

Comments
 (0)