Release #6
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Release is called by duckdb's InvokeCI -> NotifyExternalRepositories job | |
| name: Release | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| duckdb-python-sha: | |
| type: string | |
| description: The commit to build against (defaults to latest commit of current ref) | |
| required: false | |
| duckdb-sha: | |
| type: string | |
| description: The DuckDB submodule commit or ref to build against | |
| required: true | |
| stable-version: | |
| type: string | |
| description: Release a stable version (vX.Y.Z-((rc|post)N)) | |
| required: false | |
| pypi-index: | |
| type: choice | |
| description: Which PyPI to use | |
| required: true | |
| options: | |
| - test | |
| - prod | |
| nightly-stale-after-days: | |
| type: string | |
| description: After how many days should nightlies be considered stale | |
| required: true | |
| default: 4 | |
| store-s3: | |
| type: boolean | |
| description: Also store test packages in S3 (always true for prod) | |
| default: false | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| build_sdist: | |
| name: Build an sdist and determine versions | |
| uses: ./.github/workflows/packaging_sdist.yml | |
| with: | |
| testsuite: none | |
| duckdb-python-sha: ${{ inputs.duckdb-python-sha != '' && inputs.duckdb-python-sha || github.sha }} | |
| duckdb-sha: ${{ inputs.duckdb-sha }} | |
| set-version: ${{ inputs.stable-version }} | |
| workflow_state: | |
| name: Set state for the release workflow | |
| needs: build_sdist | |
| outputs: | |
| pypi_state: ${{ steps.index_check.outputs.pypi_state }} | |
| ci_env: ${{ steps.ci_env_check.outputs.ci_env }} | |
| s3_url: ${{ steps.s3_check.outputs.s3_url }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - id: index_check | |
| name: Check version on PyPI | |
| run: | | |
| set -ex | |
| pypi_hostname=${{ inputs.pypi-index == 'test' && 'test.' || '' }}pypi.org | |
| # install duckdb | |
| curl https://install.duckdb.org | sh | |
| # query pypi | |
| result=$(cat <<EOF | ${HOME}/.duckdb/cli/latest/duckdb | xargs | |
| ---- Output lines | |
| .mode line | |
| ---- Query that fetches the given version's age, if the version already exists | |
| SELECT | |
| today() - (file.value->>'upload_time_iso_8601')::DATE AS age, | |
| FROM read_json('https://${pypi_hostname}/pypi/duckdb/json') AS jd | |
| CROSS JOIN json_each(jd.releases) AS rel(key, value) | |
| CROSS JOIN unnest(FROM_JSON(rel.value, '["JSON"]')) AS file(value) | |
| WHERE rel.key='${{ needs.build_sdist.outputs.package-version }}' | |
| LIMIT 1; | |
| EOF | |
| ) | |
| if [ -z "$result" ]; then | |
| pypi_state=VERSION_NOT_FOUND | |
| elif [ "${result#age = }" -ge "${{ inputs.nightly-stale-after-days }}" ]; then | |
| pypi_state=VERSION_STALE | |
| else | |
| pypi_state=VERSION_FOUND | |
| fi | |
| echo "pypi_state=${pypi_state}" >> $GITHUB_OUTPUT | |
| - id: ci_env_check | |
| name: Determine CI environment | |
| run: | | |
| set -eu | |
| if [[ test == "${{ inputs.pypi-index }}" ]]; then | |
| ci_env=pypi-test | |
| elif [[ prod == "${{ inputs.pypi-index }}" ]]; then | |
| ci_env=pypi-prod${{ inputs.stable-version == '' && '-nightly' || '' }} | |
| else | |
| echo "::error::Invalid value for inputs.pypi-index: ${{ inputs.pypi-index }}" | |
| exit 1 | |
| fi | |
| echo "ci_env=${ci_env}" >> "$GITHUB_OUTPUT" | |
| echo "::notice::Using CI environment ${ci_env}" | |
| - id: s3_check | |
| name: Generate S3 upload URL | |
| if: github.repository_owner == 'duckdb' | |
| run: | | |
| set -eu | |
| should_store=${{ (inputs.pypi-index == 'prod' || inputs.store-s3) && '1' || '0' }} | |
| if [[ $should_store == 0 ]]; then | |
| echo "::notice::S3 upload disabled in inputs, not generating S3 URL" | |
| exit 0 | |
| fi | |
| if [[ VERSION_NOT_FOUND != "${{ steps.index_check.outputs.pypi_state }}" ]]; then | |
| echo "::warning::S3 upload disabled because package version already uploaded to PyPI" | |
| exit 0 | |
| fi | |
| sha=${{ github.sha }} | |
| dsha=${{ inputs.duckdb-sha }} | |
| version=${{ needs.build_sdist.outputs.package-version }} | |
| s3_url="s3://duckdb-staging/python/${version}/${sha:0:10}-duckdb-${dsha:0:10}/" | |
| echo "::notice::Generated S3 URL: ${s3_url}" | |
| echo "s3_url=${s3_url}" >> $GITHUB_OUTPUT | |
| bump_submodule: | |
| name: Bump submodule to given SHA if version on PyPI is stale | |
| needs: workflow_state | |
| if: ${{ needs.workflow_state.outputs.pypi_state == 'VERSION_STALE' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout DuckDB Python | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.duckdb-python-sha }} | |
| fetch-depth: 0 | |
| submodules: true | |
| - name: Create PR to bump DuckDB submodule | |
| shell: bash | |
| if: ${{ inputs.duckdb-sha }} | |
| run: | | |
| cd external/duckdb | |
| git fetch origin | |
| git checkout ${{ inputs.duckdb-sha }} | |
| cd ../../ | |
| git add external/duckdb | |
| git commit -m "Bump submodule" | |
| run_url=https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| msg="Created from ${run_url}. Version on PyPI is older than ${{ inputs.nightly-stale-after-days }} days." | |
| gh pr create -B ${{ github.ref_name }} --title "Bump submodule" --body "${msg}" | |
| build_wheels: | |
| name: Build and test releases | |
| needs: workflow_state | |
| if: ${{ needs.workflow_state.outputs.pypi_state == 'VERSION_NOT_FOUND' }} | |
| uses: ./.github/workflows/packaging_wheels.yml | |
| with: | |
| minimal: false | |
| testsuite: all | |
| duckdb-python-sha: ${{ inputs.duckdb-python-sha != '' && inputs.duckdb-python-sha || github.sha }} | |
| duckdb-sha: ${{ inputs.duckdb-sha }} | |
| set-version: ${{ inputs.stable-version }} | |
| upload_s3: | |
| name: Upload Artifacts to S3 | |
| runs-on: ubuntu-latest | |
| needs: [build_sdist, build_wheels, workflow_state] | |
| if: ${{ needs.workflow_state.outputs.s3_url }} | |
| steps: | |
| - name: Fetch artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: '{sdist,wheel}*' | |
| path: artifacts/ | |
| merge-multiple: true | |
| - name: Upload Artifacts | |
| env: | |
| AWS_ENDPOINT_URL: ${{ secrets.S3_DUCKDB_STAGING_ENDPOINT }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} | |
| run: | | |
| aws s3 cp artifacts ${{ needs.workflow_state.outputs.s3_url }} --recursive | |
| publish_pypi: | |
| name: Publish Artifacts to PyPI | |
| runs-on: ubuntu-latest | |
| needs: [workflow_state, build_sdist, build_wheels] | |
| environment: | |
| name: ${{ needs.workflow_state.outputs.ci_env }} | |
| permissions: | |
| # this is needed for the OIDC flow that is used with trusted publishing on PyPI | |
| id-token: write | |
| steps: | |
| - if: ${{ vars.PYPI_HOST == '' }} | |
| run: | | |
| echo "Error: PYPI_HOST is not set in CI environment '${{ needs.workflow_state.outputs.ci_env }}'" | |
| exit 1 | |
| - name: Fetch artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: '{sdist,wheel}*' | |
| path: packages/ | |
| merge-multiple: true | |
| - name: Upload artifacts to PyPI | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| repository-url: 'https://${{ vars.PYPI_HOST }}/legacy/' | |
| packages-dir: packages | |
| verbose: 'true' | |
| cleanup_nightlies: | |
| name: Remove Nightlies from PyPI | |
| needs: [workflow_state, publish_pypi] | |
| if: ${{ inputs.stable-version == '' }} | |
| uses: ./.github/workflows/cleanup_pypi.yml | |
| with: | |
| environment: ${{ needs.workflow_state.outputs.ci_env }} | |
| secrets: | |
| # reusable workflows and secrets are not great: https://github.com/actions/runner/issues/3206 | |
| PYPI_CLEANUP_OTP: ${{secrets.PYPI_CLEANUP_OTP}} | |
| PYPI_CLEANUP_PASSWORD: ${{secrets.PYPI_CLEANUP_PASSWORD}} | |
| summary: | |
| name: Release summary | |
| runs-on: ubuntu-latest | |
| needs: [build_sdist, workflow_state, build_wheels, upload_s3, publish_pypi, cleanup_nightlies] | |
| if: always() | |
| steps: | |
| - run: | | |
| sha=${{ github.sha }} | |
| dsha=${{ inputs.duckdb-sha }} | |
| pversion=${{ needs.build_sdist.outputs.package-version }} | |
| long_pversion="${pversion} (${sha:0:10})" | |
| pypi_host=${{ inputs.pypi-index == 'test' && 'test.' || '' }}pypi.org | |
| pypi_duckdb_url=https://${pypi_host}/project/duckdb/${pversion}/ | |
| was_released=${{ needs.publish_pypi.result == 'success' && '1' || '0' }} | |
| if [[ $was_released == 1 ]]; then | |
| echo "## Version ${long_pversion} successfully released" >> $GITHUB_STEP_SUMMARY | |
| echo "* Package URL: [${pypi_duckdb_url}](${pypi_duckdb_url})" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "## Version ${long_pversion} was not released" >> $GITHUB_STEP_SUMMARY | |
| echo "* Package index state before release: ${{ needs.workflow_state.outputs.pypi_state }}" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| echo "* Package index: ${pypi_host}" >> $GITHUB_STEP_SUMMARY | |
| echo "* Vendored DuckDB Version: ${{ needs.build_sdist.outputs.duckdb-version }} (${dsha:0:10})" >> $GITHUB_STEP_SUMMARY | |
| echo "* S3 upload status: ${{ needs.upload_s3.result == 'success' && needs.workflow_state.outputs.s3_url || needs.upload_s3.result }}" >> $GITHUB_STEP_SUMMARY | |
| echo "* CI Environment: ${{ needs.workflow_state.outputs.ci_env }}" >> $GITHUB_STEP_SUMMARY |