Skip to content

Commit 9379fcf

Browse files
vcschappvcschapp
committed
Auto-publish Python packages to PyPI on main branch version change
This commit adds a GitHub Actions workflow that automatically publishes the Python schema packages to PyPI when they have a version number change that is pushed to the main branch.* Packages are published in topological order, which is for now determined by the hard-coded `def level` function in `scripts/package-versions.py`. Eventually, we also want to auto-cut a GitHub Release after a successful publish, but let's do that later. *: For now, PyPI = Overture internal CodeArtifact repo, but eventually it will be public PyPI.
1 parent 5ac1834 commit 9379fcf

16 files changed

Lines changed: 254 additions & 75 deletions

File tree

‎.github/workflows/publish-python-packages.yaml‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
name: Publish Python packages to PyPI
2+
3+
on:
4+
push:
5+
branches: [main]
6+
paths:
7+
- '**/pyproject.toml'
8+
- 'packages/**/__about__.py'
9+
inputs:
10+
aws_iam_role_name:
11+
description: The name of the IAM role to assume for accessing CodeArtifact
12+
type: string
13+
required: false
14+
default: GithubActions_Schema_CodeArtifact_Publish
15+
domain:
16+
description: The CodeArtifact domain name
17+
type: string
18+
required: false
19+
default: overture-pypi
20+
repository:
21+
description: The CodeArtifact repository name
22+
type: string
23+
required: false
24+
default: overture
25+
26+
permissions:
27+
id-token: write
28+
contents: read
29+
30+
jobs:
31+
check:
32+
if: github.event.repository.full_name == github.repository
33+
uses: ./.github/workflows/reusable-check-python-package-versions.yaml
34+
with:
35+
before_commit: ${{ github.event.before }}
36+
after_commit: ${{ github.event.after }}
37+
38+
publish:
39+
needs: [check]
40+
if: github.event.repository.full_name == github.repository && needs.check.outputs.num_changed_packages > 0
41+
runs-on: ubuntu-latest
42+
strategy:
43+
matrix:
44+
include: ${{ fromJson(needs.check.outputs.changed_packages) }}
45+
steps:
46+
- name: Install uv
47+
uses: astral-sh/setup-uv@v4
48+
with:
49+
version: latest
50+
51+
- name: Check out code
52+
uses: actions/checkout@v4
53+
54+
- name: Sync code to make packages visible to Python
55+
run: uv sync --all-packages
56+
57+
- name: Configure AWS credentials
58+
uses: aws-actions/configure-aws-credentials@v4
59+
with:
60+
aws-region: us-west-2
61+
role-to-assume: arn:aws:iam::505071440022:role/GithubActions_Schema_CodeArtifact_Publish
62+
role-session-name: GitHubActions_${{github.job}}_${{github.run_id}}
63+
64+
- name: Get CodeArtifact publish URL
65+
id: get-code-artifact-params
66+
run: |
67+
echo 'token<<EOF' >> $GITHUB_OUTPUT
68+
./.github/workflows/scripts/code-artifact.sh token \
69+
505071440022 us-west-2 overture-pypi >> $GITHUB_OUTPUT
70+
echo EOF >> $GITHUB_OUTPUT
71+
echo 'publish_url<<EOF' >> $GITHUB_OUTPUT
72+
./.github/workflows/scripts/code-artifact.sh publish-url \
73+
505071440022 us-west-2 overture-pypi overture >> $GITHUB_OUTPUT
74+
echo EOF >> $GITHUB_OUTPUT
75+
76+
- name: Publish package ${{ matrix.package }} version ${{ matrix.after }} to PyPI
77+
run: |
78+
package="${{ matrix.package }}"
79+
before="${{ matrix.before }}"
80+
after="${{ matrix.after }}"
81+
printf 'Publishing package %s version %s to PyPI (previous version %s)...\n' "$package" "$after" "$before"
82+
uv build --package "$package"
83+
wheel="dist/${package//-/_}-${after}-py3-none-any.whl"
84+
if [ ! -f "$wheel" ]; then
85+
echo " Wheel file [$wheel] not found. Aborting!"
86+
exit 1
87+
fi
88+
tarball="dist/${package//-/_}-${after}.tar.gz"
89+
if [ ! -f "$tarball" ]; then
90+
echo " Source tarball file [$tarball] not found. Aborting!"
91+
exit 1
92+
fi
93+
uv publish "$wheel" "$tarball" \
94+
-t "${{ steps.get-code-artifact-params.outputs.token }}" \
95+
--publish-url "${{ steps.get-code-artifact-params.outputs.publish_url }}"

‎.github/workflows/reusable-check-python-package-versions.yaml‎

Lines changed: 67 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,22 +15,56 @@ on:
1515
PR or the latest commit in a push.
1616
type: string
1717
required: true
18+
aws_account_id:
19+
description: The AWS account ID that owns the CodeArtifact domain
20+
type: string
21+
required: false
22+
default: 505071440022
23+
aws_region:
24+
description: The AWS region where the CodeArtifact repository is hosted
25+
type: string
26+
required: false
27+
default: us-west-2
28+
aws_iam_role_name:
29+
description: The name of the IAM role to assume for accessing CodeArtifact
30+
type: string
31+
required: false
32+
default: GithubActions_Schema_CodeArtifact_ReadOnly
33+
domain:
34+
description: The CodeArtifact domain name
35+
type: string
36+
required: false
37+
default: overture-pypi
38+
repository:
39+
description: The CodeArtifact repository name
40+
type: string
41+
required: false
42+
default: overture
43+
outputs:
44+
changed_packages:
45+
description: >-
46+
A JSON array of packages with changed versions, including package name, old version, and
47+
new version, in the format: `[ {"package": "p1", "before": "v1", "after": "v2"}, ... ]`
48+
value: ${{ jobs.check-python-package-versions.outputs.changed_packages }}
49+
num_changed_packages:
50+
description: The number of packages with changed versions
51+
value: ${{ jobs.check-python-package-versions.outputs.num_changed_packages }}
1852

19-
jobs:
20-
get-index-url:
21-
uses: ./.github/workflows/reusable-get-code-artifact-index-url.yaml
2253

54+
jobs:
2355
check-python-package-versions:
24-
needs: get-index-url
2556
runs-on: ubuntu-latest
57+
outputs:
58+
changed_packages: ${{ steps.save-changes.outputs.changed_packages }}
59+
num_changed_packages: ${{ steps.save-changes.outputs.num_changed_packages }}
2660
steps:
2761
- name: Install jq
2862
run: sudo apt-get update && sudo apt-get install -y jq
2963

3064
- name: Install uv
3165
uses: astral-sh/setup-uv@v4
3266
with:
33-
version: "latest"
67+
version: latest
3468

3569
- name: Set up Python
3670
uses: actions/setup-python@v5
@@ -69,13 +103,40 @@ jobs:
69103
- name: Print changed versions
70104
run: cat /tmp/package-version-diff.json
71105

106+
- name: Save changed versions as output
107+
id: save-changes
108+
run: |
109+
echo 'changed_packages<<EOF' >> $GITHUB_OUTPUT
110+
cat /tmp/package-version-diff.json >> $GITHUB_OUTPUT
111+
echo EOF >> $GITHUB_OUTPUT
112+
printf 'num_changed_packages=%s\n' "$(jq -c '. | length' /tmp/package-version-diff.json)" >> $GITHUB_OUTPUT
113+
114+
- name: Configure AWS credentials
115+
if: steps.save-changes.outputs.num_changed_packages > 0
116+
uses: aws-actions/configure-aws-credentials@v4
117+
with:
118+
aws-region: ${{ inputs.aws_region }}
119+
role-to-assume: arn:aws:iam::${{ inputs.aws_account_id }}:role/${{ inputs.aws_iam_role_name }}
120+
role-session-name: GitHubActions_${{github.job}}_${{github.run_id}}
121+
122+
- name: Get CodeArtifact index URL
123+
id: get-code-artifact-index-url
124+
if: steps.save-changes.outputs.num_changed_packages > 0
125+
run: |
126+
echo 'index_url<<EOF' >> $GITHUB_OUTPUT
127+
./.github/workflows/scripts/code-artifact.sh index-url \
128+
"${{ inputs.aws_account_id }}" "${{ inputs.aws_region }}" \
129+
"${{ inputs.domain }}" "${{ inputs.repository }}" >> $GITHUB_OUTPUT
130+
echo EOF >> $GITHUB_OUTPUT
131+
72132
- name: Fail if any of the new versions already exist in the repo
133+
if: steps.save-changes.outputs.num_changed_packages > 0
73134
run: |
74135
jq -c '.[]' /tmp/package-version-diff.json | while read -r entry; do
75136
package=$(echo "$entry" | jq -r '.package')
76137
after=$(echo "$entry" | jq -r '.after')
77138
exit_code=0
78-
output=$(uv run pip download "${package}==${after}" --index-url "${{ needs.get-index-url.outputs.index_url }}simple/" --no-deps -d /tmp --quiet 2>&1) || exit_code=$?
139+
output=$(uv run pip download "${package}==${after}" --index-url "${{ steps.get-code-artifact-index-url.outputs.index_url }}" --no-deps -d /tmp --quiet 2>&1) || exit_code=$?
79140
if [[ $exit_code -eq 0 || (
80141
"${output,,}" != *"could not find a version"* &&
81142
"${output,,}" != *"no matching distributions"*

‎.github/workflows/reusable-get-code-artifact-index-url.yaml‎

Lines changed: 0 additions & 57 deletions
This file was deleted.

‎.github/workflows/scripts/code-artifact.sh‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
readonly subcommand="$1"
6+
7+
function token() {
8+
local -r aws_account_id="$1"
9+
local -r aws_region="$2"
10+
local -r domain="$3"
11+
12+
aws codeartifact get-authorization-token \
13+
--region "$aws_region" \
14+
--domain "$domain" \
15+
--domain-owner "$aws_account_id" \
16+
--query authorizationToken \
17+
--output text
18+
}
19+
20+
function repo_url() {
21+
local -r token="$1"
22+
local -r credentials="${token:+aws:$token@}"
23+
local -r aws_account_id="$2"
24+
local -r aws_region="$3"
25+
local -r domain="$4"
26+
local -r repository="$5"
27+
local -r suffix="$6"
28+
29+
printf "https://%s%s-%s.d.codeartifact.%s.amazonaws.com/pypi/%s%s\n" \
30+
"$credentials" "$domain" "$aws_account_id" "$aws_region" "$repository" "$suffix"
31+
}
32+
33+
case "$subcommand" in
34+
token)
35+
if [ $# -ne 4 ]; then
36+
>&2 echo "Usage: $0 token <aws_account_id> <aws_region> <domain>"
37+
exit 1
38+
fi
39+
token "$2" "$3" "$4"
40+
;;
41+
42+
index-url|publish-url)
43+
if [ $# -ne 5 ]; then
44+
>&2 echo "Usage: $0 $subcommand <aws_account_id> <aws_region> <domain> <repository>"
45+
exit 1
46+
fi
47+
48+
if [ "$subcommand" = "index-url" ]; then
49+
repo_url "$(token "$2" "$3" "$4")" "$2" "$3" "$4" "$5" "/simple/"
50+
else
51+
repo_url "" "$2" "$3" "$4" "$5" ""
52+
fi
53+
;;
54+
55+
*)
56+
>&2 echo "Unknown subcommand: ${subcommand:-<missing>}"
57+
>&2 echo "Valid subcommands: token | index-url | publish-url"
58+
exit 1
59+
;;
60+
esac

‎.github/workflows/scripts/package-versions.py‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
from importlib import metadata
44
from pathlib import Path
55
import json
6+
import re
67
import sys
78

89

@@ -35,6 +36,9 @@ def compare(before_file: str, after_file: str):
3536
Compare two JSON files containing package versions and print the packages that have a version
3637
number change as a JSON array.
3738
39+
The output JSON array is sorted in topological order by package name, so those changed packages
40+
that do not depend on other changed packages appear first.
41+
3842
Form of the JSON array:
3943
4044
[ {"package": "p1", "before": "v1", "after": "v2"}, ... ]
@@ -48,7 +52,23 @@ def compare(before_file: str, after_file: str):
4852
before_dict = {item["package"]: item["version"] for item in before_array}
4953
after_dict = {item["package"]: item["version"] for item in after_array}
5054

51-
combined_keys = sorted(list(set(before_dict.keys()) | set(after_dict.keys())))
55+
def level(package: str) -> int:
56+
"""
57+
Return the level of a package for topological sorting.
58+
59+
This is brittle and hard to keep in sync, so we should replace it with a version that
60+
dynamically computes dependencies in the future.
61+
"""
62+
if package == "overture-schema-system":
63+
return 0
64+
elif package in ["overture-schema-core"]:
65+
return 1
66+
elif re.fullmatch(r'overture-schema-.*-theme', package) or package in ["overture-schema", "overture-schema-cli", "overture-schema-annex"]:
67+
return 2
68+
else:
69+
raise ValueError(f"Unknown package for level computation: {package}")
70+
71+
combined_keys = sorted(list(set(before_dict.keys()) | set(after_dict.keys())), key=level)
5272

5373
changed_packages = []
5474
for package in combined_keys:

‎packages/overture-schema-addresses-theme/src/overture/schema/addresses/__about__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
__version__ = "0.1.0"
1+
__version__ = "0.1.1.dev1"

‎packages/overture-schema-annex/src/overture/schema/__about__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
__version__ = "0.1.0"
1+
__version__ = "0.1.1.dev1"

‎packages/overture-schema-base-theme/src/overture/schema/base/__about__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
__version__ = "0.1.0"
1+
__version__ = "0.1.1.dev1"

‎packages/overture-schema-buildings-theme/src/overture/schema/buildings/__about__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
__version__ = "0.1.0"
1+
__version__ = "0.1.1.dev1"

‎packages/overture-schema-cli/src/overture/schema/cli/__about__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
__version__ = "0.1.0"
1+
__version__ = "0.1.1.dev1"

0 commit comments

Comments
 (0)