Skip to content

Commit d5d58b8

Browse files
committed
fix: ci pipelines
1 parent 6128c53 commit d5d58b8

12 files changed

Lines changed: 411 additions & 214 deletions
Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
#!/usr/bin/env python3
2+
"""Detect which packages have changed in a PR or push to main."""
3+
4+
import json
5+
import os
6+
import subprocess
7+
import sys
8+
from pathlib import Path
9+
10+
11+
def get_all_packages() -> list[str]:
12+
"""Get all packages in the monorepo."""
13+
packages_dir = Path("packages")
14+
packages = []
15+
16+
for item in packages_dir.iterdir():
17+
if item.is_dir() and (item / "pyproject.toml").exists():
18+
packages.append(item.name)
19+
20+
return sorted(packages)
21+
22+
23+
def get_changed_packages(base_sha: str, head_sha: str) -> list[str]:
24+
"""Get packages that have changed between two commits."""
25+
try:
26+
# Get changed files
27+
result = subprocess.run(
28+
["git", "diff", "--name-only", f"{base_sha}...{head_sha}"],
29+
capture_output=True,
30+
text=True,
31+
check=True,
32+
)
33+
34+
changed_files = result.stdout.strip().split("\n")
35+
36+
# Extract package names from paths like "packages/uipath-llamaindex/..."
37+
changed_packages = set()
38+
for file_path in changed_files:
39+
if file_path.startswith("packages/"):
40+
parts = file_path.split("/")
41+
if len(parts) >= 2:
42+
package_name = parts[1]
43+
# Verify it's a real package
44+
if (Path("packages") / package_name / "pyproject.toml").exists():
45+
changed_packages.add(package_name)
46+
47+
return sorted(changed_packages)
48+
49+
except subprocess.CalledProcessError as e:
50+
print(f"Error running git diff: {e}", file=sys.stderr)
51+
return []
52+
53+
54+
def main():
55+
"""Main entry point."""
56+
event_name = os.getenv("GITHUB_EVENT_NAME", "")
57+
base_sha = os.getenv("BASE_SHA", "")
58+
head_sha = os.getenv("HEAD_SHA", "")
59+
60+
# On push to main, test all packages
61+
if event_name == "push":
62+
packages = get_all_packages()
63+
print(f"Push to main - testing all {len(packages)} packages:")
64+
for pkg in packages:
65+
print(f" - {pkg}")
66+
67+
# On PR, detect changed packages
68+
elif event_name == "pull_request":
69+
if not base_sha or not head_sha:
70+
print(
71+
"Error: BASE_SHA and HEAD_SHA must be set for pull requests",
72+
file=sys.stderr,
73+
)
74+
sys.exit(1)
75+
76+
packages = get_changed_packages(base_sha, head_sha)
77+
print(f"Pull request - detected {len(packages)} changed package(s):")
78+
for pkg in packages:
79+
print(f" - {pkg}")
80+
81+
else:
82+
print(f"Unknown event: {event_name}", file=sys.stderr)
83+
sys.exit(1)
84+
85+
# Output as JSON for GitHub Actions
86+
packages_json = json.dumps(packages)
87+
print(f"\nPackages JSON: {packages_json}")
88+
89+
# Write to GitHub output
90+
github_output = os.getenv("GITHUB_OUTPUT")
91+
if github_output:
92+
with open(github_output, "a") as f:
93+
f.write(f"packages={packages_json}\n")
94+
f.write(f"count={len(packages)}\n")
95+
96+
97+
if __name__ == "__main__":
98+
main()

.github/workflows/ci.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ jobs:
1919
uses: ./.github/workflows/commitlint.yml
2020

2121
lint:
22-
uses: ./.github/workflows/lint.yml
22+
uses: ./.github/workflows/lint-packages.yml
2323

2424
test:
25-
uses: ./.github/workflows/test.yml
25+
uses: ./.github/workflows/test-packages.yml

.github/workflows/discover-packages.yml

Lines changed: 0 additions & 37 deletions
This file was deleted.

.github/workflows/discover-testcases.yml

Lines changed: 0 additions & 47 deletions
This file was deleted.

.github/workflows/integration_tests.yml

Lines changed: 72 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,89 @@ name: Integration testing
33
on:
44
push:
55
branches: [ main, develop ]
6+
paths:
7+
- 'packages/**'
8+
- '.github/workflows/integration_tests.yml'
9+
- '.github/scripts/detect_changed_packages.py'
610
pull_request:
711
branches: [ main ]
12+
paths:
13+
- 'packages/**'
14+
- '.github/workflows/integration_tests.yml'
15+
- '.github/scripts/detect_changed_packages.py'
816

917
permissions:
1018
contents: read
1119
pull-requests: write
1220
actions: read
1321

1422
jobs:
23+
detect-changed-packages:
24+
runs-on: ubuntu-latest
25+
outputs:
26+
packages: ${{ steps.detect.outputs.packages }}
27+
count: ${{ steps.detect.outputs.count }}
28+
steps:
29+
- name: Checkout
30+
uses: actions/checkout@v4
31+
with:
32+
fetch-depth: 0
33+
34+
- name: Setup Python
35+
uses: actions/setup-python@v5
36+
with:
37+
python-version: '3.11'
38+
39+
- name: Detect changed packages
40+
id: detect
41+
run: |
42+
python .github/scripts/detect_changed_packages.py
43+
1544
discover-testcases:
16-
uses: ./.github/workflows/discover-testcases.yml
45+
needs: detect-changed-packages
46+
if: needs.detect-changed-packages.outputs.count > 0
47+
runs-on: ubuntu-latest
48+
outputs:
49+
testcases: ${{ steps.discover.outputs.testcases }}
50+
steps:
51+
- name: Checkout
52+
uses: actions/checkout@v4
53+
54+
- name: Discover testcases for changed packages
55+
id: discover
56+
run: |
57+
# Get list of packages to check
58+
PACKAGES='${{ needs.detect-changed-packages.outputs.packages }}'
59+
60+
# Find all testcases in changed packages only
61+
testcases_array="[]"
62+
63+
for package_name in $(echo "$PACKAGES" | jq -r '.[]'); do
64+
testcases_dir="packages/$package_name/testcases"
65+
66+
if [ -d "$testcases_dir" ]; then
67+
echo "Discovering testcases in $package_name"
68+
69+
# Find all testcase folders in this package
70+
for testcase_dir in "$testcases_dir"/*; do
71+
if [ -d "$testcase_dir" ]; then
72+
testcase_name=$(basename "$testcase_dir")
73+
# Skip 'common' directory if it exists
74+
if [ "$testcase_name" != "common" ]; then
75+
echo " Found: $testcase_name"
76+
testcases_array=$(echo "$testcases_array" | jq -c ". += [{\"package\": \"$package_name\", \"testcase\": \"$testcase_name\"}]")
77+
fi
78+
fi
79+
done
80+
fi
81+
done
82+
83+
echo "Testcases matrix: $testcases_array"
84+
echo "testcases=$testcases_array" >> $GITHUB_OUTPUT
85+
1786
integration-tests:
18-
needs: [discover-testcases]
87+
needs: [detect-changed-packages, discover-testcases]
88+
if: needs.discover-testcases.outputs.testcases != '[]'
1989
runs-on: ubuntu-latest
2090
timeout-minutes: 10
2191
container:

.github/workflows/lint-custom-version.yml

Lines changed: 32 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,26 +4,50 @@ on:
44
workflow_call:
55
pull_request:
66
types: [opened, synchronize, labeled, unlabeled]
7+
paths:
8+
- 'packages/**'
9+
- '.github/workflows/lint-custom-version.yml'
10+
- '.github/scripts/detect_changed_packages.py'
11+
12+
permissions:
13+
contents: read
14+
pull-requests: read
715

816
jobs:
9-
discover-packages:
17+
detect-changed-packages:
1018
if: contains(github.event.pull_request.labels.*.name, 'test-core-dev-version')
11-
uses: ./.github/workflows/discover-packages.yml
19+
runs-on: ubuntu-latest
20+
outputs:
21+
packages: ${{ steps.detect.outputs.packages }}
22+
count: ${{ steps.detect.outputs.count }}
23+
steps:
24+
- name: Checkout
25+
uses: actions/checkout@v4
26+
with:
27+
fetch-depth: 0
28+
29+
- name: Setup Python
30+
uses: actions/setup-python@v5
31+
with:
32+
python-version: '3.11'
33+
34+
- name: Detect changed packages
35+
id: detect
36+
run: |
37+
python .github/scripts/detect_changed_packages.py
1238
1339
lint-with-custom-version:
1440
name: Lint ${{ matrix.package }} - Custom Version
15-
needs: [discover-packages]
41+
needs: detect-changed-packages
42+
if: contains(github.event.pull_request.labels.*.name, 'test-core-dev-version') && needs.detect-changed-packages.outputs.count > 0
1643
runs-on: ubuntu-latest
17-
if: contains(github.event.pull_request.labels.*.name, 'test-core-dev-version')
1844
defaults:
1945
run:
2046
working-directory: packages/${{ matrix.package }}
2147
strategy:
48+
fail-fast: false
2249
matrix:
23-
package: ${{ fromJson(needs.discover-packages.outputs.packages) }}
24-
permissions:
25-
contents: read
26-
pull-requests: read
50+
package: ${{ fromJson(needs.detect-changed-packages.outputs.packages) }}
2751

2852
steps:
2953
- name: Checkout

0 commit comments

Comments
 (0)