Skip to content

Commit fc564fb

Browse files
moratommoratom
authored
Merge pull request #1787 from luxonis/feat/artifacts_keeper_upload
Feat/artifacts keeper upload
2 parents f7314a7 + f254fed commit fc564fb

6 files changed

Lines changed: 132 additions & 119 deletions

File tree

‎.github/workflows/python-main.yml‎

Lines changed: 25 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -400,8 +400,6 @@ jobs:
400400
run: echo "BUILD_COMMIT_HASH=${{github.sha}}" >> $GITHUB_ENV
401401
- name: Install combined wheel and run a smoke-test
402402
if: startsWith(github.ref, 'refs/tags/v') != true
403-
env:
404-
ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
405403
run: |
406404
set -euo pipefail
407405
@@ -417,13 +415,6 @@ jobs:
417415
python -m pip install --force-reinstall $wheel
418416
419417
EXPECTED_VERSION="$ver" python bindings/python/ci/smoke_depthai.py
420-
- name: Upload combined wheels to artifactory
421-
if: github.event_name == 'workflow_dispatch' || github.event_name == 'push'
422-
run: cd bindings/python && bash ./ci/upload-artifactory.sh --snapshot
423-
env:
424-
ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
425-
ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }}
426-
ARTIFACTORY_PASS: ${{ secrets.ARTIFACTORY_PASS }}
427418
428419
# This job builds wheels for x86_64 arch
429420
build-linux-x86_64:
@@ -527,8 +518,6 @@ jobs:
527518
run: echo "BUILD_COMMIT_HASH=${{github.sha}}" >> $GITHUB_ENV
528519
- name: Install combined wheel and run a smoke-test
529520
if: startsWith(github.ref, 'refs/tags/v') != true
530-
env:
531-
ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
532521
run: |
533522
set -euo pipefail
534523
@@ -547,13 +536,6 @@ jobs:
547536
548537
# Smoke test: fail hard on any exception or version mismatch
549538
EXPECTED_VERSION="$ver" "$PYBIN" bindings/python/ci/smoke_depthai.py
550-
- name: Upload combined wheels to artifactory
551-
if: github.event_name == 'workflow_dispatch' || github.event_name == 'push'
552-
run: cd bindings/python && bash ./ci/upload-artifactory.sh --snapshot
553-
env:
554-
ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
555-
ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }}
556-
ARTIFACTORY_PASS: ${{ secrets.ARTIFACTORY_PASS }}
557539
558540
# This job builds wheels for ARM64 arch
559541
build-linux-arm64:
@@ -661,8 +643,6 @@ jobs:
661643
run: echo "BUILD_COMMIT_HASH=${{github.sha}}" >> $GITHUB_ENV
662644
- name: Install combined wheel and run a smoke-test
663645
if: startsWith(github.ref, 'refs/tags/v') != true
664-
env:
665-
ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
666646
run: |
667647
set -euo pipefail
668648
@@ -681,13 +661,6 @@ jobs:
681661
"$PYBIN" -m pip install --force-reinstall $wheel
682662
683663
EXPECTED_VERSION="$ver" "$PYBIN" bindings/python/ci/smoke_depthai.py
684-
- name: Upload combined wheels to artifactory
685-
if: github.event_name == 'workflow_dispatch' || github.event_name == 'push'
686-
run: cd bindings/python && bash ./ci/upload-artifactory.sh --snapshot
687-
env:
688-
ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
689-
ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }}
690-
ARTIFACTORY_PASS: ${{ secrets.ARTIFACTORY_PASS }}
691664
692665
combine-windows-x86_64-wheels:
693666
needs: build-windows-x86_64
@@ -723,8 +696,6 @@ jobs:
723696
- name: Install combined wheel and run a smoke-test
724697
if: startsWith(github.ref, 'refs/tags/v') != true
725698
shell: pwsh
726-
env:
727-
ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
728699
run: |
729700
$env:PYTHONIOENCODING = 'utf-8'
730701
$ErrorActionPreference = "Stop" # Fail on any error
@@ -743,17 +714,29 @@ jobs:
743714
# Smoke test (no heredoc; YAML-safe). Fail on import error or version mismatch.
744715
$env:EXPECTED_VERSION = $ver
745716
python bindings/python/ci/smoke_depthai.py
746-
- name: Upload combined wheels to artifactory
747-
if: github.event_name == 'workflow_dispatch' || github.event_name == 'push'
748-
run: cd bindings/python && bash ./ci/upload-artifactory.sh --snapshot
717+
718+
upload-wheels-to-ak:
719+
if: github.event_name == 'workflow_dispatch' || github.event_name == 'push'
720+
needs: [combine-macos-wheels, combine-linux-x86_64-wheels, combine-linux-arm64-wheels, combine-windows-x86_64-wheels]
721+
runs-on: ubuntu-latest
722+
steps:
723+
- uses: actions/checkout@v3
724+
725+
- uses: actions/download-artifact@v4
726+
with:
727+
pattern: audited-wheels-combined-*
728+
merge-multiple: true
729+
path: bindings/python/wheelhouse/audited/
730+
731+
- name: Upload combined wheels to Artifact Keeper
732+
run: cd bindings/python && bash ./ci/upload-ak.sh --snapshot
749733
env:
750-
ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
751-
ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }}
752-
ARTIFACTORY_PASS: ${{ secrets.ARTIFACTORY_PASS }}
734+
AK_URL: ${{ secrets.AK_URL }}
735+
AK_TOKEN: ${{ secrets.AK_TOKEN }}
753736

754737
notify-slack:
755738
if: github.event_name == 'workflow_dispatch' || github.event_name == 'push'
756-
needs: [combine-macos-wheels, combine-linux-x86_64-wheels, combine-linux-arm64-wheels, combine-windows-x86_64-wheels]
739+
needs: [upload-wheels-to-ak]
757740
runs-on: ubuntu-latest
758741
steps:
759742
- uses: actions/checkout@v3
@@ -816,7 +799,7 @@ jobs:
816799
- name: Check if version matches
817800
run: cd bindings/python && python3.8 -c 'import find_version as v; exit(0) if "${{ github.ref_name }}" == f"v{v.get_package_version()}" else exit(1)'
818801

819-
# Deploy to PyPi and Artifactory. Only when a commit is tagged
802+
# Deploy to PyPI and Artifact Keeper. Only when a commit is tagged
820803
deploy:
821804
if: startsWith(github.ref, 'refs/tags/v')
822805
needs: [release]
@@ -839,12 +822,11 @@ jobs:
839822
PYPI_SERVER: ${{ secrets.PYPI_SERVER }}
840823
PYPI_USER: ${{ secrets.PYPI_USER }}
841824
PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
842-
- name: Run deploy to Artifactory
843-
run: cd bindings/python && bash ./ci/upload-artifactory.sh --release
825+
- name: Run deploy to Artifact Keeper
826+
run: cd bindings/python && bash ./ci/upload-ak.sh --release
844827
env:
845-
ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
846-
ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }}
847-
ARTIFACTORY_PASS: ${{ secrets.ARTIFACTORY_PASS }}
828+
AK_URL: ${{ secrets.AK_URL }}
829+
AK_TOKEN: ${{ secrets.AK_TOKEN }}
848830

849831
dispatch-bom-tests:
850832
needs: [build-linux-x86_64, deploy]
@@ -911,4 +893,4 @@ jobs:
911893
if [ "$CONCLUSION" != "success" ] && [ "$CONCLUSION" != "skipped" ]; then
912894
echo "Failed job"
913895
exit 1
914-
fi
896+
fi

‎.github/workflows/zoo_helper.workflow.yml‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -53,13 +53,12 @@ jobs:
5353
- name: Strip zoo_helper
5454
run: strip ./build/zoo_helper # Shrinks the size of the binary a little bit by removing symbols: https://www.man7.org/linux/man-pages/man1/strip.1.html
5555

56-
- name: Upload zoo_helper to artifactory
56+
- name: Upload zoo_helper to Artifact Keeper
5757
run: |
58-
bash ./ci/upload-artifactory-zoo-helper.sh
58+
bash ./ci/upload-ak-zoo-helper.sh
5959
env:
60-
ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
61-
ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }}
62-
ARTIFACTORY_PASS: ${{ secrets.ARTIFACTORY_PASS }}
60+
AK_URL: ${{ secrets.AK_URL }}
61+
AK_TOKEN: ${{ secrets.AK_TOKEN }}
6362
ZOO_HELPER_PLATFORM: linux-x86_64
6463

6564
build-zoo-helper-linux-arm64:
@@ -109,11 +108,10 @@ jobs:
109108
- name: Strip zoo_helper
110109
run: strip ./build/zoo_helper # Shrinks the size of the binary a little bit by removing symbols: https://www.man7.org/linux/man-pages/man1/strip.1.html
111110

112-
- name: Upload zoo_helper to artifactory
111+
- name: Upload zoo_helper to Artifact Keeper
113112
run: |
114-
bash ./ci/upload-artifactory-zoo-helper.sh
113+
bash ./ci/upload-ak-zoo-helper.sh
115114
env:
116-
ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
117-
ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }}
118-
ARTIFACTORY_PASS: ${{ secrets.ARTIFACTORY_PASS }}
115+
AK_URL: ${{ secrets.AK_URL }}
116+
AK_TOKEN: ${{ secrets.AK_TOKEN }}
119117
ZOO_HELPER_PLATFORM: linux-arm64

‎bindings/python/ci/upload-ak.sh‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
#!/bin/bash
2+
set -euo pipefail
3+
4+
: "${AK_URL:?AK_URL must be set}"
5+
: "${AK_TOKEN:?AK_TOKEN must be set}"
6+
7+
case "$1" in
8+
--snapshot)
9+
AK_REPO="luxonis-python-snapshot-local"
10+
;;
11+
--release)
12+
AK_REPO="luxonis-python-release-local"
13+
;;
14+
*)
15+
echo "Error: Unknown option $1"
16+
echo "Usage: $0 [--snapshot | --release]"
17+
exit 1
18+
;;
19+
esac
20+
21+
AK_BASE_URL="${AK_URL%/}"
22+
AK_PYPI_REPOSITORY_URL="${AK_BASE_URL}/pypi/${AK_REPO}/"
23+
AK_PYPI_SIMPLE_URL="${AK_BASE_URL}/pypi/${AK_REPO}/simple/depthai/"
24+
25+
python3 -m pip install -U pip
26+
python3 -m pip install -U twine
27+
python3 -m pip install -U packaging
28+
29+
shopt -s nullglob
30+
wheels=(wheelhouse/audited/*.whl)
31+
if [ "${#wheels[@]}" -eq 0 ]; then
32+
echo "No wheels found in wheelhouse/audited/"
33+
exit 1
34+
fi
35+
36+
for wheel in "${wheels[@]}"; do
37+
echo "Uploading ${wheel}"
38+
python3 -m twine upload \
39+
--repository-url "${AK_PYPI_REPOSITORY_URL}" \
40+
--username "__token__" \
41+
--password "${AK_TOKEN}" \
42+
"${wheel}"
43+
done
44+
45+
index_html="$(mktemp)"
46+
trap 'rm -f "${index_html}"' EXIT
47+
curl -fsSL "${AK_PYPI_SIMPLE_URL}" -o "${index_html}"
48+
49+
for wheel in "${wheels[@]}"; do
50+
wheel_name="$(basename "${wheel}")"
51+
if ! grep -Fq "${wheel_name}" "${index_html}"; then
52+
echo "Uploaded wheel is missing from the PyPI simple index: ${wheel_name}"
53+
exit 1
54+
fi
55+
done

‎bindings/python/ci/upload-artifactory.sh‎

Lines changed: 0 additions & 42 deletions
This file was deleted.

‎ci/upload-ak-zoo-helper.sh‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/bin/bash
2+
set -euo pipefail
3+
4+
: "${AK_URL:?AK_URL must be set}"
5+
: "${AK_TOKEN:?AK_TOKEN must be set}"
6+
: "${ZOO_HELPER_PLATFORM:?ZOO_HELPER_PLATFORM must be set}"
7+
8+
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
9+
10+
AK_REPO="luxonis-depthai-helper-binaries"
11+
12+
# Set paths
13+
export PATH_PREFIX="zoo_helper/$ZOO_HELPER_PLATFORM"
14+
export ZOO_HELPER_BINARY_LOCAL_PATH="build/zoo_helper"
15+
16+
# Get git hash
17+
git config --global --add safe.directory "$(pwd)"
18+
export ZOO_HELPER_GIT_HASH
19+
ZOO_HELPER_GIT_HASH="$(git rev-parse HEAD)"
20+
21+
UPLOAD_PATH="$PATH_PREFIX/$ZOO_HELPER_GIT_HASH/zoo_helper"
22+
23+
echo "----------------------------------------"
24+
echo "AK_REPO: $AK_REPO"
25+
echo "PATH_PREFIX: $PATH_PREFIX"
26+
echo "ZOO_HELPER_BINARY_LOCAL_PATH: $ZOO_HELPER_BINARY_LOCAL_PATH"
27+
echo "ZOO_HELPER_GIT_HASH: $ZOO_HELPER_GIT_HASH"
28+
echo "zoo_helper binary size: $(du -sh "$ZOO_HELPER_BINARY_LOCAL_PATH")"
29+
echo "zoo_helper upload path: $AK_REPO:$UPLOAD_PATH"
30+
echo "----------------------------------------"
31+
32+
if ! command -v ak >/dev/null 2>&1; then
33+
curl -fsSL https://raw.githubusercontent.com/artifact-keeper/artifact-keeper-cli/main/install.sh | sh
34+
fi
35+
36+
export AK_NO_INPUT="${AK_NO_INPUT:-true}"
37+
38+
AK_INSTANCE="${AK_INSTANCE:-ci}"
39+
AK_BASE_URL="${AK_URL%/}"
40+
if ! ak instance list --format quiet | grep -Fxq "$AK_INSTANCE"; then
41+
ak instance add "$AK_INSTANCE" "$AK_BASE_URL"
42+
fi
43+
44+
ak artifact push "$AK_REPO" "$ZOO_HELPER_BINARY_LOCAL_PATH" --path "$UPLOAD_PATH"

‎ci/upload-artifactory-zoo-helper.sh‎

Lines changed: 0 additions & 24 deletions
This file was deleted.

0 commit comments

Comments
 (0)