Skip to content

Use a matrix strategy #134

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
adsharma wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Use a matrix strategy #134

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
241 changes: 107 additions & 134 deletions .github/workflows/release.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,50 @@ permissions:
contents: read
env:
ARTIFACT-LINUX-X86_64-EXTENSION: sqlite-vss-linux-x86_64
ARTIFACT-LINUX-AARCH64-EXTENSION: sqlite-vss-linux-aarch64
ARTIFACT-MACOS-X86_64-EXTENSION: sqlite-vss-macos-x86_64
ARTIFACT-MACOS-AARCH64-EXTENSION: sqlite-vss-macos-aarch64
ARTIFACT-WINDOWS-X86_64-EXTENSION: sqlite-vss-windows-x86_64
ARTIFACT-LINUX-X86_64-WHEELS: sqlite-vss-linux-x86_64-wheels
ARTIFACT-LINUX-AARCH64-WHEELS: sqlite-vss-linux-aarch64-wheels
ARTIFACT-MACOS-X86_64-WHEELS: sqlite-vss-macos-x86_64-wheels
ARTIFACT-MACOS-AARCH64-WHEELS: sqlite-vss-macos-aarch64-wheels
ARTIFACT-DATASETTE-SQLITE-UTILS-WHEELS: sqlite-vss-datasette-sqlite-utils-wheels
jobs:
build-linux-x86_64-extension:
runs-on: ubuntu-20.04
build-extensions:
strategy:
matrix:
include:
- os: ubuntu-20.04
platform: linux
arch: x86_64
artifact_name: sqlite-vss-linux-x86_64
setup_deps: sudo apt-get update -y && sudo apt-get install -y cmake libgomp1
- os: ubuntu-20.04
platform: linux
arch: aarch64
artifact_name: sqlite-vss-linux-aarch64
setup_deps: sudo apt-get update -y && sudo apt-get install -y cmake libgomp1 gcc-aarch64-linux-gnu g++-aarch64-linux-gnu
cc: aarch64-linux-gnu-gcc
cxx: aarch64-linux-gnu-g++
- os: macos-latest
platform: macos
arch: x86_64
artifact_name: sqlite-vss-macos-x86_64
setup_deps: brew install llvm
cc: /usr/local/opt/llvm/bin/clang
cxx: /usr/local/opt/llvm/bin/clang++
ldflags: -L/usr/local/opt/libomp/lib/
cppflags: -I/usr/local/opt/libomp/include/
- os: [self-hosted, mm1]
platform: macos
arch: aarch64
artifact_name: sqlite-vss-macos-aarch64
cc: /opt/homebrew/opt/llvm/bin/clang
cxx: /opt/homebrew/opt/llvm/bin/clang++
ldflags: -L/opt/homebrew/opt/libomp/lib
cppflags: -I/opt/homebrew/opt/libomp/include
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
with:
Expand All @@ -25,128 +59,80 @@ jobs:
uses: actions/cache@v3
with:
path: vendor/sqlite
key: ${{ runner.os }}-${{ hashFiles('vendor/get_sqlite.sh') }}
key: ${{ runner.os }}-${{ matrix.arch }}-${{ hashFiles('vendor/get_sqlite.sh') }}
- if: steps.cache-sqlite-build.outputs.cache-hit != 'true'
run: ./vendor/get_sqlite.sh
- if: steps.cache-sqlite-build.outputs.cache-hit != 'true'
working-directory: vendor/sqlite
run: ./configure && make

# TODO how cache this?
- run: sudo apt-get update -y && sudo apt-get install -y cmake libgomp1
- run: make loadable-release static-release
- uses: actions/upload-artifact@v3
with:
name: ${{ env.ARTIFACT-LINUX-X86_64-EXTENSION }}
path: dist/release/*
build-linux-x86_64-python:
runs-on: ubuntu-20.04
needs: [build-linux-x86_64-extension]
steps:
- uses: actions/checkout@v3
- uses: actions/download-artifact@v3
with:
name: ${{ env.ARTIFACT-LINUX-X86_64-EXTENSION }}
path: dist/release/
- uses: actions/setup-python@v3
- run: pip install wheel
- run: make python-release
- uses: actions/upload-artifact@v3
with:
name: ${{ env.ARTIFACT-LINUX-X86_64-WHEELS }}
path: dist/release/wheels/*.whl
build-macos-x86_64-extension:
runs-on: macos-latest
steps:
- uses: actions/checkout@v3
with:
submodules: recursive
- id: cache-sqlite-build
uses: actions/cache@v3
with:
path: vendor/sqlite
key: ${{ runner.os }}-${{ hashFiles('vendor/get_sqlite.sh') }}
- if: steps.cache-sqlite-build.outputs.cache-hit != 'true'
run: ./vendor/get_sqlite.sh
- if: steps.cache-sqlite-build.outputs.cache-hit != 'true'
working-directory: vendor/sqlite
run: ./configure && make
- run: brew install llvm
- id: cache-cmake-build
- if: matrix.setup_deps
run: ${{ matrix.setup_deps }}
- if: matrix.platform == 'macos' && matrix.arch == 'x86_64'
id: cache-cmake-build
uses: actions/cache@v3
with:
path: build
key: ${{ runner.os }}-build
- run: make patch-openmp
- if: matrix.platform == 'macos'
run: make patch-openmp
- run: make loadable-release static-release
env:
# `brew info libomp` gives the correct one, with .a file for static openmp builds
CC: /usr/local/opt/llvm/bin/clang
CXX: /usr/local/opt/llvm/bin/clang++
LDFLAGS: "-L/usr/local/opt/libomp/lib/"
CPPFLAGS: "-I/usr/local/opt/libomp/include/"
CC: ${{ matrix.cc }}
CXX: ${{ matrix.cxx }}
LDFLAGS: ${{ matrix.ldflags }}
CPPFLAGS: ${{ matrix.cppflags }}
- uses: actions/upload-artifact@v3
with:
name: ${{ env.ARTIFACT-MACOS-X86_64-EXTENSION }}
name: ${{ matrix.artifact_name }}
path: dist/release/*
build-macos-x86_64-python:
runs-on: macos-latest
needs: [build-macos-x86_64-extension]
steps:
- uses: actions/checkout@v3
- uses: actions/download-artifact@v3
with:
name: ${{ env.ARTIFACT-MACOS-X86_64-EXTENSION }}
path: dist/release/
- uses: actions/setup-python@v3
- run: pip install wheel
- run: make python-release
- uses: actions/upload-artifact@v3
with:
name: ${{ env.ARTIFACT-MACOS-X86_64-WHEELS }}
path: dist/release/wheels/*.whl
build-macos-aarch64-extension:
runs-on: [self-hosted, mm1]
steps:
- uses: actions/checkout@v3
with:
submodules: "recursive"
- id: cache-sqlite-build
uses: actions/cache@v3
with:
path: vendor/sqlite
key: ${{ runner.os }}-${{ hashFiles('vendor/get_sqlite.sh') }}
- if: steps.cache-sqlite-build.outputs.cache-hit != 'true'
run: ./vendor/get_sqlite.sh
- if: steps.cache-sqlite-build.outputs.cache-hit != 'true'
working-directory: vendor/sqlite
run: ./configure && make
- run: make patch-openmp
- run: make loadable-release static-release
env:
# `brew info libomp` gives the correct one, with .a file for static openmp builds
CC: /opt/homebrew/opt/llvm/bin/clang
CXX: /opt/homebrew/opt/llvm/bin/clang++
LDFLAGS: "-L/opt/homebrew/opt/libomp/lib"
CPPFLAGS: "-I/opt/homebrew/opt/libomp/include"
- uses: actions/upload-artifact@v3
with:
name: ${{ env.ARTIFACT-MACOS-AARCH64-EXTENSION }}
path: dist/release/*
build-macos-aarch64-python:
runs-on: [self-hosted, mm1]
needs: [build-macos-aarch64-extension]

build-python-wheels:
strategy:
matrix:
include:
- os: ubuntu-20.04
platform: linux
arch: x86_64
extension_artifact: sqlite-vss-linux-x86_64
wheels_artifact: sqlite-vss-linux-x86_64-wheels
python_cmd: pip
make_args: ""
- os: ubuntu-20.04
platform: linux
arch: aarch64
extension_artifact: sqlite-vss-linux-aarch64
wheels_artifact: sqlite-vss-linux-aarch64-wheels
python_cmd: pip
make_args: ""
- os: macos-latest
platform: macos
arch: x86_64
extension_artifact: sqlite-vss-macos-x86_64
wheels_artifact: sqlite-vss-macos-x86_64-wheels
python_cmd: pip
make_args: ""
- os: [self-hosted, mm1]
platform: macos
arch: aarch64
extension_artifact: sqlite-vss-macos-aarch64
wheels_artifact: sqlite-vss-macos-aarch64-wheels
python_cmd: pip3
make_args: "IS_MACOS_ARM=1"
runs-on: ${{ matrix.os }}
needs: [build-extensions]
steps:
- uses: actions/checkout@v3
- uses: actions/download-artifact@v3
with:
name: ${{ env.ARTIFACT-MACOS-AARCH64-EXTENSION }}
name: ${{ matrix.extension_artifact }}
path: dist/release/
- run: pip3 install wheel
- run: make python-release IS_MACOS_ARM=1
- if: matrix.platform == 'linux' || matrix.arch == 'x86_64'
uses: actions/setup-python@v3
- run: ${{ matrix.python_cmd }} install wheel
- run: make python-release ${{ matrix.make_args }}
- uses: actions/upload-artifact@v3
with:
name: ${{ env.ARTIFACT-MACOS-AARCH64-WHEELS }}
name: ${{ matrix.wheels_artifact }}
path: dist/release/wheels/*.whl
build-datasette-sqlite-utils:
runs-on: ubuntu-20.04
Expand All @@ -160,12 +146,7 @@ jobs:
name: ${{ ENV.ARTIFACT-DATASETTE-SQLITE-UTILS-WHEELS }}
path: dist/release/wheels/*.whl
upload-deno:
needs:
[
build-macos-x86_64-extension,
build-macos-aarch64-extension,
build-linux-x86_64-extension,
]
needs: [build-extensions]
permissions:
contents: write
runs-on: ubuntu-latest
Expand All @@ -178,6 +159,7 @@ jobs:
uses: actions/github-script@v6
env:
ARTIFACT-LINUX-X86_64-EXTENSION: ${{ env.ARTIFACT-LINUX-X86_64-EXTENSION }}
ARTIFACT-LINUX-AARCH64-EXTENSION: ${{ env.ARTIFACT-LINUX-AARCH64-EXTENSION }}
ARTIFACT-MACOS-X86_64-EXTENSION: ${{ env.ARTIFACT-MACOS-X86_64-EXTENSION }}
ARTIFACT-MACOS-AARCH64-EXTENSION: ${{ env.ARTIFACT-MACOS-AARCH64-EXTENSION }}
with:
Expand All @@ -187,12 +169,7 @@ jobs:
const script = require('.github/workflows/upload-deno-assets.js')
return await script({github, context})
upload-extensions:
needs:
[
build-macos-x86_64-extension,
build-macos-aarch64-extension,
build-linux-x86_64-extension,
]
needs: [build-extensions]
permissions:
contents: write
runs-on: ubuntu-latest
Expand All @@ -210,6 +187,7 @@ jobs:
macos-x86_64: ${{ env.ARTIFACT-MACOS-X86_64-EXTENSION }}/*
macos-aarch64: ${{ env.ARTIFACT-MACOS-AARCH64-EXTENSION }}/*
linux-x86_64: ${{ env.ARTIFACT-LINUX-X86_64-EXTENSION }}/*
linux-aarch64: ${{ env.ARTIFACT-LINUX-AARCH64-EXTENSION }}/*
upload-checksums:
needs: [upload-extensions, upload-deno]
runs-on: ubuntu-latest
Expand Down Expand Up @@ -256,12 +234,7 @@ jobs:
env:
HEX_API_KEY: ${{ secrets.HEX_API_KEY }}
upload-npm:
needs:
[
build-macos-x86_64-extension,
build-macos-aarch64-extension,
build-linux-x86_64-extension,
]
needs: [build-extensions]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
Expand Down Expand Up @@ -295,19 +268,17 @@ jobs:
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
upload-pypi:
needs:
[
build-linux-x86_64-python,
build-macos-x86_64-python,
build-macos-aarch64-python,
build-datasette-sqlite-utils,
]
needs: [build-python-wheels, build-datasette-sqlite-utils]
runs-on: ubuntu-latest
steps:
- uses: actions/download-artifact@v3
with:
name: ${{ env.ARTIFACT-LINUX-X86_64-WHEELS }}
path: dist
- uses: actions/download-artifact@v3
with:
name: ${{ env.ARTIFACT-LINUX-AARCH64-WHEELS }}
path: dist
- uses: actions/download-artifact@v3
with:
name: ${{ env.ARTIFACT-MACOS-X86_64-WHEELS }}
Expand All @@ -325,12 +296,7 @@ jobs:
password: ${{ secrets.PYPI_API_TOKEN }}
skip-existing: true
upload-gem:
needs:
[
build-macos-x86_64-extension,
build-macos-aarch64-extension,
build-linux-x86_64-extension,
]
needs: [build-extensions]
permissions:
contents: write
runs-on: ubuntu-latest
Expand Down Expand Up @@ -358,10 +324,17 @@ jobs:
gem -C bindings/ruby build -o x86_64-linux.gem sqlite_vss.gemspec
env:
PLATFORM: x86_64-linux
- run: |
rm bindings/ruby/lib/*.{dylib,so,dll} || true
cp ${{ env.ARTIFACT-LINUX-AARCH64-EXTENSION }}/*.so bindings/ruby/lib
gem -C bindings/ruby build -o aarch64-linux.gem sqlite_vss.gemspec
env:
PLATFORM: aarch64-linux
- run: |
gem push bindings/ruby/x86_64-darwin.gem
gem push bindings/ruby/arm64-darwin.gem
gem push bindings/ruby/x86_64-linux.gem
gem push bindings/ruby/aarch64-linux.gem
env:
GEM_HOST_API_KEY: ${{ secrets.GEM_HOST_API_KEY }}
upload-crate:
Expand Down