Skip to content

Commit fea395b

Browse files
NathanFlurryNathanFlurry
committed
ci: speed up publish workflow
1 parent d9179cc commit fea395b

12 files changed

Lines changed: 414 additions & 135 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,8 @@ jobs:
6363
targets: wasm32-wasip1
6464
- uses: Swatinem/rust-cache@v2
6565
with:
66-
workspaces: registry/native -> registry/native/target
66+
workspaces: registry/native -> target
67+
cache-workspace-crates: true
6768
key: wasm-commands-${{ hashFiles('registry/native/Cargo.lock') }}
6869
- run: pnpm install --frozen-lockfile --filter '@rivet-dev/agentos-runtime-core...'
6970
- run: make -C registry/native commands

‎.github/workflows/publish.yaml‎

Lines changed: 94 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ env:
2121
R2_BUCKET: rivet-releases
2222
R2_ENDPOINT: https://2a94c6a0ced8d35ea63cddc86c2681e7.r2.cloudflarestorage.com
2323
SIDECAR_PLATFORMS: "linux-x64-gnu linux-arm64-gnu darwin-x64 darwin-arm64"
24-
PUBLISH_INCLUDE_REGISTRY_PACKAGES: "1"
24+
RUST_TOOLCHAIN: "1.91.1"
2525
LINUX_GNU_LLVM_VERSION: "22"
2626
LINUX_GNU_SYSROOT_TAG: "sysroot-20250207"
2727

@@ -66,7 +66,8 @@ jobs:
6666
targets: wasm32-wasip1
6767
- uses: Swatinem/rust-cache@v2
6868
with:
69-
workspaces: registry/native -> registry/native/target
69+
workspaces: registry/native -> target
70+
cache-workspace-crates: true
7071
key: wasm-commands-${{ hashFiles('registry/native/Cargo.lock') }}
7172
- run: pnpm install --frozen-lockfile --filter '@rivet-dev/agentos-runtime-core...'
7273
- run: make -C registry/native commands
@@ -79,141 +80,87 @@ jobs:
7980

8081
build-sidecar:
8182
needs: [context]
82-
name: "Build sidecars (${{ matrix.platform }})"
83+
name: "Build linux native artifacts (${{ matrix.platform }})"
8384
strategy:
8485
fail-fast: false
8586
matrix:
8687
include:
8788
- platform: linux-x64-gnu
88-
runner: [self-hosted, agentos-builder]
89+
runner: ubuntu-22.04
8990
target: x86_64-unknown-linux-gnu
9091
- platform: linux-arm64-gnu
9192
runner: ubuntu-22.04-arm
9293
target: aarch64-unknown-linux-gnu
9394
runs-on: ${{ matrix.runner }}
9495
steps:
9596
- uses: actions/checkout@v4
96-
- uses: pnpm/action-setup@v4
97-
- uses: actions/setup-node@v4
98-
with:
99-
node-version: 22
100-
cache: pnpm
101-
cache-dependency-path: pnpm-lock.yaml
102-
- uses: dtolnay/rust-toolchain@stable
103-
with:
104-
targets: ${{ matrix.target }}
105-
- uses: Swatinem/rust-cache@v2
106-
with:
107-
workspaces: . -> target
108-
key: ${{ matrix.target }}-${{ env.LINUX_GNU_SYSROOT_TAG }}-llvm-${{ env.LINUX_GNU_LLVM_VERSION }}-${{ needs.context.outputs.trigger }}
109-
- uses: actions/cache@v4
110-
with:
111-
path: ~/.cargo/.rusty_v8
112-
key: ${{ runner.os }}-${{ matrix.target }}-rusty-v8-${{ hashFiles('Cargo.lock') }}
113-
restore-keys: |
114-
${{ runner.os }}-${{ matrix.target }}-rusty-v8-
115-
- run: pnpm install --frozen-lockfile --filter='!@agentos/website'
116-
- name: Set up Linux GNU sysroot
117-
run: scripts/ci/setup-linux-gnu-sysroot.sh
118-
- name: Build sidecar binaries
119-
id: build
97+
- uses: docker/setup-buildx-action@v3
98+
- name: Resolve build profile
99+
id: mode
120100
run: |
121101
set -euo pipefail
122-
out="target/sidecar-artifacts/${{ matrix.platform }}"
123-
mkdir -p "$out"
124102
if [ "${{ needs.context.outputs.trigger }}" = "release" ]; then
125-
flag="--release"
126-
profile="release"
103+
echo "profile=release" >> "$GITHUB_OUTPUT"
127104
else
128-
flag=""
129-
profile="debug"
105+
echo "profile=debug" >> "$GITHUB_OUTPUT"
130106
fi
131-
cargo build $flag -p agentos-sidecar -p agentos-native-sidecar --target ${{ matrix.target }}
132-
cp "target/${{ matrix.target }}/${profile}/agentos-sidecar" "$out/agentos-sidecar"
133-
cp "target/${{ matrix.target }}/${profile}/agentos-native-sidecar" "$out/agentos-native-sidecar"
134-
echo "dir=$out" >> "$GITHUB_OUTPUT"
107+
- name: Build linux artifacts
108+
uses: docker/build-push-action@v6
109+
with:
110+
context: .
111+
file: docker/build/linux-gnu.Dockerfile
112+
build-args: |
113+
TARGET=${{ matrix.target }}
114+
BUILD_PROFILE=${{ steps.mode.outputs.profile }}
115+
CACHE_PLATFORM=${{ matrix.platform }}
116+
RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }}
117+
LINUX_GNU_LLVM_VERSION=${{ env.LINUX_GNU_LLVM_VERSION }}
118+
LINUX_GNU_SYSROOT_TAG=${{ env.LINUX_GNU_SYSROOT_TAG }}
119+
tags: agentos-linux-${{ matrix.platform }}
120+
load: true
121+
cache-from: type=gha,scope=linux-${{ matrix.platform }}-${{ steps.mode.outputs.profile }}
122+
cache-to: type=gha,mode=max,scope=linux-${{ matrix.platform }}-${{ steps.mode.outputs.profile }}
123+
- name: Extract linux artifacts
124+
id: extract
125+
run: |
126+
set -euo pipefail
127+
sidecar_out="target/sidecar-artifacts/${{ matrix.platform }}"
128+
plugin_out="target/plugin-artifacts/${{ matrix.platform }}"
129+
mkdir -p "$sidecar_out" "$plugin_out"
130+
cid=$(docker create agentos-linux-${{ matrix.platform }})
131+
docker cp "$cid:/artifacts/agentos-sidecar" "$sidecar_out/agentos-sidecar"
132+
docker cp "$cid:/artifacts/agentos-native-sidecar" "$sidecar_out/agentos-native-sidecar"
133+
docker cp "$cid:/artifacts/libagentos_actor_plugin.so" "$plugin_out/libagentos_actor_plugin.so"
134+
docker rm "$cid"
135+
test -f "$sidecar_out/agentos-sidecar"
136+
test -f "$sidecar_out/agentos-native-sidecar"
137+
test -f "$plugin_out/libagentos_actor_plugin.so"
138+
echo "sidecar_dir=$sidecar_out" >> "$GITHUB_OUTPUT"
139+
echo "plugin_dir=$plugin_out" >> "$GITHUB_OUTPUT"
135140
- name: Check glibc floor
136141
run: |
137142
scripts/ci/check-linux-glibc-floor.sh \
138-
"${{ steps.build.outputs.dir }}/agentos-sidecar" \
139-
"${{ steps.build.outputs.dir }}/agentos-native-sidecar"
140-
- name: Smoke-run in old Linux containers
143+
"${{ steps.extract.outputs.sidecar_dir }}/agentos-sidecar" \
144+
"${{ steps.extract.outputs.sidecar_dir }}/agentos-native-sidecar" \
145+
"${{ steps.extract.outputs.plugin_dir }}/libagentos_actor_plugin.so"
146+
- name: Smoke-run binaries in old Linux containers
141147
run: |
142148
scripts/ci/smoke-linux-artifacts.sh binary \
143-
"${{ steps.build.outputs.dir }}/agentos-sidecar" \
144-
"${{ steps.build.outputs.dir }}/agentos-native-sidecar"
149+
"${{ steps.extract.outputs.sidecar_dir }}/agentos-sidecar" \
150+
"${{ steps.extract.outputs.sidecar_dir }}/agentos-native-sidecar"
151+
- name: Smoke-run plugin in old Linux containers
152+
run: |
153+
scripts/ci/smoke-linux-artifacts.sh shared-library \
154+
"${{ steps.extract.outputs.plugin_dir }}/libagentos_actor_plugin.so"
145155
- uses: actions/upload-artifact@v4
146156
with:
147157
name: sidecar-${{ matrix.platform }}
148-
path: ${{ steps.build.outputs.dir }}
158+
path: ${{ steps.extract.outputs.sidecar_dir }}
149159
if-no-files-found: error
150-
151-
build-plugin:
152-
needs: [context]
153-
name: "Build plugin (${{ matrix.platform }})"
154-
strategy:
155-
fail-fast: false
156-
matrix:
157-
include:
158-
- platform: linux-x64-gnu
159-
runner: ubuntu-22.04
160-
target: x86_64-unknown-linux-gnu
161-
- platform: linux-arm64-gnu
162-
runner: ubuntu-22.04-arm
163-
target: aarch64-unknown-linux-gnu
164-
runs-on: ${{ matrix.runner }}
165-
steps:
166-
- uses: actions/checkout@v4
167-
- uses: pnpm/action-setup@v4
168-
- uses: actions/setup-node@v4
169-
with:
170-
node-version: 22
171-
cache: pnpm
172-
cache-dependency-path: pnpm-lock.yaml
173-
- uses: dtolnay/rust-toolchain@stable
174-
with:
175-
targets: ${{ matrix.target }}
176-
- uses: Swatinem/rust-cache@v2
177-
with:
178-
workspaces: . -> target
179-
key: plugin-${{ matrix.target }}-${{ env.LINUX_GNU_SYSROOT_TAG }}-llvm-${{ env.LINUX_GNU_LLVM_VERSION }}-${{ needs.context.outputs.trigger }}
180-
- uses: actions/cache@v4
181-
with:
182-
path: ~/.cargo/.rusty_v8
183-
key: ${{ runner.os }}-${{ matrix.target }}-rusty-v8-${{ hashFiles('Cargo.lock') }}
184-
restore-keys: |
185-
${{ runner.os }}-${{ matrix.target }}-rusty-v8-
186-
- run: pnpm install --frozen-lockfile --filter='!@agentos/website'
187-
- name: Set up Linux GNU sysroot
188-
run: scripts/ci/setup-linux-gnu-sysroot.sh
189-
- name: Build plugin cdylib
190-
id: build
191-
run: |
192-
set -euo pipefail
193-
out="target/plugin-artifacts/${{ matrix.platform }}"
194-
mkdir -p "$out"
195-
if [ "${{ needs.context.outputs.trigger }}" = "release" ]; then
196-
cargo build --release -p agentos-actor-plugin --target ${{ matrix.target }}
197-
profile="release"
198-
else
199-
cargo build -p agentos-actor-plugin --target ${{ matrix.target }}
200-
profile="debug"
201-
fi
202-
cp "target/${{ matrix.target }}/${profile}/libagentos_actor_plugin.so" \
203-
"$out/libagentos_actor_plugin.so"
204-
echo "dir=$out" >> "$GITHUB_OUTPUT"
205-
- name: Check glibc floor
206-
run: |
207-
scripts/ci/check-linux-glibc-floor.sh \
208-
"${{ steps.build.outputs.dir }}/libagentos_actor_plugin.so"
209-
- name: Smoke-run in old Linux containers
210-
run: |
211-
scripts/ci/smoke-linux-artifacts.sh shared-library \
212-
"${{ steps.build.outputs.dir }}/libagentos_actor_plugin.so"
213160
- uses: actions/upload-artifact@v4
214161
with:
215162
name: plugin-${{ matrix.platform }}
216-
path: ${{ steps.build.outputs.dir }}
163+
path: ${{ steps.extract.outputs.plugin_dir }}
217164
if-no-files-found: error
218165

219166
build-sidecar-darwin:
@@ -232,19 +179,36 @@ jobs:
232179
runs-on: ubuntu-22.04
233180
steps:
234181
- uses: actions/checkout@v4
182+
- uses: docker/setup-buildx-action@v3
235183
- name: Log in to ghcr.io
236184
run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
185+
- name: Compute build profile
186+
id: mode
187+
run: |
188+
set -euo pipefail
189+
if [ "${{ needs.context.outputs.trigger }}" = "release" ]; then
190+
echo "profile=release" >> "$GITHUB_OUTPUT"
191+
else
192+
echo "profile=debug" >> "$GITHUB_OUTPUT"
193+
fi
237194
- name: Cross-compile via osxcross
195+
uses: docker/build-push-action@v6
196+
with:
197+
context: .
198+
file: docker/build/darwin.Dockerfile
199+
build-args: |
200+
TARGET=${{ matrix.target }}
201+
CLANG=${{ matrix.clang }}
202+
BUILD_PROFILE=${{ steps.mode.outputs.profile }}
203+
CACHE_PLATFORM=${{ matrix.platform }}
204+
RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }}
205+
tags: agentos-darwin-${{ matrix.platform }}
206+
load: true
207+
cache-from: type=gha,scope=darwin-${{ matrix.platform }}-${{ steps.mode.outputs.profile }}
208+
cache-to: type=gha,mode=max,scope=darwin-${{ matrix.platform }}-${{ steps.mode.outputs.profile }}
209+
- name: Extract darwin artifacts
238210
run: |
239211
set -euo pipefail
240-
if [ "${{ needs.context.outputs.trigger }}" = "release" ]; then profile=release; else profile=debug; fi
241-
DOCKER_BUILDKIT=1 docker build \
242-
-f docker/build/darwin.Dockerfile \
243-
--build-arg TARGET=${{ matrix.target }} \
244-
--build-arg CLANG=${{ matrix.clang }} \
245-
--build-arg BUILD_PROFILE="${profile}" \
246-
-t agentos-darwin-${{ matrix.platform }} .
247-
248212
sidecar_out="target/sidecar-artifacts/${{ matrix.platform }}"
249213
plugin_out="target/plugin-artifacts/${{ matrix.platform }}"
250214
mkdir -p "$sidecar_out" "$plugin_out"
@@ -256,6 +220,14 @@ jobs:
256220
test -f "$sidecar_out/agentos-sidecar"
257221
test -f "$sidecar_out/agentos-native-sidecar"
258222
test -f "$plugin_out/libagentos_actor_plugin.dylib"
223+
case "${{ matrix.platform }}" in
224+
darwin-x64) expected_arch="x86_64" ;;
225+
darwin-arm64) expected_arch="arm64" ;;
226+
*) echo "unknown darwin platform: ${{ matrix.platform }}" >&2; exit 2 ;;
227+
esac
228+
file "$sidecar_out/agentos-sidecar" | grep -F "$expected_arch"
229+
file "$sidecar_out/agentos-native-sidecar" | grep -F "$expected_arch"
230+
file "$plugin_out/libagentos_actor_plugin.dylib" | grep -F "$expected_arch"
259231
- uses: actions/upload-artifact@v4
260232
with:
261233
name: sidecar-${{ matrix.platform }}
@@ -268,9 +240,9 @@ jobs:
268240
if-no-files-found: error
269241

270242
publish-npm:
271-
needs: [context, wasm-commands, build-sidecar, build-plugin, build-sidecar-darwin]
243+
needs: [context, wasm-commands, build-sidecar, build-sidecar-darwin]
272244
name: Publish npm
273-
if: ${{ !cancelled() && needs.wasm-commands.result == 'success' && needs.build-sidecar.result == 'success' && needs.build-plugin.result == 'success' && needs.build-sidecar-darwin.result == 'success' }}
245+
if: ${{ !cancelled() && needs.wasm-commands.result == 'success' && needs.build-sidecar.result == 'success' && needs.build-sidecar-darwin.result == 'success' }}
274246
runs-on: ubuntu-latest
275247
steps:
276248
- uses: actions/checkout@v4
@@ -330,14 +302,13 @@ jobs:
330302
pnpm --filter=publish exec tsx src/ci/bin.ts bump-versions \
331303
--version ${{ needs.context.outputs.version }} \
332304
--version-only
333-
- name: Install wasm-pack
334-
run: |
335-
rustup target add wasm32-unknown-unknown
336-
curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
337305
- run: node packages/runtime-core/scripts/copy-wasm-commands.mjs --require
338306
- name: Build TypeScript packages
307+
env:
308+
AGENTOS_SKIP_NATIVE_META_BUILD: "1"
339309
run: |
340310
npx turbo build \
311+
--filter='!@rivet-dev/agentos-plugin' \
341312
--filter='!@rivet-dev/agentos-playground' \
342313
--filter='!./examples/*' \
343314
--filter='!./examples/quickstart/*'
@@ -394,6 +365,7 @@ jobs:
394365
cp "artifacts/sidecar-${p}/agentos-sidecar" "release-assets/agentos-sidecar-${target}"
395366
cp "artifacts/sidecar-${p}/agentos-native-sidecar" "release-assets/agentos-native-sidecar-${target}"
396367
done
368+
chmod +x release-assets/agentos-sidecar-* release-assets/agentos-native-sidecar-*
397369
if [ -d crates/execution/assets/pyodide ]; then
398370
mkdir -p release-assets/pyodide
399371
cp -R crates/execution/assets/pyodide/. release-assets/pyodide/

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@ target/
1515
# TypeScript
1616
*.tsbuildinfo
1717
dist/
18-
packages/agentos/src/generated/
18+
packages/agentos/src/generated/*
19+
!packages/agentos/src/generated/actor-actions.generated.ts
1920

2021
# IDE
2122
.vscode/

‎docker/build/darwin.Dockerfile‎

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,21 +10,40 @@ FROM ghcr.io/rivet-dev/rivet/builder-base-osxcross:0e33ceb98
1010
ARG TARGET=aarch64-apple-darwin
1111
ARG CLANG=aarch64-apple-darwin20.4
1212
ARG BUILD_PROFILE=debug
13+
ARG CACHE_PLATFORM=darwin-arm64
14+
ARG RUST_TOOLCHAIN=1.91.1
1315

1416
ENV SDK=/root/osxcross/target/SDK/MacOSX11.3.sdk \
15-
RUSTC_WRAPPER=
17+
RUSTC_WRAPPER=sccache \
18+
SCCACHE_DIR=/root/.cache/sccache \
19+
SCCACHE_IDLE_TIMEOUT=0
1620

1721
WORKDIR /build
1822
COPY . .
1923

20-
RUN rustup toolchain install stable --profile minimal && \
21-
rustup default stable && \
24+
RUN rustup toolchain install "$RUST_TOOLCHAIN" --profile minimal && \
25+
rustup default "$RUST_TOOLCHAIN" && \
2226
rustup target add "$TARGET"
2327

24-
RUN corepack enable && \
25-
pnpm install --no-frozen-lockfile --filter='!@agentos/website'
28+
RUN --mount=type=cache,id=pnpm-store-agentos-darwin,target=/root/.local/share/pnpm/store,sharing=locked \
29+
corepack enable && \
30+
pnpm config set store-dir /root/.local/share/pnpm/store && \
31+
pnpm install --no-frozen-lockfile --filter='!@agentos/website'
2632

27-
RUN tu=$(echo "$TARGET" | tr 'a-z-' 'A-Z_') && \
33+
RUN --mount=type=cache,id=cargo-registry-agentos-darwin,target=/usr/local/cargo/registry,sharing=locked \
34+
--mount=type=cache,id=cargo-git-agentos-darwin,target=/usr/local/cargo/git,sharing=locked \
35+
--mount=type=cache,id=cargo-target-agentos-${CACHE_PLATFORM},target=/build/target,sharing=locked \
36+
--mount=type=cache,id=sccache-agentos-${CACHE_PLATFORM},target=/root/.cache/sccache,sharing=locked \
37+
if ! command -v sccache >/dev/null 2>&1; then \
38+
unset RUSTC_WRAPPER; \
39+
elif ! (sccache --start-server 2>/tmp/sccache-start.err && sccache --show-stats >/dev/null 2>&1); then \
40+
echo "[sccache] unavailable, disabling:"; cat /tmp/sccache-start.err 2>/dev/null || true; \
41+
sccache --stop-server >/dev/null 2>&1 || true; \
42+
unset RUSTC_WRAPPER SCCACHE_DIR; \
43+
else \
44+
echo "[sccache] enabled via local BuildKit cache"; \
45+
fi && \
46+
tu=$(echo "$TARGET" | tr 'a-z-' 'A-Z_') && \
2847
tl=$(echo "$TARGET" | tr - _) && \
2948
export BINDGEN_EXTRA_CLANG_ARGS_${tl}="--sysroot=$SDK -isystem $SDK/usr/include" && \
3049
export CFLAGS_${tl}="-B/root/osxcross/target/bin" && \
@@ -39,6 +58,7 @@ RUN tu=$(echo "$TARGET" | tr 'a-z-' 'A-Z_') && \
3958
mkdir -p /artifacts && \
4059
cp "target/$TARGET/$PROF/agentos-sidecar" /artifacts/agentos-sidecar && \
4160
cp "target/$TARGET/$PROF/agentos-native-sidecar" /artifacts/agentos-native-sidecar && \
42-
cp "target/$TARGET/$PROF/libagentos_actor_plugin.dylib" /artifacts/libagentos_actor_plugin.dylib
61+
cp "target/$TARGET/$PROF/libagentos_actor_plugin.dylib" /artifacts/libagentos_actor_plugin.dylib && \
62+
(sccache --show-stats 2>/dev/null || true)
4363

4464
CMD ["ls", "-la", "/artifacts"]

0 commit comments

Comments
 (0)