Skip to content

Commit b67f287

Browse files
SniderSnider
committed
merge: sync dev — CI matrix, artifact grid, PR gate (post #13/#14)
Co-Authored-By: Virgil <virgil@lethean.io>
2 parents 60f97bf + c65fa35 commit b67f287

8 files changed

Lines changed: 379 additions & 80 deletions

File tree

‎.github/workflows/build.yml‎

Lines changed: 170 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,96 +1,212 @@
1-
# Build the sovereign `lem` binary for every target and ship the zips.
2-
# One binary named `lem` per platform (the GPU backend — metal/rocm/cuda — loads
3-
# its kernel sidecar at runtime); targets are separated by build folder, not by
4-
# binary name. Zips flow to a rolling `dev` prerelease.
1+
# Build the sovereign `lem` binary for every hosted-buildable cell and ship the zips.
2+
#
3+
# lem links DuckDB (go-store/chathistory) via cgo, so there is NO pure-Go build and
4+
# NO cheap cross-compile: each cell is a NATIVE cgo build on a matching runner.
5+
# GitHub owns the portable cpu cells it can build natively; the ROCm/CUDA cells
6+
# (amd, cuda) stay on GitLab's homelab runner (it has the toolchain); the metal
7+
# cell needs a macOS 26 / Metal 4 box to compile the metallibs, so it rides a
8+
# self-hosted Mac lane (dormant until ENABLE_MACOS_METAL is set).
9+
#
10+
# Every build ships in two packagings of the same binary (see docs/release-artifacts.md):
11+
# Native {os}-{arch}-lem-{backend}-{version}.zip -> binary `lem`
12+
# Driver {os}-{arch}-lem-driver-{backend}-{version}.zip -> binary `lem-{backend}`
13+
# Zips flow to a rolling `dev` prerelease; version is `dev-<short-sha>`.
514
name: Build
615

716
on:
817
push:
918
branches: [dev, main]
19+
tags: ['v*'] # a version tag publishes the REAL release (guarded on-main)
1020
workflow_dispatch:
1121

1222
permissions:
1323
contents: read
1424

25+
env:
26+
GO_VERSION: "1.26"
27+
1528
jobs:
16-
build:
17-
name: lem · ${{ matrix.goos }}/${{ matrix.goarch }}
18-
runs-on: ${{ matrix.os }}
29+
# ---- cpu cells: native cgo, one per hosted os/arch (duckdb ships prebuilt libs
30+
# for darwin/{amd64,arm64}, linux/{amd64,arm64}, windows/amd64 — NOT
31+
# windows/arm64, which is therefore not a cell). The workspace (go.work)
32+
# wires cli -> local go/; GOWORK=off cannot build the binary because cli
33+
# pins a published go/inference tag that lags the local tree. ----------------
34+
cpu:
35+
name: lem cpu · ${{ matrix.os_name }}/${{ matrix.arch }}
36+
runs-on: ${{ matrix.runner }}
1937
strategy:
2038
fail-fast: false
2139
matrix:
2240
include:
23-
- { os: macos-latest, goos: darwin, goarch: arm64, metal: true }
24-
- { os: ubuntu-latest, goos: linux, goarch: amd64 }
25-
- { os: ubuntu-24.04-arm, goos: linux, goarch: arm64 }
26-
- { os: windows-latest, goos: windows, goarch: amd64 }
27-
env:
28-
CGO_ENABLED: "1"
29-
GOWORK: "off"
41+
# Dropped cells, proven by live runs 2026-07-22 (docs/release-artifacts.md):
42+
# - macos/x86_64: the macos-13 runner label is retired (queued forever) —
43+
# no hosted Intel Mac remains; revisit only on real demand.
44+
# - windows/x86_64: lem's code is not Windows-portable yet (unix-only
45+
# syscall.Kill + raw-fd reads outside build constraints) — a code
46+
# campaign, not a CI matrix entry; restore the cell when it compiles.
47+
- { runner: macos-latest, os_name: macos, arch: aarch64, exe: "" }
48+
- { runner: ubuntu-latest, os_name: linux, arch: x86_64, exe: "" }
49+
- { runner: ubuntu-24.04-arm, os_name: linux, arch: aarch64, exe: "" }
3050
steps:
3151
- uses: actions/checkout@v4
32-
with:
33-
submodules: recursive # metal needs external/mlx
52+
# cpu build needs neither external/mlx (metal) nor the ROCm sources.
3453

3554
- uses: actions/setup-go@v5
3655
with:
37-
go-version: "1.26"
38-
39-
- name: Install Task
40-
run: go install github.com/go-task/task/v3/cmd/task@latest
56+
go-version: ${{ env.GO_VERSION }}
57+
# go.sum lives in the modules, not the repo root; list both so setup-go's
58+
# cache key is honest (build touches cli, cli requires local go/).
59+
cache-dependency-path: |
60+
go/go.sum
61+
cli/go.sum
4162
42-
- name: Build lem
63+
- name: Build lem (native cgo; workspace wires cli -> local go/)
4364
shell: bash
4465
env:
45-
METAL: ${{ matrix.metal }}
66+
CGO_ENABLED: "1"
67+
EXE: ${{ matrix.exe }}
4668
run: |
47-
set -e
48-
if [ "$METAL" = "true" ]; then
49-
task metallib && task build:embed # self-contained metal lem
50-
else
51-
task build:native # plain lem (GPU sidecar loads at runtime)
52-
fi
69+
set -euo pipefail
70+
if [[ "$GITHUB_REF" == refs/tags/v* ]]; then VERSION="$GITHUB_REF_NAME"; else VERSION="dev-${GITHUB_SHA:0:12}"; fi
71+
mkdir -p bin
72+
go build -trimpath -ldflags "-X main.version=${VERSION}" -o "bin/lem${EXE}" ./cli
73+
test -s "bin/lem${EXE}"
74+
75+
- uses: actions/upload-artifact@v4
76+
with:
77+
name: lem-cpu-${{ matrix.os_name }}-${{ matrix.arch }}
78+
path: bin/lem${{ matrix.exe }}
79+
if-no-files-found: error
80+
81+
# ---- metal cell: macos/aarch64, self-contained (metallibs embedded). Hosted
82+
# macOS runners are < macOS 26 and cannot compile the Metal 4 metallibs, so
83+
# a plain build would ship a metallib-less, non-runnable binary. This lane is
84+
# DORMANT on normal pushes (the `if` is false when the variable is unset, so
85+
# it never queues or fails): register a self-hosted macOS 26 arm64 runner and
86+
# set repo variable ENABLE_MACOS_METAL=true to activate it. Interim: the metal
87+
# artifact is built manually on the owner's Mac (task metallib && task build:embed).
88+
metal:
89+
name: lem metal · macos/aarch64 (self-hosted macOS 26)
90+
if: ${{ vars.ENABLE_MACOS_METAL == 'true' }}
91+
runs-on: [self-hosted, macOS, ARM64]
92+
steps:
93+
- uses: actions/checkout@v4
94+
with:
95+
submodules: recursive # external/mlx for `task metallib`
96+
97+
- uses: actions/setup-go@v5
98+
with:
99+
go-version: ${{ env.GO_VERSION }}
100+
cache-dependency-path: |
101+
go/go.sum
102+
cli/go.sum
103+
104+
- name: Install Task
105+
run: go install github.com/go-task/task/v3/cmd/task@latest
53106

54-
- name: Assemble build folder (build/dist/<target>/lem)
107+
- name: Build self-contained metal lem (metallibs baked in)
55108
shell: bash
56-
env:
57-
TARGET: ${{ matrix.goos }}-${{ matrix.goarch }}
58109
run: |
59-
set -e
60-
mkdir -p "build/dist/$TARGET"
61-
cp bin/lem* "build/dist/$TARGET/"
110+
set -euo pipefail
111+
if [[ "$GITHUB_REF" == refs/tags/v* ]]; then VERSION="$GITHUB_REF_NAME"; else VERSION="dev-${GITHUB_SHA:0:12}"; fi
112+
task metallib
113+
task build:embed VERSION="${VERSION}"
114+
test -s bin/lem
62115
63116
- uses: actions/upload-artifact@v4
64117
with:
65-
name: lem-${{ matrix.goos }}-${{ matrix.goarch }}
66-
path: build/dist/${{ matrix.goos }}-${{ matrix.goarch }}/
118+
name: lem-metal-macos-aarch64
119+
path: bin/lem
120+
if-no-files-found: error
67121

122+
# ---- release: package each built binary under BOTH names (build once, package
123+
# twice). Two publish shapes: a dev push feeds the rolling `dev`
124+
# prerelease; a v* TAG (guarded: the tagged commit must be on main)
125+
# publishes the real versioned release. Requires cpu green; tolerates
126+
# metal skipped. ----
68127
release:
69-
name: Zip + rolling dev prerelease
70-
needs: build
71-
if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
128+
name: Zip (Native + Driver) + release
129+
needs: [cpu, metal]
130+
if: ${{ !cancelled() && needs.cpu.result == 'success' && github.event_name == 'push' && (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/v')) }}
72131
runs-on: ubuntu-latest
73132
permissions:
74133
contents: write
75134
steps:
135+
- uses: actions/checkout@v4
136+
with:
137+
fetch-depth: 0 # the tag-on-main ancestry guard needs history
138+
139+
# A tag release only publishes if the tagged commit is reachable from
140+
# main — "a tag hitting main". A tag pushed off-branch fails LOUDLY
141+
# (the run goes red) rather than silently minting a release.
142+
- name: Guard — tag must be on main
143+
if: startsWith(github.ref, 'refs/tags/v')
144+
shell: bash
145+
run: |
146+
set -euo pipefail
147+
git fetch -q origin main
148+
if git merge-base --is-ancestor "$GITHUB_SHA" origin/main; then
149+
echo "tag $GITHUB_REF_NAME is on main — releasing"
150+
else
151+
echo "::error::tag $GITHUB_REF_NAME ($GITHUB_SHA) is NOT on main — refusing to release"
152+
exit 1
153+
fi
154+
155+
- name: Resolve version + release shape
156+
id: shape
157+
shell: bash
158+
run: |
159+
set -euo pipefail
160+
if [[ "$GITHUB_REF" == refs/tags/v* ]]; then
161+
echo "version=$GITHUB_REF_NAME" >> "$GITHUB_OUTPUT"
162+
echo "tag=$GITHUB_REF_NAME" >> "$GITHUB_OUTPUT"
163+
echo "name=$GITHUB_REF_NAME" >> "$GITHUB_OUTPUT"
164+
echo "prerelease=false" >> "$GITHUB_OUTPUT"
165+
else
166+
echo "version=dev-${GITHUB_SHA:0:12}" >> "$GITHUB_OUTPUT"
167+
echo "tag=dev" >> "$GITHUB_OUTPUT"
168+
echo "name=dev (rolling)" >> "$GITHUB_OUTPUT"
169+
echo "prerelease=true" >> "$GITHUB_OUTPUT"
170+
fi
171+
76172
- uses: actions/download-artifact@v4
77173
with:
78-
path: dist # one subfolder per target (lem-<goos>-<goarch>/lem)
174+
path: dist # one dir per cell: dist/lem-<backend>-<os>-<arch>/lem[.exe]
79175

80-
- name: Zip each target
176+
- name: Package Native + Driver zips
177+
shell: bash
178+
env:
179+
VERSION: ${{ steps.shape.outputs.version }}
81180
run: |
82-
set -e
181+
set -euo pipefail
182+
mkdir -p out
83183
cd dist
84-
for d in */; do (cd "$d" && zip -r "../${d%/}.zip" .); done
85-
ls -la *.zip
184+
# ONLY the binary artifacts (lem-cpu-*, lem-metal-*) — lem-sdks also
185+
# lands in dist/ and must never enter the packaging loop.
186+
for d in lem-cpu-*/ lem-metal-*/; do
187+
[ -d "$d" ] || continue # unmatched glob (metal skipped)
188+
name="${d%/}" # e.g. lem-cpu-macos-aarch64
189+
IFS=- read -r _ backend os arch <<< "$name" # backend/os/arch from the name
190+
bin="$(ls "$d")" # lem or lem.exe
191+
ext=""; case "$bin" in *.exe) ext=".exe";; esac
192+
# Native — the binary a user runs locally, named `lem`.
193+
cp "$d/$bin" "lem${ext}"; chmod 0755 "lem${ext}"
194+
zip -jq "../out/${os}-${arch}-lem-${backend}-${VERSION}.zip" "lem${ext}"
195+
rm -f "lem${ext}"
196+
# Driver — the binary a compute box serves over the API, named `lem-<backend>`.
197+
cp "$d/$bin" "lem-${backend}${ext}"; chmod 0755 "lem-${backend}${ext}"
198+
zip -jq "../out/${os}-${arch}-lem-driver-${backend}-${VERSION}.zip" "lem-${backend}${ext}"
199+
rm -f "lem-${backend}${ext}"
200+
done
201+
ls -la ../out
86202
87-
- name: Publish rolling dev prerelease
203+
- name: Publish release
88204
uses: softprops/action-gh-release@v2
89205
with:
90-
tag_name: dev
91-
name: dev (rolling)
92-
prerelease: true
93-
files: dist/*.zip
206+
tag_name: ${{ steps.shape.outputs.tag }}
207+
name: ${{ steps.shape.outputs.name }}
208+
prerelease: ${{ steps.shape.outputs.prerelease }}
209+
files: out/*.zip
94210
env:
95211
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
96212

@@ -101,7 +217,10 @@ jobs:
101217
- uses: actions/checkout@v4
102218
- uses: actions/setup-go@v5
103219
with:
104-
go-version: "1.26"
220+
go-version: ${{ env.GO_VERSION }}
221+
cache-dependency-path: |
222+
go/go.sum
223+
cli/go.sum
105224
- uses: actions/setup-node@v4
106225
with:
107226
node-version: "20"
@@ -111,7 +230,7 @@ jobs:
111230
java-version: "21"
112231
- name: Install generators
113232
run: |
114-
npm install -g @openapitools/openapi-generator-cli@7.22.0
233+
npm install -g @openapitools/openapi-generator-cli@2.40.0 # npm wrapper is 2.x; it manages the 7.x Java generator
115234
go install github.com/go-task/task/v3/cmd/task@latest
116235
- name: Generate SDKs (lem spec -> OpenAPI 3.1 -> typed clients)
117236
run: task sdk

‎.github/workflows/ci.yml‎

Lines changed: 47 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
1-
# Portable CI for go-inference — the default-tag (no-GPU) test + coverage lane.
2-
# go-inference is a Metal/CGO project; the FULL lem build (engine/metal, the
3-
# metallibs) needs macOS 26 / Metal 4 and lives in build.yml on a self-hosted
4-
# runner. This lane runs the portable suite (Taskfile `test`/`cover`) on any OS.
1+
# PR gate + push CI for go-inference — the model-free test surface on all three
2+
# desktop OSes. The suite is testable without a model loaded (engine/metal skips
3+
# cleanly without MLX_METALLIB_PATH; engine/hip's contract tests are host-side),
4+
# so this lane is the go/no-go for PRs into dev: linux + macos are REQUIRED
5+
# checks; windows is EXPERIMENTAL (continue-on-error) until the cgo-on windows
6+
# graph is proven — its logs name the real portability offenders either way.
7+
# The full artifact build (and the metal lane) lives in build.yml.
58
name: CI
69

710
on:
@@ -13,25 +16,59 @@ on:
1316
permissions:
1417
contents: read
1518

19+
env:
20+
GO_VERSION: "1.26"
21+
1622
jobs:
1723
test:
18-
name: Portable test + coverage
19-
runs-on: ubuntu-latest
24+
name: tests · ${{ matrix.os_name }}
25+
runs-on: ${{ matrix.runner }}
26+
continue-on-error: ${{ matrix.experimental }}
27+
strategy:
28+
fail-fast: false
29+
matrix:
30+
include:
31+
- { runner: ubuntu-latest, os_name: linux, experimental: false }
32+
- { runner: macos-latest, os_name: macos, experimental: false }
33+
- { runner: windows-latest, os_name: windows, experimental: true }
2034
env:
21-
GOWORK: "off" # resolve deps from go/go.mod tags, not the workspace
22-
CGO_ENABLED: "1" # go/ links duckdb (go-duckdb/v2 ships prebuilt linux libs)
35+
CGO_ENABLED: "1" # go/ links duckdb (prebuilt libs for all three OSes)
2336
steps:
2437
- uses: actions/checkout@v4
2538

2639
- uses: actions/setup-go@v5
2740
with:
28-
go-version: "1.26"
41+
go-version: ${{ env.GO_VERSION }}
42+
cache-dependency-path: |
43+
go/go.sum
44+
cli/go.sum
2945
30-
- name: Portable tests + coverage (default tags — engine/metal excluded)
46+
# The module suite resolves from go.mod tags (GOWORK=off) — proves the
47+
# published dependency surface, not the workspace.
48+
- name: Module suite (default tags — engine/metal skips without a metallib)
3149
working-directory: go
50+
env:
51+
GOWORK: "off"
52+
shell: bash
3253
run: go test -count=1 -covermode=atomic -coverprofile=coverage.out ./...
3354

55+
# cli + examples need the workspace (cli requires the LOCAL go/ tree; the
56+
# examples module is the acceptance surface — every public feature ships
57+
# with its example, so compiling them all catches API breakage model-free).
58+
- name: TUI suite (workspace)
59+
working-directory: cli
60+
shell: bash
61+
run: go test -count=1 ./tui/...
62+
63+
- name: Examples compile + vet (workspace — the acceptance surface)
64+
working-directory: examples
65+
shell: bash
66+
run: |
67+
go build ./...
68+
go vet ./...
69+
3470
- name: Upload coverage to Codecov
71+
if: matrix.os_name == 'linux'
3572
uses: codecov/codecov-action@v5
3673
with:
3774
files: go/coverage.out

‎Taskfile.yml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,13 @@ vars:
1818
VERSION: dev
1919
GO_VERSION_LDFLAGS: '-X main.version={{.VERSION}}'
2020
GO_DARWIN_LDFLAGS: '-extldflags=-mmacosx-version-min=26.0'
21+
# Core count for cmake --parallel (metallib:mlx only). go-task evaluates global
22+
# sh-vars eagerly on every invocation, so this must succeed on EVERY platform:
23+
# getconf/nproc cover Linux, sysctl -n hw.ncpu is the macOS path, and the `echo 4`
24+
# floor keeps it green where none of the three exist (Windows, task's internal
25+
# POSIX shell has no such binaries) — a bare `sysctl -n hw.ncpu` was the CI failure.
2126
NCPU:
22-
sh: getconf _NPROCESSORS_ONLN 2>/dev/null || nproc 2>/dev/null || sysctl -n hw.ncpu
27+
sh: getconf _NPROCESSORS_ONLN 2>/dev/null || nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4
2328
env:
2429
MLX_METALLIB_PATH: '{{.ROOT_DIR}}/build/dist/lib/mlx.metallib'
2530

0 commit comments

Comments
 (0)