Skip to content

Commit 9da78de

Browse files
anandgupta42claude
andauthored
fix: add release smoke tests to prevent broken binary publishes (#463)
* fix: add release smoke tests to prevent broken binary publishes Closes #462 v0.5.10 shipped with `@altimateai/altimate-core` missing from standalone distributions, crashing on startup. Add three-layer defense: - **CI smoke test**: run the compiled `linux-x64` binary after build and before npm publish — catches runtime crashes that compile fine - **Build-time verification**: validate all `requiredExternals` are in `package.json` `dependencies` (not just `devDependencies`) so they ship in the npm wrapper package - **Local pre-release script**: `bun run pre-release` builds + smoke-tests the binary before tagging — mandatory step in RELEASING.md Also adds `smoke-test-binary.test.ts` with 3 tests: version check, standalone graceful-failure, and `--help` output. Reviewed by 5 AI models (Claude, GPT 5.2 Codex, Gemini 3.1 Pro, Kimi K2.5, MiniMax M2.5). Key fixes from review: - Include workspace `node_modules` in `NODE_PATH` (Gemini) - Restrict dep check to `dependencies` only (GPT, Gemini, Kimi) - Hard-fail pre-publish gate when binary not found (Claude, GPT) - Tighten exit code assertion for signal safety (MiniMax) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: address CodeRabbit review — explicit find pattern, Windows `path.delimiter`, binary names - Use `*altimate-code-linux-x64/bin/altimate` in pre-publish find pattern for clarity (old pattern worked but was ambiguous) - Use `path.delimiter` instead of hardcoded `:` for NODE_PATH in both `pre-release-check.ts` and `smoke-test-binary.test.ts` - Handle Windows `altimate.exe` binary name in `searchDist()` functions Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent bbae377 commit 9da78de

File tree

7 files changed

+414
-35
lines changed

7 files changed

+414
-35
lines changed

.github/meta/commit.txt

Lines changed: 24 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,26 @@
1-
fix: Codespaces — skip machine-scoped `GITHUB_TOKEN`, cap retries, fix phantom command
1+
fix: add release smoke tests to prevent broken binary publishes
22

3-
Closes #413
3+
Closes #462
44

5-
- Skip auto-enabling `github-models` and `github-copilot` providers in
6-
machine environments (Codespaces: `CODESPACES=true`, GitHub Actions:
7-
`GITHUB_ACTIONS=true`) when only machine-scoped tokens (`GITHUB_TOKEN`,
8-
`GH_TOKEN`) are available. The Codespace/Actions token lacks
9-
`models:read` scope needed for GitHub Models API.
10-
- Cap retry attempts at 5 (`RETRY_MAX_ATTEMPTS`) to prevent infinite
11-
retry loops. Log actionable warning when retries exhaust.
12-
- Replace phantom `/discover-and-add-mcps` toast with actionable message.
13-
- Add `.devcontainer/` config (Node 22, Bun 1.3.10) for Codespaces.
14-
- Add 32 adversarial e2e tests covering full Codespace/Actions env
15-
simulation, `GH_TOKEN`, token variations, config overrides, retry bounds.
16-
- Update docs to reference `mcp_discover` tool.
5+
v0.5.10 shipped with `@altimateai/altimate-core` missing from standalone
6+
distributions, crashing on startup. Add three-layer defense:
7+
8+
- **CI smoke test**: run the compiled `linux-x64` binary after build and
9+
before npm publish — catches runtime crashes that compile fine
10+
- **Build-time verification**: validate all `requiredExternals` are in
11+
`package.json` `dependencies` (not just `devDependencies`) so they
12+
ship in the npm wrapper package
13+
- **Local pre-release script**: `bun run pre-release` builds + smoke-tests
14+
the binary before tagging — mandatory step in RELEASING.md
15+
16+
Also adds `smoke-test-binary.test.ts` with 3 tests: version check,
17+
standalone graceful-failure, and `--help` output.
18+
19+
Reviewed by 5 AI models (Claude, GPT 5.2 Codex, Gemini 3.1 Pro,
20+
Kimi K2.5, MiniMax M2.5). Key fixes from review:
21+
- Include workspace `node_modules` in `NODE_PATH` (Gemini)
22+
- Restrict dep check to `dependencies` only (GPT, Gemini, Kimi)
23+
- Hard-fail pre-publish gate when binary not found (Claude, GPT)
24+
- Tighten exit code assertion for signal safety (MiniMax)
25+
26+
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

.github/workflows/release.yml

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,25 @@ jobs:
9999
GH_REPO: ${{ env.GH_REPO }}
100100
MODELS_DEV_API_JSON: test/tool/fixtures/models-api.json
101101

102+
# Smoke-test: verify the compiled binary actually starts.
103+
# Only possible for native linux-x64 builds on the ubuntu runner.
104+
# This catches missing externals (e.g. @altimateai/altimate-core)
105+
# that compile fine but crash at runtime.
106+
- name: Smoke test binary
107+
if: matrix.name == 'linux-x64'
108+
run: |
109+
BINARY=$(find packages/opencode/dist -name altimate -type f | head -1)
110+
if [ -z "$BINARY" ]; then
111+
echo "::error::No binary found in dist/"
112+
exit 1
113+
fi
114+
chmod +x "$BINARY"
115+
116+
# Set NODE_PATH so the binary can resolve external NAPI modules
117+
# (mirrors what the npm bin wrapper does at runtime)
118+
NODE_PATH="$(pwd)/packages/opencode/node_modules:$(pwd)/node_modules" "$BINARY" --version
119+
echo "Smoke test passed: binary starts and prints version"
120+
102121
- name: Upload build artifact
103122
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
104123
with:
@@ -168,6 +187,21 @@ jobs:
168187
# env:
169188
# AUR_SSH_PRIVATE_KEY: ${{ secrets.AUR_SSH_PRIVATE_KEY }}
170189

190+
# Smoke-test a linux-x64 binary from the downloaded artifacts before publishing.
191+
# This is the last gate before npm publish — catches runtime crashes that
192+
# compile-time checks miss (e.g. missing NAPI externals like v0.5.10).
193+
- name: Pre-publish smoke test
194+
run: |
195+
BINARY=$(find packages/opencode/dist -path '*altimate-code-linux-x64/bin/altimate' -type f | head -1)
196+
if [ -z "$BINARY" ]; then
197+
echo "::error::No linux-x64 binary found in artifacts — cannot verify release"
198+
exit 1
199+
else
200+
chmod +x "$BINARY"
201+
NODE_PATH="$(pwd)/packages/opencode/node_modules:$(pwd)/node_modules" "$BINARY" --version
202+
echo "Pre-publish smoke test passed"
203+
fi
204+
171205
- name: Publish to npm
172206
run: bun run packages/opencode/script/publish.ts
173207
env:

docs/RELEASING.md

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,23 @@ Add a new section at the top of `CHANGELOG.md`:
4848
- ...
4949
```
5050

51-
### 2. Commit and tag
51+
### 2. Run pre-release sanity check
52+
53+
**MANDATORY** — this catches broken binaries before they reach users:
54+
55+
```bash
56+
cd packages/opencode
57+
bun run pre-release
58+
```
59+
60+
This verifies:
61+
- All required NAPI externals are in `package.json` dependencies
62+
- They're installed in `node_modules`
63+
- A local build produces a binary that actually starts
64+
65+
Do NOT proceed if any check fails.
66+
67+
### 3. Commit and tag
5268

5369
```bash
5470
git add -A
@@ -57,7 +73,7 @@ git tag v0.5.0
5773
git push origin main v0.5.0
5874
```
5975

60-
### 3. What happens automatically
76+
### 4. What happens automatically
6177

6278
The `v*` tag triggers `.github/workflows/release.yml` which:
6379

@@ -67,7 +83,7 @@ The `v*` tag triggers `.github/workflows/release.yml` which:
6783
4. **Updates AUR** — pushes PKGBUILD update to `altimate-code-bin`
6884
5. **Publishes Docker image** — to `ghcr.io/altimateai/altimate-code`
6985

70-
### 4. Verify
86+
### 5. Verify
7187

7288
After the workflow completes:
7389

packages/opencode/package.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@
1212
"build:local": "bun run script/build.ts --single --skip-install",
1313
"local": "./script/local.sh",
1414
"dev": "bun run --conditions=browser ./src/index.ts",
15-
"db": "bun drizzle-kit"
15+
"db": "bun drizzle-kit",
16+
"pre-release": "bun run script/pre-release-check.ts"
1617
},
1718
"bin": {
1819
"altimate": "./bin/altimate",

packages/opencode/script/build.ts

Lines changed: 47 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,21 @@ const targets = targetIndexFlag !== undefined
193193

194194
await $`rm -rf dist`
195195

196+
// Packages excluded from the compiled binary — must be resolvable from
197+
// node_modules at runtime. Split into required (must ship with the wrapper
198+
// package) and optional (user installs on demand).
199+
const requiredExternals = [
200+
// NAPI native module — cannot be embedded in Bun single-file executable.
201+
"@altimateai/altimate-core",
202+
]
203+
const optionalExternals = [
204+
// Database drivers — native addons, users install on demand per warehouse
205+
"pg", "snowflake-sdk", "@google-cloud/bigquery", "@databricks/sql",
206+
"mysql2", "mssql", "oracledb", "duckdb",
207+
// Optional infra packages — native addons or heavy optional deps
208+
"keytar", "ssh2", "dockerode",
209+
]
210+
196211
const binaries: Record<string, string> = {}
197212
if (!skipInstall) {
198213
await $`bun install --os="*" --cpu="*" @opentui/core@${pkg.dependencies["@opentui/core"]}`
@@ -225,27 +240,11 @@ for (const item of targets) {
225240
tsconfig: "./tsconfig.json",
226241
plugins: [solidPlugin],
227242
sourcemap: "external",
228-
// Packages excluded from the compiled binary — resolved from node_modules
229-
// at runtime. Bun compiled binaries resolve externals via standard Node
230-
// resolution from the binary's location, walking up to the wrapper
231-
// package's node_modules.
232-
//
233243
// IMPORTANT: Without code splitting, Bun inlines dynamic import() targets
234244
// into the main chunk. Any external require() in those targets will fail
235245
// at startup — not when the import() is called. Only mark packages as
236246
// external when they truly cannot be bundled (e.g. NAPI native addons).
237-
external: [
238-
// NAPI native module — cannot be embedded in Bun single-file executable.
239-
// The JS loader dynamically require()s platform-specific .node binaries
240-
// (e.g. @altimateai/altimate-core-darwin-arm64).
241-
// Must be installed as a dependency of the published wrapper package.
242-
"@altimateai/altimate-core",
243-
// Database drivers — native addons, users install on demand per warehouse
244-
"pg", "snowflake-sdk", "@google-cloud/bigquery", "@databricks/sql",
245-
"mysql2", "mssql", "oracledb", "duckdb",
246-
// Optional infra packages — native addons or heavy optional deps
247-
"keytar", "ssh2", "dockerode",
248-
],
247+
external: [...requiredExternals, ...optionalExternals],
249248
compile: {
250249
autoloadBunfig: false,
251250
autoloadDotenv: false,
@@ -293,6 +292,37 @@ for (const item of targets) {
293292
binaries[name] = Script.version
294293
}
295294

295+
// ---------------------------------------------------------------------------
296+
// Build-time verification: ensure required externals are in package.json
297+
// dependencies so they ship with the npm wrapper package. This catches the
298+
// scenario where a new NAPI module is added to `external` but not to
299+
// package.json dependencies — which would compile fine but crash at runtime.
300+
// ---------------------------------------------------------------------------
301+
{
302+
// Only check dependencies (not devDependencies) — publish.ts only ships
303+
// dependencies to end users. A required external in devDependencies would
304+
// pass this check but be missing for npm users.
305+
const pkgDeps: Record<string, string> = {
306+
...pkg.dependencies,
307+
}
308+
const missing = requiredExternals.filter((ext) => !pkgDeps[ext])
309+
if (missing.length > 0) {
310+
const msg =
311+
`Required external(s) not in package.json: ${missing.join(", ")}\n` +
312+
`These packages are marked as external in the binary build but are not\n` +
313+
`listed as dependencies. The binary will crash at runtime.\n` +
314+
`Add them to "dependencies" in packages/opencode/package.json.`
315+
if (Script.release) {
316+
console.error(`FATAL: ${msg}`)
317+
process.exit(1)
318+
} else {
319+
console.warn(`WARNING: ${msg}`)
320+
}
321+
} else {
322+
console.log(`Verified ${requiredExternals.length} required external(s) are in package.json`)
323+
}
324+
}
325+
296326
if (Script.release) {
297327
for (const key of Object.keys(binaries)) {
298328
const archiveName = key.replace(/^@altimateai\//, "")
Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
#!/usr/bin/env bun
2+
3+
/**
4+
* Pre-release sanity check — run BEFORE tagging a release.
5+
*
6+
* Verifies:
7+
* 1. All required external NAPI modules are in package.json dependencies
8+
* 2. The publish script will include them in the wrapper package
9+
* 3. A local build produces a binary that actually starts
10+
*
11+
* Usage: bun run packages/opencode/script/pre-release-check.ts
12+
*/
13+
14+
import fs from "fs"
15+
import path from "path"
16+
import { spawnSync } from "child_process"
17+
import { fileURLToPath } from "url"
18+
19+
const __dirname = path.dirname(fileURLToPath(import.meta.url))
20+
const pkgDir = path.resolve(__dirname, "..")
21+
const repoRoot = path.resolve(pkgDir, "../..")
22+
23+
const pkg = JSON.parse(fs.readFileSync(path.join(pkgDir, "package.json"), "utf-8"))
24+
25+
let failures = 0
26+
27+
function pass(msg: string) {
28+
console.log(` ✓ ${msg}`)
29+
}
30+
31+
function fail(msg: string) {
32+
console.error(` ✗ ${msg}`)
33+
failures++
34+
}
35+
36+
// ---------------------------------------------------------------------------
37+
// Check 1: Required externals are in package.json dependencies
38+
// ---------------------------------------------------------------------------
39+
console.log("\n[1/4] Checking required externals in package.json...")
40+
41+
const requiredExternals = ["@altimateai/altimate-core"]
42+
43+
for (const ext of requiredExternals) {
44+
if (pkg.dependencies?.[ext]) {
45+
pass(`${ext} is in dependencies (${pkg.dependencies[ext]})`)
46+
} else {
47+
fail(`${ext} is NOT in dependencies — binary will crash at runtime`)
48+
}
49+
}
50+
51+
// ---------------------------------------------------------------------------
52+
// Check 2: Required externals are resolvable in node_modules
53+
// ---------------------------------------------------------------------------
54+
console.log("\n[2/4] Checking required externals are installed...")
55+
56+
for (const ext of requiredExternals) {
57+
try {
58+
require.resolve(ext)
59+
pass(`${ext} resolves from node_modules`)
60+
} catch {
61+
fail(`${ext} is NOT installed — run \`bun install\``)
62+
}
63+
}
64+
65+
// ---------------------------------------------------------------------------
66+
// Check 3: Build and smoke-test the binary
67+
// ---------------------------------------------------------------------------
68+
console.log("\n[3/4] Building local binary...")
69+
70+
const buildResult = spawnSync("bun", ["run", "build:local"], {
71+
cwd: pkgDir,
72+
encoding: "utf-8",
73+
timeout: 120_000,
74+
env: {
75+
...process.env,
76+
MODELS_DEV_API_JSON: path.join(pkgDir, "test/tool/fixtures/models-api.json"),
77+
},
78+
})
79+
80+
if (buildResult.status !== 0) {
81+
fail(`Build failed:\n${buildResult.stderr}`)
82+
} else {
83+
pass("Local build succeeded")
84+
85+
// Find the binary — walk recursively for scoped packages (@altimateai/...)
86+
const distDir = path.join(pkgDir, "dist")
87+
let binaryPath: string | undefined
88+
const binaryNames = process.platform === "win32" ? ["altimate.exe", "altimate"] : ["altimate"]
89+
function searchDist(dir: string): string | undefined {
90+
if (!fs.existsSync(dir)) return undefined
91+
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
92+
if (!entry.isDirectory()) continue
93+
const sub = path.join(dir, entry.name)
94+
for (const name of binaryNames) {
95+
const candidate = path.join(sub, "bin", name)
96+
if (fs.existsSync(candidate)) return candidate
97+
}
98+
const nested = searchDist(sub)
99+
if (nested) return nested
100+
}
101+
return undefined
102+
}
103+
binaryPath = searchDist(distDir)
104+
105+
if (!binaryPath) {
106+
fail("No binary found in dist/ after build")
107+
} else {
108+
console.log("\n[4/4] Smoke-testing compiled binary...")
109+
110+
// Resolve NODE_PATH like the bin wrapper does — start from pkgDir
111+
// to include workspace-level node_modules where NAPI modules live
112+
const nodePaths: string[] = []
113+
let current = pkgDir
114+
for (;;) {
115+
const nm = path.join(current, "node_modules")
116+
if (fs.existsSync(nm)) nodePaths.push(nm)
117+
const parent = path.dirname(current)
118+
if (parent === current) break
119+
current = parent
120+
}
121+
122+
const smokeResult = spawnSync(binaryPath, ["--version"], {
123+
encoding: "utf-8",
124+
timeout: 15_000,
125+
env: {
126+
...process.env,
127+
NODE_PATH: nodePaths.join(path.delimiter),
128+
OPENCODE_DISABLE_TELEMETRY: "1",
129+
},
130+
})
131+
132+
if (smokeResult.status === 0) {
133+
const version = (smokeResult.stdout ?? "").trim()
134+
pass(`Binary starts successfully (${version})`)
135+
} else {
136+
const output = (smokeResult.stdout ?? "") + (smokeResult.stderr ?? "")
137+
fail(`Binary crashed on startup:\n${output}`)
138+
}
139+
}
140+
}
141+
142+
// ---------------------------------------------------------------------------
143+
// Summary
144+
// ---------------------------------------------------------------------------
145+
console.log("")
146+
if (failures > 0) {
147+
console.error(`FAILED: ${failures} check(s) failed. Do NOT tag a release.`)
148+
process.exit(1)
149+
} else {
150+
console.log("ALL CHECKS PASSED. Safe to tag a release.")
151+
}

0 commit comments

Comments
 (0)