refactor(registry): unify package build/publish lifecycle in agentos-toolchain (#222)

Author: NathanFlurry

.claude/skills/publish-registry/SKILL.md

@@ -0,0 +1,36 @@
+---
+name: publish-registry
+description: Publish @agentos-software/* registry packages (per-package semver; dist-tag dev by default, latest only deliberately). Use whenever the user asks to publish or release registry software/agent packages.
+---
+
+# Publish registry packages
+
+Registry packages version **independently** (per-package semver in each
+`package.json`). Publishing never moves `latest` unless asked. Full lifecycle
+reference: `registry/README.md`.
+
+1. **Build** (skip what's already built):
+
+   ```bash
+   just registry-native            # native wasm binaries, once per checkout (slow)
+   just registry-build [pkg]       # stage bin/ + assemble dist/package
+   just registry-status --remote   # local state vs published dist-tags
+   ```
+
+2. **Bump the version** in `registry/software/<pkg>/package.json` (or
+   `registry/agent/<pkg>/`) and commit it.
+
+3. **Publish**:
+
+   ```bash
+   just registry-publish <pkg>            # dist-tag dev (safe default)
+   just registry-publish <pkg> latest     # DELIBERATE release — moves latest
+   just registry-publish-all [tag]        # every built software package
+   ```
+
+Notes:
+- secure-exec **previews** (publish.yaml, no version input) automatically
+  include all registry packages under the branch dist-tag — no manual step for
+  preview consumers.
+- agent-os pins per-package: `just agentos-pkgs-update [tag]` /
+  `just agentos-pkgs-set-version <pkg> <v>` over there.

.claude/skills/release-preview/SKILL.md

@@ -0,0 +1,34 @@
+---
+name: release-preview
+description: Cut a secure-exec release-preview — npm-only branch-dist-tag publish (registry packages included), no crates.io. Use when the user asks for a preview / release-preview of secure-exec, or to hand a build to a downstream.
+---
+
+# Release-preview secure-exec
+
+A preview publishes the `@secure-exec/*` packages AND the `@agentos-software/*`
+registry packages to npm under the sanitized branch dist-tag, versioned
+`0.0.0-<branch>.<sha>`, from a fast debug build. No crates.io publish (it has
+no preview track — crate changes reach downstreams via their clone-at-sha
+builds), no git tag, no release assets.
+
+1. **Push the branch** you want previewed (jj colocated; use
+   `jj --config snapshot.max-new-file-size=16777216 ...` if large assets
+   complain).
+
+2. **Dispatch + watch**:
+
+   ```bash
+   just release-preview <branch>
+   run=$(gh run list -R rivet-dev/secure-exec --workflow=publish.yaml -L1 --json databaseId --jq '.[0].databaseId')
+   gh run watch -R rivet-dev/secure-exec "$run" --exit-status
+   ```
+
+3. **Consume**: `npm install @secure-exec/core@<sanitized-branch>` (same tag for
+   the registry packages). agent-os does NOT normally consume these directly —
+   it pins a sha in `.github/refs/secure-exec` and its own release-preview
+   auto-cuts the matching secure-exec preview (branch `agentos-dep-<sha7>`).
+
+Notes:
+- Release-preview is for previews ONLY; never cut a release with it — releases
+  go through the `release` skill.
+- On failure: `gh run view <run> --log-failed`, fix, re-dispatch, re-watch.

.claude/skills/release/SKILL.md

@@ -0,0 +1,40 @@
+---
+name: release
+description: Cut a stable secure-exec release — npm + crates.io in lockstep, plus the manual @secure-exec/core wasm publish. Use whenever the user asks to release secure-exec (a real version, not a preview).
+---
+
+# Release secure-exec (stable)
+
+Releases publish the `@secure-exec/*` npm packages AND the `secure-exec-*`
+crates at the SAME version. Previews are a different flow (npm-only branch
+dist-tag; see CLAUDE.md "Preview-publishing").
+
+1. **Cut the release** from a clean, pushed main checkout:
+
+   ```bash
+   just release --patch -y        # or --minor / --major / --version <v>; -rc. versions get the rc tag
+   ```
+
+   This bumps versions, commits, pushes, and dispatches `publish.yaml` with the
+   version input → release build: npm `@latest`, crates.io, GitHub release assets.
+
+2. **Watch it green**:
+
+   ```bash
+   run=$(gh run list -R rivet-dev/secure-exec --workflow=publish.yaml -L1 --json databaseId --jq '.[0].databaseId')
+   gh run watch -R rivet-dev/secure-exec "$run" --exit-status
+   ```
+
+3. **Manual step — `@secure-exec/core`** (CI-excluded; its tarball vendors the
+   wasm commands and CI does not build them):
+
+   ```bash
+   make -C registry/native wasm
+   cd packages/core && npm publish     # npm, NOT pnpm; same version CI used; prepack fails loud if commands absent
+   ```
+
+4. **Registry packages are NOT part of a release** — they version per-package;
+   use the `publish-registry` skill.
+
+5. Downstream: an agent-os release passes this version via
+   `just release --secure-exec-version <v>` (agent-os `release-agentos` skill).

.github/workflows/publish.yaml

@@ -243,6 +243,9 @@ jobs:
           pnpm --dir packages/core run copy-commands
           test -f packages/core/commands/sh
       - name: Bump package versions for build
+        env:
+          # Previews publish the registry packages under the branch dist-tag too.
+          PUBLISH_INCLUDE_REGISTRY_PACKAGES: ${{ needs.context.outputs.trigger != 'release' && '1' || '' }}
         run: |
           pnpm --filter=publish exec tsx src/ci/bin.ts bump-versions \
             --version ${{ needs.context.outputs.version }} \
@@ -253,12 +256,15 @@ jobs:
             --filter='!./examples/*' \
             --filter='!@secure-exec/website'
       - name: Finalize package versions for publish
+        env:
+          PUBLISH_INCLUDE_REGISTRY_PACKAGES: ${{ needs.context.outputs.trigger != 'release' && '1' || '' }}
         run: |
           pnpm --filter=publish exec tsx src/ci/bin.ts bump-versions \
             --version ${{ needs.context.outputs.version }}
       - name: Publish npm packages
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+          PUBLISH_INCLUDE_REGISTRY_PACKAGES: ${{ needs.context.outputs.trigger != 'release' && '1' || '' }}
         run: |
           pnpm --filter=publish exec tsx src/ci/bin.ts publish-npm \
             --tag ${{ needs.context.outputs.npm_tag }} \

CLAUDE.md

@@ -82,14 +82,14 @@ Every bound that protects a shared resource — memory/heap, CPU/wall-clock, fd/
 - JavaScript host-emulation config (`CreateVmConfig.jsRuntime`) mirrors esbuild's vocabulary so users carry over a known mental model. The host environment presented to guest JS is a `platform`; its values are esbuild's exactly — `node` | `browser` | `neutral` — plus the one sanctioned extension `bare` (language-only: ECMAScript spec globals + WebAssembly, nothing host-provided), for which esbuild has no equivalent. Do not invent other platform names. Wherever a JS runtime/resolution config property has an esbuild equivalent, take esbuild's name and value spelling over any other source (esbuild > tsconfig > ad-hoc); introduce a non-esbuild name only when esbuild has no equivalent concept (e.g. `moduleResolution`, `allowedBuiltins`).
 - `packages/core/` is `@secure-exec/core`, the generic TypeScript protocol, client, descriptor, and runtime asset package.
 - `packages/build-tools/` is `@secure-exec/build-tools`, the workspace-only generator package for V8 bridge and base filesystem assets. A fresh checkout must run `pnpm install` before any `cargo` build (including when a downstream like agent-os path-deps these crates): `v8-runtime/build.rs` generates the V8 bridge assets from `packages/build-tools/node_modules` and panics if they are absent.
-- Registry software, filesystem, and tool packages live under `registry/` with the `@secure-exec/*` npm scope.
+- Registry software and agent packages live under `registry/` with the `@agentos-software/*` npm scope (tool packages keep `@secure-exec/*`). Their build/publish lifecycle is owned by `@rivet-dev/agentos-toolchain` (`packages/agentos-toolchain`: `stage`/`build`/`pack`/`publish`) driven by the `just registry-*` recipes; the full flow is documented in `registry/README.md`. Never add package-local copy scripts or Makefile staging — declare commands/aliases/stubs in the package's `agentos-package.json` and let `stage` populate the gitignored `bin/`.
 
 ## Build And Assets
 
 - The VM base filesystem artifact is derived from Alpine Linux, but runtime source should stay generic.
 - Rebuild the base filesystem (requires Docker) with `pnpm --dir packages/build-tools build:base-filesystem`. The one script snapshots Alpine, applies the secure-exec transforms, and writes the single canonical `packages/core/fixtures/base-filesystem.json`, mirroring the same bytes into the crate-vendored `crates/sidecar/assets/` and `crates/vfs/assets/` copies (those exist only as the `cargo publish` fallback; never hand-edit them).
 - The V8 bridge bundle is generated from `packages/build-tools/scripts/build-v8-bridge.mjs`; keep its generated assets aligned with bridge-contract changes.
-- `registry/native` owns the Rust-to-WASM command build; package-local `registry/software/*/wasm/` output is release material.
+- `registry/native` owns the Rust-to-WASM command build (`just registry-native`, or `just registry-native-cmd <name>` for one `cmd-<name>` crate); its `target/wasm32-wasip1/release/commands/` output feeds `agentos-toolchain stage`, which populates each package's gitignored `bin/` for `agentos-toolchain build` to assemble into `dist/package/`.
 
 ## npm Compatibility
 
@@ -110,13 +110,13 @@ Every bound that protects a shared resource — memory/heap, CPU/wall-clock, fd/
 
 ### Release Tracks
 
-- **secure-exec runtime** — `@secure-exec/*` npm packages and `secure-exec-*` crates; releases keep npm/crates in sync, previews are npm-only. See "Preview-publishing" and "Publishing" for details.
+- **secure-exec runtime** — `@secure-exec/*` npm packages and `secure-exec-*` crates; releases keep npm/crates in sync, previews are npm-only. See "Release-previewing" and "Publishing" for details.
 - **`@agentos-software/*` registry packages** — generic VM software from secure-exec `registry/software/*` plus agent adapters from secure-exec `registry/agent/*`; versioned independently of secure-exec runtime packages.
 - **agent-os product/API** — `@rivet-dev/agentos*`, AgentOs APIs, sidecar wrapper, docs, quickstarts, and examples; see agent-os `CLAUDE.md` for its pinning workflow.
 
-### Preview-publishing
+### Release-previewing
 
-Dispatch `.github/workflows/publish.yaml` (workflow_dispatch) with no version input to cut a **preview** (debug sidecar build, npm-only, dist-tag = sanitized branch name) — for handing a build to a downstream (agent-os) or external project. **Preview-publish is for previews ONLY; never cut a release with it.** Caveats: WASM-bearing packages (`@secure-exec/core`, `@agentos-software/*`) publish MANUALLY (see Publishing), and the crates.io job is skipped on preview — a *crate* change only reaches consumers locally (path dep / `[patch]`) or via a real release.
+`just release-preview <branch>` dispatches `.github/workflows/publish.yaml` (workflow_dispatch, no version input) to cut a **preview** (debug sidecar build, npm-only, dist-tag = sanitized branch name) — for handing a build to a downstream (agent-os) or external project. **Preview-publish is for previews ONLY; never cut a release with it.** Caveats: WASM-bearing packages (`@secure-exec/core`, `@agentos-software/*`) publish MANUALLY (see Publishing), and the crates.io job is skipped on preview — a *crate* change only reaches consumers locally (path dep / `[patch]`) or via a real release.
 
 ### Testing a local build from an external project (same machine)
 
@@ -125,11 +125,16 @@ Dispatch `.github/workflows/publish.yaml` (workflow_dispatch) with no version in
 
 ## Publishing
 
+Workflow skills (follow these rather than improvising):
+- `.claude/skills/release` — stable release (npm + crates.io lockstep, incl. the manual `@secure-exec/core` wasm publish).
+- `.claude/skills/release-preview` — branch preview (`just release-preview <branch>`; npm-only, branch dist-tag, registry packages included).
+- `.claude/skills/publish-registry` — `@agentos-software/*` registry packages (per-package semver, `dev` tag default, `latest` deliberate).
+- agent-os side: its `.claude/skills/{bump-secure-exec,release-preview,release}` cover consuming/releasing against secure-exec.
+
 - **The `@secure-exec/*` npm packages and the `secure-exec-*` Cargo crates are always published at the same version** (npm and crates stay in sync), so a downstream pins both to one `<v>`. See "Release Tracks" for how this differs from `@agentos-software/*` and agent-os releases.
-- CI (`.github/workflows/publish.yaml`) does NOT build or publish the WASM command binaries. There is no `build-commands` job and nothing restores a `wasm-commands` artifact — the workflow only builds/publishes the sidecar binary and the pure-TS packages.
-- WASM-bearing packages are ALWAYS published MANUALLY: `@secure-exec/core` (which vendors `registry/native` commands into `packages/core/commands` via `copy-wasm-commands.mjs`, guarded by its `prepack --require`) and the `@agentos-software/*` registry software. `@secure-exec/core` is in `EXCLUDED` in `scripts/publish/src/lib/packages.ts`, so CI never publishes it.
-- Manual core flow: build the commands locally (`make -C registry/native wasm`), then `npm publish` (not `pnpm publish`) `@secure-exec/core` at the **same version** CI used for that release so dependents resolving `@secure-exec/core@<version>` succeed. `prepack` vendors the commands and fails loud if they are absent.
-- Rationale: building WASM in CI was slow/flaky and repeatedly shipped tarballs missing the command set (the `wasm/` output is a gitignored build artifact). Keeping the WASM publish manual makes the vendored command set authoritative and avoids empty-package regressions.
+- CI (`.github/workflows/publish.yaml`) DOES build and vendor the core WASM command set: the "Build and vendor core WASM commands" step runs `make -C registry/native wasm` + `packages/core run copy-commands` before the npm publish, and core's `prepack` fails loud if the commands are absent — so a published `@secure-exec/core` tarball always carries the command set. (`EXCLUDED` in `scripts/publish/src/lib/packages.ts` contains only the private `publish` package.)
+- `@agentos-software/*` registry software releases stay MANUAL and per-package (`just registry-publish <pkg> [tag]`; dist-tag `dev` unless `latest` is passed deliberately); PREVIEWS include them automatically under the branch dist-tag (`PUBLISH_INCLUDE_REGISTRY_PACKAGES`).
+- `copy-wasm-commands.mjs` (core's vendoring) is the ONE sanctioned package-local copy script: it vendors the baseline command set into the published `@secure-exec/core` tarball at build/prepack time. Every registry package instead declares commands in `agentos-package.json` and lets `agentos-toolchain stage` populate `bin/`.
 
 ## Website
 

justfile

@@ -20,10 +20,83 @@ docs-verify:
 	node website/scripts/compare-fixture.mjs
 	node website/scripts/compare-visual.mjs
 
+# --- Registry (@agentos-software/* packages) -------------------------------
+# Full flow + package format: registry/README.md.
+
+# Compile ALL native wasm command binaries (slow; needed once per checkout)
+registry-native:
+	make -C registry/native wasm
+
+# Recompile ONE command binary (cargo package cmd-<CMD>), e.g. `just registry-native-cmd sh`
+registry-native-cmd CMD:
+	make -C registry/native wasm-cmd CMD="{{ CMD }}"
+
+# Build one registry package (stage bin/ + tsc + assemble dist/package), or all when PKG is empty.
+# Bootstrap note: on a fresh checkout the `agentos-toolchain` bin symlinks are
+# only created by `pnpm install` AFTER the toolchain's dist exists — so prime it.
+registry-build PKG="":
+	#!/usr/bin/env bash
+	set -euo pipefail
+	npx turbo build --filter '@rivet-dev/agentos-toolchain' >/dev/null
+	pnpm install --frozen-lockfile >/dev/null
+	if [ -n "{{ PKG }}" ]; then
+		dir=""
+		for base in registry/software registry/agent; do
+			[ -d "$base/{{ PKG }}" ] && dir="$base/{{ PKG }}"
+		done
+		[ -n "$dir" ] || { echo "ERROR: no registry/software/{{ PKG }} or registry/agent/{{ PKG }}"; exit 1; }
+		npx turbo build --filter "./$dir"
+	else
+		npx turbo build --filter './registry/software/*' --filter './registry/agent/*'
+	fi
+
+# Run the registry integration tests (registry/tests)
+registry-test *args:
+	pnpm --dir registry test "$@"
+
+# Publish one software/agent package. Dist-tag defaults to `dev`; pass
+# TAG=latest ONLY for a deliberate release (it moves the latest pointer).
+registry-publish PKG TAG="dev":
+	#!/usr/bin/env bash
+	set -euo pipefail
+	dir=""
+	for base in registry/software registry/agent; do
+		[ -d "$base/{{ PKG }}" ] && dir="$base/{{ PKG }}"
+	done
+	[ -n "$dir" ] || { echo "ERROR: no registry/software/{{ PKG }} or registry/agent/{{ PKG }}"; exit 1; }
+	if [ "{{ TAG }}" = "latest" ]; then
+		node packages/agentos-toolchain/dist/cli.js publish "$dir" --latest
+	else
+		node packages/agentos-toolchain/dist/cli.js publish "$dir" --tag "{{ TAG }}"
+	fi
+
+# Publish ALL built software packages (skips unbuilt ones with a notice)
+registry-publish-all TAG="dev":
+	#!/usr/bin/env bash
+	set -euo pipefail
+	for dir in registry/software/*/; do
+		[ -f "$dir/package.json" ] || continue
+		if [ ! -f "$dir/dist/index.js" ]; then
+			echo "SKIP: $dir (not built)"
+			continue
+		fi
+		if [ "{{ TAG }}" = "latest" ]; then
+			node packages/agentos-toolchain/dist/cli.js publish "$dir" --latest
+		else
+			node packages/agentos-toolchain/dist/cli.js publish "$dir" --tag "{{ TAG }}"
+		fi
+	done
+
+# Show per-package state (version, staged bin/, assembled dist). --remote adds npm dist-tags.
+registry-status *args:
+	node registry/scripts/status.mjs "$@"
+
 release *args:
 	pnpm --filter=publish release "$@"
 
-preview-publish REF:
+# Cut a release-preview (debug build, npm-only, branch dist-tag; also publishes
+# the registry packages under that tag) — see the release-preview skill.
+release-preview REF:
 	gh workflow run .github/workflows/publish.yaml --ref "{{ REF }}"
 
 test-bounded cmd='pnpm test':