|
| 1 | +#!/usr/bin/env bash |
| 2 | +# |
| 3 | +# clone-and-build-codex-wasi.sh — REPRODUCIBLE wasm32-wasip1 build of the codex-exec |
| 4 | +# `--session-turn` engine from the pinned codex WASI fork. |
| 5 | +# |
| 6 | +# Unlike scripts/build-codex-wasi.sh (which builds an EXISTING local checkout and |
| 7 | +# relies on that machine's [patch.crates-io] paths + a pre-patched crates.io cache), |
| 8 | +# this script is self-contained and reproducible from a clean environment: |
| 9 | +# |
| 10 | +# 1. Read the pin from toolchain/codex-ref ("<owner>/<repo>@<sha>"). |
| 11 | +# 2. Shallow-clone the fork at that exact SHA into a gitignored scratch dir. |
| 12 | +# 3. Rewrite the clone's [patch.crates-io] to point at the toolchain's reproducible |
| 13 | +# stubs (toolchain/stubs/{reqwest-shim,portable-pty-wasi,ctrlc}); drop tokio's |
| 14 | +# machine-path patch so tokio 1.52.3 comes through the vendored+patched tree. |
| 15 | +# 4. Strip the hardcoded `-L .../self-contained` from the clone's .cargo/config.toml |
| 16 | +# (that absolute rustup path is machine-specific; it is recomputed and passed via |
| 17 | +# RUSTFLAGS at build time instead). |
| 18 | +# 5. `cargo vendor` the workspace (+ the std library deps needed by -Z build-std) and |
| 19 | +# apply toolchain/std-patches/crates/* to the vendored sources (tokio wasi-process, |
| 20 | +# path-dedot, rustls-native-certs, socket2, ...) via scripts/patch-vendor.sh. |
| 21 | +# 6. Build codex-exec for wasm32-wasip1 by reusing the fork's own |
| 22 | +# scripts/build-wasi-codex-exec.sh (sysroot massaging + build-std + wasm-opt). |
| 23 | +# 7. Install the optimized artifact to software/codex/wasm/{codex,codex-exec}. |
| 24 | +# |
| 25 | +# Usage: |
| 26 | +# toolchain/scripts/clone-and-build-codex-wasi.sh |
| 27 | +# |
| 28 | +# Env (all optional; sensible defaults): |
| 29 | +# CODEX_BUILD_DIR scratch root for the clone/vendor/target (default: toolchain/.codex-build) |
| 30 | +# WASI_SDK_DIR wasi-sdk C toolchain (default: toolchain/c/vendor/wasi-sdk) |
| 31 | +# DEST_DIR install destination (default: software/codex/wasm) |
| 32 | +# TOOLCHAIN rust toolchain (default: nightly-2026-03-01) |
| 33 | +# CODEX_GIT_BASE git host base (default: https://github.com) |
| 34 | +# STOP_AFTER "vendor" -> stop right after vendor+patch (bounded verification: |
| 35 | +# proves clone + patch-injection + vendor/patch succeed without the |
| 36 | +# full ~29MB build). Unset -> full build + install. |
| 37 | +# KEEP_SYSROOT forwarded to the fork build script (1 = skip sysroot restore). |
| 38 | +set -euo pipefail |
| 39 | + |
| 40 | +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" |
| 41 | +TOOLCHAIN_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" |
| 42 | +AGENTOS_ROOT="$(cd "$TOOLCHAIN_DIR/.." && pwd)" |
| 43 | + |
| 44 | +STUBS="$TOOLCHAIN_DIR/stubs" |
| 45 | +CODEX_REF_FILE="$TOOLCHAIN_DIR/codex-ref" |
| 46 | + |
| 47 | +BUILD_ROOT="${CODEX_BUILD_DIR:-$TOOLCHAIN_DIR/.codex-build}" |
| 48 | +CHECKOUT="$BUILD_ROOT/checkout" # git repo root (rivet-dev/codex) |
| 49 | +WORKSPACE="$CHECKOUT/codex-rs" # cargo workspace (the fork nests it here) |
| 50 | +WASI_SDK_DIR="${WASI_SDK_DIR:-$TOOLCHAIN_DIR/c/vendor/wasi-sdk}" |
| 51 | +DEST_DIR="${DEST_DIR:-$AGENTOS_ROOT/software/codex/wasm}" |
| 52 | +TOOLCHAIN="${TOOLCHAIN:-nightly-2026-03-01}" |
| 53 | +CODEX_GIT_BASE="${CODEX_GIT_BASE:-https://github.com}" |
| 54 | +STOP_AFTER="${STOP_AFTER:-}" |
| 55 | + |
| 56 | +# --- preflight --------------------------------------------------------------- |
| 57 | +[ -f "$CODEX_REF_FILE" ] || { echo "ERROR: pin file not found: $CODEX_REF_FILE" >&2; exit 1; } |
| 58 | +[ -x "$WASI_SDK_DIR/bin/clang" ] || { |
| 59 | + echo "ERROR: wasi-sdk clang not found at $WASI_SDK_DIR/bin/clang" >&2 |
| 60 | + echo " Run: make -C toolchain/c wasi-sdk" >&2 |
| 61 | + exit 1 |
| 62 | +} |
| 63 | +command -v cargo >/dev/null 2>&1 || { echo "ERROR: cargo not on PATH" >&2; exit 1; } |
| 64 | + |
| 65 | +# --- 1. parse the pin -------------------------------------------------------- |
| 66 | +REF="$(tr -d '[:space:]' < "$CODEX_REF_FILE")" |
| 67 | +REPO="${REF%@*}" # owner/repo |
| 68 | +SHA="${REF#*@}" # commit sha |
| 69 | +[ -n "$REPO" ] && [ -n "$SHA" ] && [ "$REPO" != "$SHA" ] || { |
| 70 | + echo "ERROR: malformed codex-ref '$REF' (expected '<owner>/<repo>@<sha>')" >&2; exit 1; } |
| 71 | +URL="$CODEX_GIT_BASE/$REPO.git" |
| 72 | + |
| 73 | +echo "== codex-ref: $REPO @ $SHA ==" |
| 74 | +echo "== clone url: $URL ==" |
| 75 | +echo "== scratch: $BUILD_ROOT ==" |
| 76 | + |
| 77 | +# --- 2. shallow clone at the exact SHA (idempotent by SHA) ------------------- |
| 78 | +STAMP="$CHECKOUT/.codex-built-sha" |
| 79 | +if [ ! -f "$STAMP" ] || [ "$(cat "$STAMP" 2>/dev/null)" != "$SHA" ]; then |
| 80 | + echo "== fetching fork at $SHA ==" |
| 81 | + rm -rf "$CHECKOUT" |
| 82 | + mkdir -p "$CHECKOUT" |
| 83 | + git -C "$CHECKOUT" init -q |
| 84 | + git -C "$CHECKOUT" remote add origin "$URL" |
| 85 | + # Prefer a shallow fetch of the exact commit (GitHub allows reachable SHA-1 |
| 86 | + # wants). Fall back to an unshallow fetch if the host rejects arbitrary SHAs. |
| 87 | + if ! git -C "$CHECKOUT" fetch --depth 1 origin "$SHA" 2>/dev/null; then |
| 88 | + echo " (shallow SHA fetch unsupported; fetching history)" |
| 89 | + git -C "$CHECKOUT" fetch origin |
| 90 | + fi |
| 91 | + git -C "$CHECKOUT" checkout -q "$SHA" |
| 92 | + echo "$SHA" > "$STAMP" |
| 93 | +else |
| 94 | + echo "== reusing existing clone at $SHA ==" |
| 95 | +fi |
| 96 | +[ -f "$WORKSPACE/Cargo.toml" ] || { |
| 97 | + echo "ERROR: expected cargo workspace at $WORKSPACE/Cargo.toml" >&2; exit 1; } |
| 98 | + |
| 99 | +# --- 3. rewrite [patch.crates-io] -> reproducible toolchain stubs ------------ |
| 100 | +echo "== rewriting [patch.crates-io] -> toolchain/stubs/* ==" |
| 101 | +python3 - "$WORKSPACE/Cargo.toml" "$STUBS" <<'PY' |
| 102 | +import re, sys |
| 103 | +cargo, stubs = sys.argv[1], sys.argv[2] |
| 104 | +lines = open(cargo).read().split('\n') |
| 105 | +# Crates whose patch we own: reqwest/portable-pty/ctrlc get repointed at the |
| 106 | +# reproducible toolchain stubs; tokio's patch is dropped so tokio 1.52.3 resolves |
| 107 | +# through the vendored+patched sources (std-patches/crates/tokio/0001). |
| 108 | +OWNED = ('portable-pty', 'reqwest', 'ctrlc', 'tokio') |
| 109 | +owned_re = re.compile(r'^\s*#?\s*(?:%s)\s*=' % '|'.join(map(re.escape, OWNED))) |
| 110 | +inject = [ |
| 111 | + '# [patch.crates-io] injected by clone-and-build-codex-wasi.sh (reproducible stubs)', |
| 112 | + 'portable-pty = {{ path = "{}/portable-pty-wasi" }}'.format(stubs), |
| 113 | + 'reqwest = {{ path = "{}/reqwest-shim" }}'.format(stubs), |
| 114 | + 'ctrlc = {{ path = "{}/ctrlc" }}'.format(stubs), |
| 115 | + '# tokio: vendored+patched tokio 1.52.3 (std-patches/crates/tokio), not a path patch', |
| 116 | +] |
| 117 | +in_patch = False |
| 118 | +injected = False |
| 119 | +out = [] |
| 120 | +for ln in lines: |
| 121 | + s = ln.strip() |
| 122 | + if s.startswith('[') and s.endswith(']'): |
| 123 | + in_patch = (s == '[patch.crates-io]') |
| 124 | + out.append(ln) |
| 125 | + if in_patch: # inject our lines right after the header |
| 126 | + out.extend(inject); injected = True |
| 127 | + continue |
| 128 | + if in_patch and owned_re.match(ln): # drop any prior line for an owned crate |
| 129 | + continue |
| 130 | + out.append(ln) |
| 131 | +if not injected: # no [patch.crates-io] in the fork: add one |
| 132 | + out += ['', '[patch.crates-io]'] + inject |
| 133 | +open(cargo, 'w').write('\n'.join(out)) |
| 134 | +PY |
| 135 | +echo " --- resulting [patch.crates-io] head ---" |
| 136 | +sed -n '/^\[patch.crates-io\]/,/^\[/p' "$WORKSPACE/Cargo.toml" | sed 's/^/ /' |
| 137 | + |
| 138 | +# --- 4. strip the hardcoded -L .../self-contained from .cargo/config.toml ---- |
| 139 | +CLONE_CARGO_CFG="$WORKSPACE/.cargo/config.toml" |
| 140 | +if [ -f "$CLONE_CARGO_CFG" ]; then |
| 141 | + echo "== stripping machine -L from .cargo/config.toml ==" |
| 142 | + python3 - "$CLONE_CARGO_CFG" <<'PY' |
| 143 | +import re, sys |
| 144 | +p = sys.argv[1] |
| 145 | +s = open(p).read() |
| 146 | +# Drop the "-C", "link-arg=-L<abs>/self-contained" pair anywhere in a rustflags |
| 147 | +# array; the self-contained dir is recomputed and passed via RUSTFLAGS at build. |
| 148 | +s = re.sub(r'"-C",\s*"link-arg=-L[^"]*self-contained",\s*', '', s) |
| 149 | +open(p, 'w').write(s) |
| 150 | +PY |
| 151 | +fi |
| 152 | + |
| 153 | +# --- 5. vendor the workspace (+ std deps) and apply crate patches ------------ |
| 154 | +RUST_STD_SRC="$(rustc "+$TOOLCHAIN" --print sysroot)/lib/rustlib/src/rust" |
| 155 | +[ -d "$RUST_STD_SRC/library/std" ] || { |
| 156 | + echo "ERROR: rust-src not found at $RUST_STD_SRC (rustup component add rust-src)" >&2; exit 1; } |
| 157 | + |
| 158 | +echo "== cargo vendor (workspace + std library deps for -Z build-std) ==" |
| 159 | +cd "$WORKSPACE" |
| 160 | +mkdir -p .cargo |
| 161 | +cargo "+$TOOLCHAIN" vendor \ |
| 162 | + --sync "$RUST_STD_SRC/library/std/Cargo.toml" \ |
| 163 | + --sync "$RUST_STD_SRC/library/test/Cargo.toml" \ |
| 164 | + "$WORKSPACE/vendor" > "$WORKSPACE/.cargo/vendor-sources.toml" |
| 165 | + |
| 166 | +# Append the source-replacement config so cargo builds from the patched vendor/ |
| 167 | +# tree. Guard against double-append on a reused clone. |
| 168 | +if ! grep -q 'source.crates-io' "$CLONE_CARGO_CFG" 2>/dev/null; then |
| 169 | + printf '\n' >> "$CLONE_CARGO_CFG" |
| 170 | + cat "$WORKSPACE/.cargo/vendor-sources.toml" >> "$CLONE_CARGO_CFG" |
| 171 | +fi |
| 172 | + |
| 173 | +echo "== applying toolchain/std-patches/crates/* to vendored sources ==" |
| 174 | +# patch-vendor.sh exits non-zero if ANY crate patch fails to apply. codex's |
| 175 | +# dependency graph overlaps the command build's only partially, so some patches |
| 176 | +# legitimately do not apply and are BENIGN for codex: |
| 177 | +# - crossterm: codex uses the nornagon crossterm *git fork* (its own wasi |
| 178 | +# support), not the crates.io crossterm the patch targets. |
| 179 | +# - socket2 0.6.4: no longer carries the `not(target_env="p1")` exclusion the |
| 180 | +# patch removes; codex builds against stock socket2 (verified in the fork). |
| 181 | +# So don't let its exit code abort us; instead assert the patches codex REQUIRES. |
| 182 | +VENDOR_DIR="$WORKSPACE/vendor" "$SCRIPT_DIR/patch-vendor.sh" || \ |
| 183 | + echo " (patch-vendor reported failures; verifying codex-critical patches below)" |
| 184 | + |
| 185 | +echo "== verifying codex-critical crate patches applied ==" |
| 186 | +assert_patched() { # <file> <needle> <label> |
| 187 | + if grep -q -- "$2" "$1" 2>/dev/null; then |
| 188 | + echo " OK: $3" |
| 189 | + else |
| 190 | + echo "ERROR: required patch missing: $3 ($1)" >&2 |
| 191 | + exit 1 |
| 192 | + fi |
| 193 | +} |
| 194 | +assert_patched "$WORKSPACE/vendor/path-dedot/src/lib.rs" \ |
| 195 | + 'cfg(any(unix, target_family = "wasm"))' 'path-dedot wasi unix-paths' |
| 196 | +assert_patched "$WORKSPACE/vendor/rustls-native-certs/src/lib.rs" \ |
| 197 | + 'target_os = "wasi"' 'rustls-native-certs wasi empty-certs' |
| 198 | +assert_patched "$WORKSPACE/vendor/tokio/src/process/mod.rs" \ |
| 199 | + 'path = "wasi.rs"' 'tokio wasi-process routing' |
| 200 | +[ -f "$WORKSPACE/vendor/tokio/src/process/wasi.rs" ] || { |
| 201 | + echo "ERROR: tokio companion src/process/wasi.rs not installed" >&2; exit 1; } |
| 202 | +echo " OK: tokio wasi.rs companion installed" |
| 203 | + |
| 204 | +if [ "$STOP_AFTER" = "vendor" ]; then |
| 205 | + echo "" |
| 206 | + echo "== STOP_AFTER=vendor: verifying codex crates compile past the patched frontier ==" |
| 207 | + SELF_CONTAINED="$(rustc "+$TOOLCHAIN" --print sysroot)/lib/rustlib/wasm32-wasip1/lib/self-contained" |
| 208 | + export CC_wasm32_wasip1="$WASI_SDK_DIR/bin/clang" |
| 209 | + export AR_wasm32_wasip1="$WASI_SDK_DIR/bin/llvm-ar" |
| 210 | + export CFLAGS_wasm32_wasip1="--sysroot=$WASI_SDK_DIR/share/wasi-sysroot -D_WASI_EMULATED_SIGNAL -D_WASI_EMULATED_PTHREAD -D_WASI_EMULATED_MMAN -D_WASI_EMULATED_PROCESS_CLOCKS" |
| 211 | + # Compile-check the crates whose wasi patches this script injects (path-dedot, |
| 212 | + # rustls-native-certs, tokio) plus their reverse-deps, without the full link. |
| 213 | + RUSTFLAGS="-C link-arg=-L$SELF_CONTAINED --cfg tokio_unstable" \ |
| 214 | + cargo "+$TOOLCHAIN" build --target wasm32-wasip1 -Z build-std \ |
| 215 | + -p path-dedot -p rustls-native-certs -p tokio |
| 216 | + echo "== vendor/patch frontier OK ==" |
| 217 | + exit 0 |
| 218 | +fi |
| 219 | + |
| 220 | +# --- 6. build via the fork's own reproducible build script ------------------- |
| 221 | +echo "== building codex-exec (fork scripts/build-wasi-codex-exec.sh) ==" |
| 222 | +SELF_CONTAINED="$(rustc "+$TOOLCHAIN" --print sysroot)/lib/rustlib/wasm32-wasip1/lib/self-contained" |
| 223 | +BUILD_SCRIPT="$WORKSPACE/scripts/build-wasi-codex-exec.sh" |
| 224 | +[ -x "$BUILD_SCRIPT" ] || { echo "ERROR: fork build script missing: $BUILD_SCRIPT" >&2; exit 1; } |
| 225 | + |
| 226 | +INSTALL=0 \ |
| 227 | +TOOLCHAIN="$TOOLCHAIN" \ |
| 228 | +KEEP_SYSROOT="${KEEP_SYSROOT:-0}" \ |
| 229 | +WASI_SDK_DIR="$WASI_SDK_DIR" \ |
| 230 | +RUSTFLAGS="-C link-arg=-L$SELF_CONTAINED --cfg tokio_unstable" \ |
| 231 | + "$BUILD_SCRIPT" |
| 232 | + |
| 233 | +# --- 7. install optimized artifact ------------------------------------------ |
| 234 | +OUT="$WORKSPACE/target/wasm32-wasip1/release/codex-exec.opt.wasm" |
| 235 | +[ -f "$OUT" ] || { echo "ERROR: expected build output missing: $OUT" >&2; exit 1; } |
| 236 | +mkdir -p "$DEST_DIR" |
| 237 | +cp "$OUT" "$DEST_DIR/codex-exec" |
| 238 | +cp "$OUT" "$DEST_DIR/codex" |
| 239 | +echo "== installed $(wc -c < "$OUT") bytes to $DEST_DIR/{codex-exec,codex} ==" |
| 240 | +echo "DONE" |
0 commit comments