Skip to content

Commit 78f8754

Browse files
HackWidMaddyHackWidMaddy
committed
codex nvidia integration
0 parents  commit 78f8754

4,508 files changed

Lines changed: 1124563 additions & 0 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.bazelignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# Without this, Bazel will consider BUILD.bazel files in
2+
# .git/sl/origbackups (which can be populated by Sapling SCM).
3+
.git
4+
codex-rs/target

‎.bazelrc‎

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
common --repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
2+
common --repo_env=BAZEL_NO_APPLE_CPP_TOOLCHAIN=1
3+
# Dummy xcode config so we don't need to build xcode_locator in repo rule.
4+
common --xcode_version_config=//:disable_xcode
5+
6+
common --disk_cache=~/.cache/bazel-disk-cache
7+
common --repo_contents_cache=~/.cache/bazel-repo-contents-cache
8+
common --repository_cache=~/.cache/bazel-repo-cache
9+
common --remote_cache_compression
10+
startup --experimental_remote_repo_contents_cache
11+
12+
common --experimental_platform_in_output_dir
13+
14+
# Runfiles strategy rationale: codex-rs/utils/cargo-bin/README.md
15+
common --noenable_runfiles
16+
17+
common --enable_platform_specific_config
18+
common:linux --host_platform=//:local_linux
19+
common:windows --host_platform=//:local_windows
20+
common --@rules_cc//cc/toolchains/args/archiver_flags:use_libtool_on_macos=False
21+
common --@llvm//config:experimental_stub_libgcc_s
22+
23+
# TODO(zbarsky): rules_rust doesn't implement this flag properly with remote exec...
24+
# common --@rules_rust//rust/settings:pipelined_compilation
25+
26+
common --incompatible_strict_action_env
27+
# Not ideal, but We need to allow dotslash to be found
28+
common:linux --test_env=PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin
29+
common:macos --test_env=PATH=/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin
30+
31+
# Pass through some env vars Windows needs to use powershell?
32+
common:windows --test_env=SYSTEMROOT
33+
common:windows --test_env=COMSPEC
34+
common:windows --test_env=WINDIR
35+
# Rust's libtest harness runs test bodies on std-spawned threads. The default
36+
# 2 MiB stack can be too small for large async test futures on Windows CI; see
37+
# https://github.com/openai/codex/pull/19067 for the motivating failure.
38+
common --test_env=RUST_MIN_STACK=8388608 # 8 MiB
39+
40+
common --test_output=errors
41+
common --bes_results_url=https://app.buildbuddy.io/invocation/
42+
common --bes_backend=grpcs://remote.buildbuddy.io
43+
common --remote_cache=grpcs://remote.buildbuddy.io
44+
common --remote_download_toplevel
45+
common --nobuild_runfile_links
46+
common --remote_timeout=3600
47+
common --noexperimental_throttle_remote_action_building
48+
common --experimental_remote_execution_keepalive
49+
common --grpc_keepalive_time=30s
50+
common --experimental_remote_downloader=grpcs://remote.buildbuddy.io
51+
52+
# This limits both in-flight executions and concurrent downloads. Even with high number
53+
# of jobs execution will still be limited by CPU cores, so this just pays a bit of
54+
# memory in exchange for higher download concurrency.
55+
common --jobs=30
56+
57+
common:remote --extra_execution_platforms=//:rbe
58+
common:remote --remote_executor=grpcs://remote.buildbuddy.io
59+
common:remote --jobs=800
60+
# TODO(team): Evaluate if this actually helps, zbarsky is not sure, everything seems bottlenecked on `core` either way.
61+
# Enable pipelined compilation since we are not bound by local CPU count.
62+
#common:remote --@rules_rust//rust/settings:pipelined_compilation
63+
64+
# GitHub Actions CI configs.
65+
common:ci --remote_download_minimal
66+
common:ci --keep_going
67+
common:ci --verbose_failures
68+
common:ci --build_metadata=REPO_URL=https://github.com/openai/codex.git
69+
common:ci --build_metadata=ROLE=CI
70+
common:ci --build_metadata=VISIBILITY=PUBLIC
71+
# rules_rust derives debug level from Bazel toolchain/compilation-mode settings,
72+
# not Cargo profiles. Keep CI Rust actions explicit and lean.
73+
common:ci --@rules_rust//rust/settings:extra_rustc_flag=-Cdebuginfo=0
74+
common:ci --@rules_rust//rust/settings:extra_exec_rustc_flag=-Cdebuginfo=0
75+
76+
# Disable disk cache in CI since we have a remote one and aren't using persistent workers.
77+
common:ci --disk_cache=
78+
79+
# Shared config for the main Bazel CI workflow.
80+
common:ci-bazel --config=ci
81+
common:ci-bazel --build_metadata=TAG_workflow=bazel
82+
# Bazel CI cross-compiles in several legs, and the V8-backed code-mode tests
83+
# are not stable in that setup yet. Keep running the rest of the Rust
84+
# integration suites through the workspace-root launcher.
85+
common:ci-bazel --test_env=CODEX_BAZEL_TEST_SKIP_FILTERS=suite::code_mode::
86+
87+
# Shared config for Bazel-backed Rust linting.
88+
build:clippy --aspects=@rules_rust//rust:defs.bzl%rust_clippy_aspect
89+
build:clippy --output_groups=+clippy_checks
90+
build:clippy --@rules_rust//rust/settings:clippy.toml=//codex-rs:clippy.toml
91+
# Keep this deny-list in sync with `codex-rs/Cargo.toml` `[workspace.lints.clippy]`.
92+
# Cargo applies those lint levels to member crates that opt into `[lints] workspace = true`
93+
# in their own `Cargo.toml`, but `rules_rust` Bazel clippy does not read Cargo lint levels.
94+
# `clippy.toml` can configure lint behavior, but it cannot set allow/warn/deny/forbid levels.
95+
build:clippy --@rules_rust//rust/settings:clippy_flag=-Dwarnings
96+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::await_holding_invalid_type
97+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::await_holding_lock
98+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::expect_used
99+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::identity_op
100+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::manual_clamp
101+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::manual_filter
102+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::manual_find
103+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::manual_flatten
104+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::manual_map
105+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::manual_memcpy
106+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::manual_non_exhaustive
107+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::manual_ok_or
108+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::manual_range_contains
109+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::manual_retain
110+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::manual_strip
111+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::manual_try_fold
112+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::manual_unwrap_or
113+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::needless_borrow
114+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::needless_borrowed_reference
115+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::needless_collect
116+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::needless_late_init
117+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::needless_option_as_deref
118+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::needless_question_mark
119+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::needless_update
120+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::redundant_clone
121+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::redundant_closure
122+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::redundant_closure_for_method_calls
123+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::redundant_static_lifetimes
124+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::trivially_copy_pass_by_ref
125+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::uninlined_format_args
126+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::unnecessary_filter_map
127+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::unnecessary_lazy_evaluations
128+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::unnecessary_sort_by
129+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::unnecessary_to_owned
130+
build:clippy --@rules_rust//rust/settings:clippy_flag=--deny=clippy::unwrap_used
131+
132+
# Shared config for Bazel-backed argument-comment-lint.
133+
build:argument-comment-lint --aspects=//tools/argument-comment-lint:lint_aspect.bzl%rust_argument_comment_lint_aspect
134+
build:argument-comment-lint --output_groups=argument_comment_lint_checks
135+
build:argument-comment-lint --@rules_rust//rust/toolchain/channel=nightly
136+
137+
# Rearrange caches on Windows so they're on the same volume as the checkout.
138+
common:ci-windows --config=ci-bazel
139+
common:ci-windows --build_metadata=TAG_os=windows
140+
common:ci-windows --repo_contents_cache=D:/a/.cache/bazel-repo-contents-cache
141+
142+
# We prefer to run the build actions entirely remotely so we can dial up the concurrency.
143+
# We have platform-specific tests, so we want to execute the tests on all platforms using the strongest sandboxing available on each platform.
144+
145+
# On linux, we can do a full remote build/test, by targeting the right (x86/arm) runners, so we have coverage of both.
146+
# Linux crossbuilds don't work until we untangle the libc constraint mess.
147+
common:ci-linux --config=ci-bazel
148+
common:ci-linux --build_metadata=TAG_os=linux
149+
common:ci-linux --config=remote
150+
common:ci-linux --strategy=remote
151+
common:ci-linux --platforms=//:rbe
152+
153+
# On mac, we can run all the build actions remotely but test actions locally.
154+
common:ci-macos --config=ci-bazel
155+
common:ci-macos --build_metadata=TAG_os=macos
156+
common:ci-macos --config=remote
157+
common:ci-macos --strategy=remote
158+
common:ci-macos --strategy=TestRunner=darwin-sandbox,local
159+
160+
# On Windows, use Linux remote execution for build actions but keep test actions
161+
# on the Windows runner so Bazel's normal test sharding and flaky-test retries
162+
# still run against Windows binaries.
163+
common:ci-windows-cross --config=ci-windows
164+
common:ci-windows-cross --build_metadata=TAG_windows_cross_compile=true
165+
common:ci-windows-cross --config=remote
166+
common:ci-windows-cross --host_platform=//:rbe
167+
common:ci-windows-cross --strategy=remote
168+
common:ci-windows-cross --strategy=TestRunner=local
169+
common:ci-windows-cross --local_test_jobs=4
170+
common:ci-windows-cross --test_env=RUST_TEST_THREADS=1
171+
# Native Windows CI still covers the PowerShell tests. The cross-built gnullvm
172+
# binaries currently hang in PowerShell AST parser tests when those binaries are
173+
# run on the Windows runner.
174+
common:ci-windows-cross --test_env=CODEX_BAZEL_TEST_SKIP_FILTERS=suite::code_mode::,powershell
175+
common:ci-windows-cross --platforms=//:windows_x86_64_gnullvm
176+
common:ci-windows-cross --extra_execution_platforms=//:rbe,//:windows_x86_64_msvc
177+
common:ci-windows-cross --extra_toolchains=//:windows_gnullvm_tests_on_msvc_host_toolchain
178+
179+
# Linux-only V8 CI config.
180+
common:ci-v8 --config=ci
181+
common:ci-v8 --build_metadata=TAG_workflow=v8
182+
common:ci-v8 --build_metadata=TAG_os=linux
183+
common:ci-v8 --config=remote
184+
common:ci-v8 --strategy=remote
185+
186+
# Source-built Bazel V8 artifacts use the in-process sandbox by default. This
187+
# does not affect Cargo's default prebuilt rusty_v8 path.
188+
common --@v8//:v8_enable_pointer_compression=True
189+
common --@v8//:v8_enable_sandbox=True
190+
191+
# Keep currently published rusty_v8 release artifacts non-sandboxed until the
192+
# artifact migration ships matching Rust feature selection for Cargo consumers.
193+
common:v8-release-compat --@v8//:v8_enable_pointer_compression=False
194+
common:v8-release-compat --@v8//:v8_enable_sandbox=False
195+
196+
# Optional per-user local overrides.
197+
try-import %workspace%/user.bazelrc

‎.bazelversion‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
9.0.0

‎.codespellignore‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
iTerm
2+
iTerm2
3+
psuedo
4+
SOM
5+
te
6+
TE

‎.codespellrc‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
[codespell]
2+
# Ref: https://github.com/codespell-project/codespell#using-a-config-file
3+
skip = .git*,vendor,*-lock.yaml,*.lock,.codespellrc,*test.ts,*.jsonl,frame*.txt,*.snap,*.snap.new
4+
check-hidden = true
5+
ignore-regex = ^\s*"image/\S+": ".*|\b(afterAll)\b
6+
ignore-words-list = ratatui,ser,iTerm,iterm2,iterm,te,TE,PASE,SEH

‎.devcontainer/Dockerfile‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
FROM ubuntu:24.04
2+
3+
ARG DEBIAN_FRONTEND=noninteractive
4+
# enable 'universe' because musl-tools & clang live there
5+
RUN apt-get update && \
6+
apt-get install -y --no-install-recommends \
7+
software-properties-common && \
8+
add-apt-repository --yes universe
9+
10+
# now install build deps
11+
RUN apt-get update && \
12+
apt-get install -y --no-install-recommends \
13+
build-essential curl git ca-certificates \
14+
pkg-config libcap-dev clang musl-tools libssl-dev just && \
15+
rm -rf /var/lib/apt/lists/*
16+
17+
# Ubuntu 24.04 ships with user 'ubuntu' already created with UID 1000.
18+
USER ubuntu
19+
20+
# install Rust + musl target as dev user
21+
RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal && \
22+
~/.cargo/bin/rustup target add aarch64-unknown-linux-musl && \
23+
~/.cargo/bin/rustup component add clippy rustfmt
24+
25+
ENV PATH="/home/ubuntu/.cargo/bin:${PATH}"
26+
27+
WORKDIR /workspace

‎.devcontainer/Dockerfile.secure‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
FROM mcr.microsoft.com/devcontainers/base:ubuntu-24.04
2+
3+
ARG TZ
4+
ARG DEBIAN_FRONTEND=noninteractive
5+
ARG NODE_MAJOR=22
6+
ARG RUST_TOOLCHAIN=1.92.0
7+
# Keep this in sync with .devcontainer/codex-install/package.json and pnpm-lock.yaml.
8+
ARG CODEX_NPM_VERSION=0.121.0
9+
10+
ENV TZ="$TZ"
11+
ENV COREPACK_ENABLE_DOWNLOAD_PROMPT=0
12+
13+
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
14+
15+
# Devcontainers run as a non-root user, so enable bubblewrap's setuid mode.
16+
RUN apt-get update \
17+
&& apt-get install -y --no-install-recommends \
18+
build-essential \
19+
curl \
20+
git \
21+
ca-certificates \
22+
pkg-config \
23+
clang \
24+
musl-tools \
25+
libssl-dev \
26+
libsqlite3-dev \
27+
just \
28+
python3 \
29+
python3-pip \
30+
jq \
31+
less \
32+
man-db \
33+
unzip \
34+
ripgrep \
35+
fzf \
36+
fd-find \
37+
zsh \
38+
dnsutils \
39+
iproute2 \
40+
ipset \
41+
iptables \
42+
aggregate \
43+
bubblewrap \
44+
&& chmod u+s /usr/bin/bwrap \
45+
&& apt-get clean \
46+
&& rm -rf /var/lib/apt/lists/*
47+
48+
COPY .devcontainer/codex-install/package.json \
49+
.devcontainer/codex-install/pnpm-lock.yaml \
50+
.devcontainer/codex-install/pnpm-workspace.yaml \
51+
/opt/codex-install/
52+
53+
RUN curl -fsSL "https://deb.nodesource.com/setup_${NODE_MAJOR}.x" | bash - \
54+
&& apt-get update \
55+
&& apt-get install -y --no-install-recommends nodejs \
56+
&& test "$(node -p "require('/opt/codex-install/package.json').dependencies['@openai/codex']")" = "${CODEX_NPM_VERSION}" \
57+
&& cd /opt/codex-install \
58+
&& corepack pnpm install --prod --frozen-lockfile \
59+
&& ln -s /opt/codex-install/node_modules/.bin/codex /usr/local/bin/codex \
60+
&& apt-get clean \
61+
&& rm -rf /var/lib/apt/lists/*
62+
63+
COPY .devcontainer/init-firewall.sh /usr/local/bin/init-firewall.sh
64+
COPY .devcontainer/post_install.py /opt/post_install.py
65+
COPY .devcontainer/post-start.sh /opt/post_start.sh
66+
67+
RUN chmod 500 /usr/local/bin/init-firewall.sh \
68+
&& chmod 755 /opt/post_start.sh \
69+
&& chmod 644 /opt/post_install.py \
70+
&& chown vscode:vscode /opt/post_install.py
71+
72+
RUN install -d -m 0775 -o vscode -g vscode /commandhistory /workspace \
73+
&& touch /commandhistory/.bash_history /commandhistory/.zsh_history \
74+
&& chown vscode:vscode /commandhistory/.bash_history /commandhistory/.zsh_history
75+
76+
USER vscode
77+
ENV PATH="/home/vscode/.cargo/bin:${PATH}"
78+
WORKDIR /workspace
79+
80+
RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain "${RUST_TOOLCHAIN}" \
81+
&& rustup component add clippy rustfmt rust-src \
82+
&& rustup target add x86_64-unknown-linux-musl aarch64-unknown-linux-musl

‎.devcontainer/README.md‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# Containerized Development
2+
3+
We provide two container paths:
4+
5+
- `devcontainer.json` keeps the existing Codex contributor setup for working on this repository.
6+
- `devcontainer.secure.json` adds a customer-oriented profile with stricter outbound network controls.
7+
8+
## Codex contributor profile
9+
10+
Use `devcontainer.json` when you are developing Codex itself. This is the same lightweight arm64 container that already exists in the repo.
11+
12+
## Secure customer profile
13+
14+
Use `devcontainer.secure.json` when you want a stricter runtime profile for running Codex inside a project container:
15+
16+
- installs the Codex CLI plus common build tools
17+
- installs bubblewrap in setuid mode for Codex's Linux sandbox
18+
- disables Docker's outer seccomp and AppArmor profiles so bubblewrap can construct Codex's inner sandbox
19+
- enables firewall startup with an allowlist-driven outbound policy
20+
- blocks IPv6 by default so the allowlist cannot be bypassed over AAAA routes
21+
- requires `NET_ADMIN` and `NET_RAW` so the firewall can be installed at startup
22+
23+
This profile keeps the stricter networking isolated to the customer path instead of changing the default Codex contributor container.
24+
25+
Start it from the CLI with:
26+
27+
```bash
28+
devcontainer up --workspace-folder . --config .devcontainer/devcontainer.secure.json
29+
```
30+
31+
In VS Code, choose **Dev Containers: Open Folder in Container...** and select `.devcontainer/devcontainer.secure.json`.
32+
33+
## Docker
34+
35+
To build the contributor image locally for x64 and then run it with the repo mounted under `/workspace`:
36+
37+
```shell
38+
CODEX_DOCKER_IMAGE_NAME=codex-linux-dev
39+
docker build --platform=linux/amd64 -t "$CODEX_DOCKER_IMAGE_NAME" ./.devcontainer
40+
docker run --platform=linux/amd64 --rm -it -e CARGO_TARGET_DIR=/workspace/codex-rs/target-amd64 -v "$PWD":/workspace -w /workspace/codex-rs "$CODEX_DOCKER_IMAGE_NAME"
41+
```
42+
43+
Note that `/workspace/target` will contain the binaries built for your host platform, so we include `-e CARGO_TARGET_DIR=/workspace/codex-rs/target-amd64` in the `docker run` command so that the binaries built inside your container are written to a separate directory.
44+
45+
For arm64, specify `--platform=linux/arm64` instead for both `docker build` and `docker run`.
46+
47+
Currently, the contributor `Dockerfile` works for both x64 and arm64 Linux, though you need to run `rustup target add x86_64-unknown-linux-musl` yourself to install the musl toolchain for x64.
48+
49+
The secure profile's capability, seccomp, and AppArmor options are required when you want Codex's bubblewrap sandbox to run inside Docker as the non-root devcontainer user. Without them, Docker's default runtime profile can block bubblewrap's namespace setup before Codex's own seccomp filter is installed. This keeps the Docker relaxation explicit in the profile that is meant to run Codex inside a project container, while the default contributor profile stays lightweight.

‎.devcontainer/codex-install/package.json‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"name": "codex-devcontainer-install",
3+
"private": true,
4+
"description": "Locked Codex CLI install boundary for the secure devcontainer.",
5+
"dependencies": {
6+
"@openai/codex": "0.121.0"
7+
},
8+
"engines": {
9+
"node": ">=22",
10+
"pnpm": ">=10.33.0"
11+
},
12+
"packageManager": "pnpm@10.33.0+sha512.10568bb4a6afb58c9eb3630da90cc9516417abebd3fabbe6739f0ae795728da1491e9db5a544c76ad8eb7570f5c4bb3d6c637b2cb41bfdcdb47fa823c8649319"
13+
}

0 commit comments

Comments
 (0)