Skip to content

Commit d0af02d

Browse files
yuki-97yuki-97
authored
Merge branch 'main' into ci/bump-nanov3-async-gym-timeout
2 parents b6357cd + a3fee46 commit d0af02d

33 files changed

Lines changed: 900 additions & 55 deletions

‎.github/workflows/config/.secrets.baseline‎

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,42 @@
161161
"line_number": 11
162162
}
163163
],
164+
"examples/configs/recipes/llm/performance/grpo-nemotron3-super-120BA12B-32n4g-async-1off.yaml": [
165+
{
166+
"type": "Base64 High Entropy String",
167+
"filename": "examples/configs/recipes/llm/performance/grpo-nemotron3-super-120BA12B-32n4g-async-1off.yaml",
168+
"hashed_secret": "4702220fec9261c7d9b0c427aceeca118b66c702",
169+
"is_verified": false,
170+
"line_number": 21
171+
}
172+
],
173+
"examples/configs/recipes/llm/performance/grpo-nemotron3-super-120BA12B-32n4g.yaml": [
174+
{
175+
"type": "Base64 High Entropy String",
176+
"filename": "examples/configs/recipes/llm/performance/grpo-nemotron3-super-120BA12B-32n4g.yaml",
177+
"hashed_secret": "4702220fec9261c7d9b0c427aceeca118b66c702",
178+
"is_verified": false,
179+
"line_number": 15
180+
}
181+
],
182+
"examples/configs/recipes/llm/performance/grpo-nemotron3-super-120BA12B-32n8g-async-1off.yaml": [
183+
{
184+
"type": "Base64 High Entropy String",
185+
"filename": "examples/configs/recipes/llm/performance/grpo-nemotron3-super-120BA12B-32n8g-async-1off.yaml",
186+
"hashed_secret": "4702220fec9261c7d9b0c427aceeca118b66c702",
187+
"is_verified": false,
188+
"line_number": 16
189+
}
190+
],
191+
"examples/configs/recipes/llm/performance/grpo-nemotron3-super-120BA12B-32n8g.yaml": [
192+
{
193+
"type": "Base64 High Entropy String",
194+
"filename": "examples/configs/recipes/llm/performance/grpo-nemotron3-super-120BA12B-32n8g.yaml",
195+
"hashed_secret": "4702220fec9261c7d9b0c427aceeca118b66c702",
196+
"is_verified": false,
197+
"line_number": 11
198+
}
199+
],
164200
"tests/unit/test_version_check.py": [
165201
{
166202
"type": "Hex High Entropy String",
@@ -171,5 +207,5 @@
171207
}
172208
]
173209
},
174-
"generated_at": "2026-06-30T19:40:15Z"
210+
"generated_at": "2026-07-15T20:07:55Z"
175211
}

‎docker/install_apptainer.sh‎

Lines changed: 124 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,126 @@
11
#!/bin/bash
22
set -euo pipefail
3-
apt-get update
4-
apt-get install -y --no-install-recommends software-properties-common
5-
add-apt-repository -y ppa:apptainer/ppa
6-
apt-get update
7-
apt-get install -y --no-install-recommends apptainer
8-
ln -sf /usr/bin/apptainer /usr/bin/singularity
9-
apt-get clean && rm -rf /var/lib/apt/lists/*
3+
4+
APPTAINER_VERSION=1.4.5
5+
APPTAINER_DEB_SHA256=70f19af846501acfbc2e42e7cfeee9ee11ddbbfa1c3502d0d99cde34e8e0af05
6+
APPTAINER_SOURCE_SHA256=d323a8b9a0a9e5e131b396d0049fdaa99beceb83a3d7ffb80dd91d15331e3b9a
7+
GO_VERSION=1.23.6
8+
GO_ARM64_SHA256=561c780e8f4a8955d32bf72e46af0b5ee5e0debe1e4633df9a03781878219202
9+
DEB_ARCH="$(dpkg --print-architecture)"
10+
export DEBIAN_FRONTEND=noninteractive
11+
12+
install_amd64_deb() {
13+
local deb_path="/tmp/apptainer_${APPTAINER_VERSION}_amd64.deb"
14+
local deb_url="https://github.com/apptainer/apptainer/releases/download/v${APPTAINER_VERSION}/apptainer_${APPTAINER_VERSION}_amd64.deb"
15+
16+
apt-get update
17+
apt-get install -y --no-install-recommends ca-certificates wget
18+
wget --progress=dot:giga -O "${deb_path}" "${deb_url}"
19+
echo "${APPTAINER_DEB_SHA256} ${deb_path}" | sha256sum -c -
20+
apt-get install -y --no-install-recommends "${deb_path}"
21+
rm -f "${deb_path}"
22+
}
23+
24+
install_go_arm64() {
25+
local go_tarball="/tmp/go${GO_VERSION}.linux-arm64.tar.gz"
26+
27+
wget --progress=dot:giga -O "${go_tarball}" "https://go.dev/dl/go${GO_VERSION}.linux-arm64.tar.gz"
28+
echo "${GO_ARM64_SHA256} ${go_tarball}" | sha256sum -c -
29+
rm -rf /usr/local/go
30+
tar -C /usr/local -xzf "${go_tarball}"
31+
rm -f "${go_tarball}"
32+
export PATH="/usr/local/go/bin:${PATH}"
33+
}
34+
35+
verify_install() {
36+
apptainer --version
37+
singularity --version
38+
}
39+
40+
install_arm64_from_source() {
41+
local build_dir="/tmp/apptainer-build"
42+
local source_tarball="/tmp/apptainer-${APPTAINER_VERSION}.tar.gz"
43+
local source_url="https://github.com/apptainer/apptainer/releases/download/v${APPTAINER_VERSION}/apptainer-${APPTAINER_VERSION}.tar.gz"
44+
local build_packages=(
45+
autoconf
46+
automake
47+
build-essential
48+
curl
49+
dh-apparmor
50+
git
51+
libfuse3-dev
52+
liblzo2-dev
53+
liblz4-dev
54+
liblzma-dev
55+
libseccomp-dev
56+
libsubid-dev
57+
libtool
58+
libzstd-dev
59+
pkg-config
60+
wget
61+
zlib1g-dev
62+
)
63+
local runtime_packages=(
64+
ca-certificates
65+
cryptsetup
66+
fakeroot
67+
fuse3
68+
libfuse3-3
69+
liblzo2-2
70+
liblz4-1
71+
liblzma5
72+
libseccomp2
73+
libsubid4
74+
libzstd1
75+
squashfs-tools
76+
tzdata
77+
uidmap
78+
zlib1g
79+
)
80+
81+
apt-get update
82+
apt-get install -y --no-install-recommends "${runtime_packages[@]}" "${build_packages[@]}"
83+
84+
install_go_arm64
85+
86+
rm -rf "${build_dir}"
87+
mkdir -p "${build_dir}"
88+
wget --progress=dot:giga -O "${source_tarball}" "${source_url}"
89+
echo "${APPTAINER_SOURCE_SHA256} ${source_tarball}" | sha256sum -c -
90+
tar -C "${build_dir}" --strip-components=1 -xzf "${source_tarball}"
91+
rm -f "${source_tarball}"
92+
93+
cd "${build_dir}"
94+
./scripts/download-dependencies
95+
./scripts/compile-dependencies
96+
./mconfig --without-suid
97+
make -C builddir
98+
make -C builddir install
99+
./scripts/install-dependencies
100+
rm -rf "${build_dir}"
101+
102+
apt-get install -y --no-install-recommends "${runtime_packages[@]}"
103+
apt-get purge -y --auto-remove "${build_packages[@]}"
104+
rm -rf /usr/local/go /root/go /root/.cache/go-build
105+
}
106+
107+
case "${DEB_ARCH}" in
108+
amd64)
109+
install_amd64_deb
110+
ln -sf /usr/bin/apptainer /usr/bin/singularity
111+
verify_install
112+
;;
113+
arm64)
114+
install_arm64_from_source
115+
ln -sf /usr/local/bin/apptainer /usr/bin/apptainer
116+
ln -sf /usr/local/bin/apptainer /usr/bin/singularity
117+
verify_install
118+
;;
119+
*)
120+
echo "Unsupported architecture for Apptainer ${APPTAINER_VERSION}: ${DEB_ARCH}" >&2
121+
exit 1
122+
;;
123+
esac
124+
125+
apt-get clean
126+
rm -rf /var/lib/apt/lists/*

‎docs/guides/nemotron-3-super.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ docker buildx build \
2222
FROM nvcr.io/nvidia/nemo-rl:v0.7.0
2323
2424
ARG NEMO_GYM_CUDA=cu130
25-
ARG NEMO_GYM_VLLM_VERSION=0.17.0
25+
ARG NEMO_GYM_VLLM_VERSION=0.20.0
2626
ARG WHEEL_ARCH=x86_64
2727
RUN <<'RUNEOF' bash -exu -o pipefail
2828
cd /opt/nemo-rl

‎examples/configs/recipes/llm/grpo-qwen3-1.7b-6n8g-megatron-super-rlvr.yaml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@
44
#
55
# Resource layout (6 nodes total):
66
# - 1 node: policy training + colocated vllm (Qwen3-1.7B is tiny)
7-
# - 5 nodes: nemo-gym judge servers (genrm, nl2bash, safety)
8-
# inherited from stage1_rlvr.yaml (env.nemo_gym.num_gpu_nodes: 5)
7+
# - 5 nodes: nemo-gym judge servers
8+
# genrm dp=2×tp=8 (2 nodes) + nl2bash dp=2×tp=8 (2 nodes) + safety dp=8×tp=1 (1 node)
99
#
1010
# Data: nvidia/Nemotron-RL-Super-Training-Blends (rlvr1 split).
1111
# See docs/guides/nemotron-3-super.md for download instructions.
@@ -53,6 +53,7 @@ data:
5353
env:
5454
nemo_gym:
5555
uv_venv_dir: "${oc.env:HF_HOME}/gym_venvs"
56+
num_gpu_nodes: 5
5657
genrm_compare_resources_server:
5758
resources_servers:
5859
genrm_compare:
@@ -61,6 +62,7 @@ env:
6162
responses_api_models:
6263
genrm_model:
6364
vllm_serve_kwargs:
65+
data_parallel_size: 2 # reduced from 4 to fit within 5 gym nodes (2+2+1)
6466
max_num_seqs: 16
6567
nl2bash_judge_model:
6668
responses_api_models:

‎examples/configs/recipes/llm/performance/grpo-nemotron3-super-120BA12B-32n4g-async-1off.yaml‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
defaults: ../../../grpo_math_1B.yaml
2+
3+
grpo:
4+
num_prompts_per_step: 32
5+
num_generations_per_prompt: 8
6+
7+
async_grpo:
8+
enabled: true
9+
max_trajectory_age_steps: 1
10+
in_flight_weight_updates: true
11+
recompute_kv_cache_after_weight_updates: false
12+
13+
loss_fn:
14+
use_importance_sampling_correction: true
15+
force_on_policy_ratio: true
16+
17+
checkpointing:
18+
checkpoint_dir: results/grpo-nemotron3-super-120BA12B-32n4g-async-1off
19+
20+
policy:
21+
model_name: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16
22+
tokenizer:
23+
name: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16
24+
train_global_batch_size: 256
25+
train_micro_batch_size: 1
26+
logprob_batch_size: 1
27+
logprob_chunk_size: 2048
28+
max_total_sequence_length: 8192
29+
make_sequence_length_divisible_by: 8
30+
dtensor_cfg:
31+
enabled: false
32+
megatron_cfg:
33+
enabled: true
34+
tensor_model_parallel_size: 2
35+
expert_model_parallel_size: 16
36+
sequence_parallel: true
37+
activation_checkpointing: true
38+
bias_activation_fusion: false
39+
moe_router_dtype: fp32
40+
moe_router_bias_update_rate: 0.001
41+
moe_router_enable_expert_bias: true
42+
defer_fp32_logits: true
43+
env_vars:
44+
PYTORCH_CUDA_ALLOC_CONF: expandable_segments:False
45+
NRL_REFIT_BUFFER_MEMORY_RATIO: "0.005"
46+
NRL_REFIT_NUM_BUFFERS: "1"
47+
# # HybridEP settings. Currently disabled
48+
# # due to incompatibility with sequence packing
49+
# moe_token_dispatcher_type: flex
50+
# moe_flex_dispatcher_backend: hybridep
51+
# moe_hybridep_num_sms: 32
52+
# MTP — disabled
53+
mtp_num_layers: 0
54+
generation:
55+
colocated:
56+
enabled: false
57+
resources:
58+
gpus_per_node: 4
59+
num_nodes: 16
60+
vllm_cfg:
61+
async_engine: true
62+
tensor_parallel_size: 4
63+
gpu_memory_utilization: 0.7
64+
env_vars:
65+
NRL_REFIT_BUFFER_MEMORY_RATIO: "0.005"
66+
NRL_REFIT_NUM_BUFFERS: "1"
67+
68+
logger:
69+
wandb_enabled: true
70+
tensorboard_enabled: true
71+
wandb:
72+
project: nemo-rl
73+
name: grpo-nemotron3-super-120BA12B-32n4g-async-1off
74+
75+
cluster:
76+
gpus_per_node: 4
77+
num_nodes: 32
78+
segment_size: 8

‎examples/configs/recipes/llm/performance/grpo-nemotron3-super-120BA12B-32n4g.yaml‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
defaults: ../../../grpo_math_1B.yaml
2+
3+
grpo:
4+
num_prompts_per_step: 32
5+
num_generations_per_prompt: 8
6+
7+
loss_fn:
8+
use_importance_sampling_correction: true
9+
force_on_policy_ratio: true
10+
11+
checkpointing:
12+
checkpoint_dir: results/grpo-nemotron3-super-120BA12B-32n4g
13+
14+
policy:
15+
model_name: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16
16+
tokenizer:
17+
name: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16
18+
train_global_batch_size: 256
19+
train_micro_batch_size: 1
20+
logprob_batch_size: 1
21+
logprob_chunk_size: 2048
22+
max_total_sequence_length: 8192
23+
make_sequence_length_divisible_by: 8
24+
dtensor_cfg:
25+
enabled: false
26+
megatron_cfg:
27+
enabled: true
28+
tensor_model_parallel_size: 2
29+
expert_model_parallel_size: 16
30+
sequence_parallel: true
31+
activation_checkpointing: true
32+
bias_activation_fusion: false
33+
moe_router_dtype: fp32
34+
moe_router_bias_update_rate: 0.001
35+
moe_router_enable_expert_bias: true
36+
# # HybridEP settings. Currently disabled
37+
# # due to incompatibility with sequence packing
38+
# moe_token_dispatcher_type: flex
39+
# moe_flex_dispatcher_backend: hybridep
40+
# moe_hybridep_num_sms: 32
41+
defer_fp32_logits: true
42+
# MTP — disabled
43+
mtp_num_layers: 0
44+
env_vars:
45+
PYTORCH_CUDA_ALLOC_CONF: expandable_segments:False
46+
NRL_REFIT_BUFFER_MEMORY_RATIO: "0.005"
47+
NRL_REFIT_NUM_BUFFERS: "1"
48+
generation:
49+
vllm_cfg:
50+
async_engine: true
51+
tensor_parallel_size: 4
52+
gpu_memory_utilization: 0.7
53+
env_vars:
54+
NRL_REFIT_BUFFER_MEMORY_RATIO: "0.005"
55+
NRL_REFIT_NUM_BUFFERS: "1"
56+
57+
logger:
58+
wandb_enabled: true
59+
tensorboard_enabled: true
60+
wandb:
61+
project: nemo-rl
62+
name: grpo-nemotron3-super-120BA12B-32n4g
63+
64+
cluster:
65+
gpus_per_node: 4
66+
num_nodes: 32
67+
segment_size: 8

0 commit comments

Comments
 (0)