Skip to content

Commit 7f85d2b

Browse files
s-chundis-chundi
committed
merge
2 parents 34de1c3 + f508739 commit 7f85d2b

41 files changed

Lines changed: 4872 additions & 5184 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.env.megatron‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
export UV_CACHE_DIR=/opt/dlami/nvme/uv-cache-suhaschundi
2+
export UV_PROJECT_ENVIRONMENT=/opt/dlami/nvme/skyrl-venv-suhaschundi
3+
export UV_PYTHON=3.12
4+
5+
SKYRL_SRC=/mnt/efs/suhaschundi/SkyRL
6+
SYNC_MARKER=$UV_PROJECT_ENVIRONMENT/.synced
7+
if [[ ! -d "$UV_PROJECT_ENVIRONMENT" \
8+
|| "$SKYRL_SRC/uv.lock" -nt "$SYNC_MARKER" \
9+
|| "$SKYRL_SRC/pyproject.toml" -nt "$SYNC_MARKER" ]]; then
10+
echo "[env.megatron] syncing venv to NVMe at $UV_PROJECT_ENVIRONMENT..."
11+
mkdir -p "$UV_CACHE_DIR"
12+
(cd "$SKYRL_SRC" && uv sync --frozen) && touch "$SYNC_MARKER"
13+
fi
14+
15+
# Ensure vllm is present: lock/source-override conflict makes uv silently
16+
# skip vllm on fresh venvs. Install PyPI 0.19 as a baseline; swap to local
17+
# editable for VLM runs that need a newer version.
18+
if ! "$UV_PROJECT_ENVIRONMENT/bin/python" -c "import vllm" 2>/dev/null; then
19+
echo "[env.megatron] installing vllm baseline..."
20+
uv pip install --python "$UV_PROJECT_ENVIRONMENT/bin/python" "vllm==0.19.0" --no-deps
21+
fi
22+
unset SKYRL_SRC SYNC_MARKER
23+
24+
export CUDNN_PATH=$UV_PROJECT_ENVIRONMENT/lib/python3.12/site-packages/nvidia/cudnn
25+
export CPATH=$CUDNN_PATH/include:$CPATH
26+
export LIBRARY_PATH=$CUDNN_PATH/lib:$LIBRARY_PATH
27+
export LD_LIBRARY_PATH=$CUDNN_PATH/lib:$LD_LIBRARY_PATH
28+
export LD_LIBRARY_PATH=$(python -c 'import os, glob; print(":".join(sorted(set(os.path.dirname(p) for p in glob.glob(os.environ["UV_PROJECT_ENVIRONMENT"] + "/lib/python3.12/site-packages/nvidia/*/lib/*.so*")))))'):$LD_LIBRARY_PATH
29+
30+
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
31+
export NVIDIA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
32+
export RAY_enable_metrics_agent=0

‎.github/workflows/gpu_ci_h100.yaml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,5 +65,7 @@ jobs:
6565
ANYSCALE_CLI_TOKEN: ${{ secrets.ANYSCALE_CLI_TOKEN_STAGING }}
6666
ANYSCALE_HOST: https://console.anyscale-staging.com
6767
run: |
68-
anyscale job submit -f ci/anyscale_gpu_ci_h100.yaml --timeout 5400
69-
anyscale job wait --name skyrl-train-gpu-ci-h100 --timeout 5400
68+
COMMIT_SHA="${{ github.event.pull_request.head.sha || github.sha }}"
69+
JOB_NAME="skyrl-train-gpu-ci-h100-${COMMIT_SHA:0:7}-${{ github.run_id }}"
70+
anyscale job submit -f ci/anyscale_gpu_ci_h100.yaml --name "$JOB_NAME" --timeout 5400
71+
anyscale job wait --name "$JOB_NAME" --timeout 5400

‎.github/workflows/gpu_e2e_ci.yaml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ jobs:
4242
WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
4343
run: |
4444
envsubst < ci/anyscale_gpu_e2e_test.yaml > ci/anyscale_gpu_e2e_test_envsubst.yaml
45-
anyscale job submit -f ci/anyscale_gpu_e2e_test_envsubst.yaml --timeout 5000
46-
anyscale job wait --cloud sky-anyscale-aws-us-east-1 --name skyrl-train-gpu-e2e-test --timeout 5000
45+
COMMIT_SHA="${{ github.event.pull_request.head.sha || github.sha }}"
46+
JOB_NAME="skyrl-train-gpu-e2e-test-${COMMIT_SHA:0:7}-${{ github.run_id }}"
47+
anyscale job submit -f ci/anyscale_gpu_e2e_test_envsubst.yaml --name "$JOB_NAME" --timeout 5000
48+
anyscale job wait --cloud sky-anyscale-aws-us-east-1 --name "$JOB_NAME" --timeout 5000
4749
rm -f ci/anyscale_gpu_e2e_test_envsubst.yaml

‎.github/workflows/gpu_e2e_ci_fully_async.yaml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ jobs:
4242
WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
4343
run: |
4444
envsubst < ci/anyscale_gpu_e2e_test_fully_async.yaml > ci/anyscale_gpu_e2e_test_fully_async_envsubst.yaml
45-
anyscale job submit -f ci/anyscale_gpu_e2e_test_fully_async_envsubst.yaml --timeout 5000
46-
anyscale job wait --cloud sky-anyscale-aws-us-east-1 --name skyrl-train-gpu-e2e-test-fully-async --timeout 5000
45+
COMMIT_SHA="${{ github.event.pull_request.head.sha || github.sha }}"
46+
JOB_NAME="skyrl-train-gpu-e2e-test-fully-async-${COMMIT_SHA:0:7}-${{ github.run_id }}"
47+
anyscale job submit -f ci/anyscale_gpu_e2e_test_fully_async_envsubst.yaml --name "$JOB_NAME" --timeout 5000
48+
anyscale job wait --cloud sky-anyscale-aws-us-east-1 --name "$JOB_NAME" --timeout 5000
4749
rm -f ci/anyscale_gpu_e2e_test_fully_async_envsubst.yaml

‎.github/workflows/gpu_e2e_ci_megatron.yaml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ jobs:
4242
WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
4343
run: |
4444
envsubst < ci/anyscale_gpu_e2e_test_megatron.yaml > ci/anyscale_gpu_e2e_test_megatron_envsubst.yaml
45-
anyscale job submit -f ci/anyscale_gpu_e2e_test_megatron_envsubst.yaml --timeout 4500
46-
anyscale job wait --cloud sky-anyscale-aws-us-east-1 --name skyrl-train-gpu-e2e-test-megatron --timeout 4500
45+
COMMIT_SHA="${{ github.event.pull_request.head.sha || github.sha }}"
46+
JOB_NAME="skyrl-train-gpu-e2e-test-megatron-${COMMIT_SHA:0:7}-${{ github.run_id }}"
47+
anyscale job submit -f ci/anyscale_gpu_e2e_test_megatron_envsubst.yaml --name "$JOB_NAME" --timeout 4500
48+
anyscale job wait --cloud sky-anyscale-aws-us-east-1 --name "$JOB_NAME" --timeout 4500
4749
rm -f ci/anyscale_gpu_e2e_test_megatron_envsubst.yaml

‎.github/workflows/gpu_e2e_ci_sft.yaml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ jobs:
4242
WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
4343
run: |
4444
envsubst < ci/anyscale_gpu_e2e_test_sft.yaml > ci/anyscale_gpu_e2e_test_sft_envsubst.yaml
45-
anyscale job submit -f ci/anyscale_gpu_e2e_test_sft_envsubst.yaml --timeout 4500
46-
anyscale job wait --cloud sky-anyscale-aws-us-east-1 --name skyrl-train-gpu-e2e-test-sft --timeout 4500
45+
COMMIT_SHA="${{ github.event.pull_request.head.sha || github.sha }}"
46+
JOB_NAME="skyrl-train-gpu-e2e-test-sft-${COMMIT_SHA:0:7}-${{ github.run_id }}"
47+
anyscale job submit -f ci/anyscale_gpu_e2e_test_sft_envsubst.yaml --name "$JOB_NAME" --timeout 4500
48+
anyscale job wait --cloud sky-anyscale-aws-us-east-1 --name "$JOB_NAME" --timeout 4500
4749
rm -f ci/anyscale_gpu_e2e_test_sft_envsubst.yaml

‎.github/workflows/gpu_e2e_ci_tinker.yaml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ jobs:
4242
WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
4343
run: |
4444
envsubst < ci/anyscale_gpu_e2e_test_tinker.yaml > ci/anyscale_gpu_e2e_test_tinker_envsubst.yaml
45-
anyscale job submit -f ci/anyscale_gpu_e2e_test_tinker_envsubst.yaml --timeout 12000
46-
anyscale job wait --cloud sky-anyscale-aws-us-east-1 --name skyrl-train-gpu-e2e-test-tinker --timeout 12000
45+
COMMIT_SHA="${{ github.event.pull_request.head.sha || github.sha }}"
46+
JOB_NAME="skyrl-train-gpu-e2e-test-tinker-${COMMIT_SHA:0:7}-${{ github.run_id }}"
47+
anyscale job submit -f ci/anyscale_gpu_e2e_test_tinker_envsubst.yaml --name "$JOB_NAME" --timeout 12000
48+
anyscale job wait --cloud sky-anyscale-aws-us-east-1 --name "$JOB_NAME" --timeout 12000
4749
rm -f ci/anyscale_gpu_e2e_test_tinker_envsubst.yaml

‎.github/workflows/gpu_e2e_ci_tinker_fully_async.yaml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ jobs:
4242
WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
4343
run: |
4444
envsubst < ci/anyscale_gpu_e2e_test_tinker_fully_async.yaml > ci/anyscale_gpu_e2e_test_tinker_fully_async_envsubst.yaml
45-
anyscale job submit -f ci/anyscale_gpu_e2e_test_tinker_fully_async_envsubst.yaml --timeout 12000
46-
anyscale job wait --cloud sky-anyscale-aws-us-east-1 --name skyrl-train-gpu-e2e-test-tinker-fully-async --timeout 12000
45+
COMMIT_SHA="${{ github.event.pull_request.head.sha || github.sha }}"
46+
JOB_NAME="skyrl-train-gpu-e2e-test-tinker-fully-async-${COMMIT_SHA:0:7}-${{ github.run_id }}"
47+
anyscale job submit -f ci/anyscale_gpu_e2e_test_tinker_fully_async_envsubst.yaml --name "$JOB_NAME" --timeout 12000
48+
anyscale job wait --cloud sky-anyscale-aws-us-east-1 --name "$JOB_NAME" --timeout 12000
4749
rm -f ci/anyscale_gpu_e2e_test_tinker_fully_async_envsubst.yaml

‎.github/workflows/gpu_skyrl.yaml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,5 +52,7 @@ jobs:
5252
ANYSCALE_CLI_TOKEN: ${{ secrets.ANYSCALE_CLI_TOKEN }}
5353
ANYSCALE_HOST: https://console.anyscale.com
5454
run: |
55-
anyscale job submit -f ci/anyscale_gpu_ci.yaml --timeout 10000
56-
anyscale job wait --cloud sky-anyscale-aws-us-east-1 --name skyrl-tx-gpu-ci --timeout 10000
55+
COMMIT_SHA="${{ github.event.pull_request.head.sha || github.sha }}"
56+
JOB_NAME="skyrl-tx-gpu-ci-${COMMIT_SHA:0:7}-${{ github.run_id }}"
57+
anyscale job submit -f ci/anyscale_gpu_ci.yaml --name "$JOB_NAME" --timeout 10000
58+
anyscale job wait --cloud sky-anyscale-aws-us-east-1 --name "$JOB_NAME" --timeout 10000

‎.github/workflows/gpu_skyrl_train.yaml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,5 +65,7 @@ jobs:
6565
ANYSCALE_CLI_TOKEN: ${{ secrets.ANYSCALE_CLI_TOKEN }}
6666
ANYSCALE_HOST: https://console.anyscale.com
6767
run: |
68-
anyscale job submit -f ci/anyscale_gpu_ci_skyrl_train.yaml --timeout 12000
69-
anyscale job wait --cloud sky-anyscale-aws-us-east-1 --name skyrl-train-gpu-ci --timeout 12000
68+
COMMIT_SHA="${{ github.event.pull_request.head.sha || github.sha }}"
69+
JOB_NAME="skyrl-train-gpu-ci-${COMMIT_SHA:0:7}-${{ github.run_id }}"
70+
anyscale job submit -f ci/anyscale_gpu_ci_skyrl_train.yaml --name "$JOB_NAME" --timeout 12000
71+
anyscale job wait --cloud sky-anyscale-aws-us-east-1 --name "$JOB_NAME" --timeout 12000

0 commit comments

Comments
 (0)