Merge branch 'main' into dev/ba/add-checkpointing-documentation

Author: jwilber

.devcontainer/devcontainer.json .devcontainer/framework/devcontainer.json.devcontainer/devcontainer.json renamed to .devcontainer/framework/devcontainer.json

@@ -1,5 +1,5 @@
 {
-	"name": "BioNeMo2 Development Container",
+	"name": "BioNeMo2 Framework Dev Container",
 	"runArgs": [
 		"--gpus=all",
 		"--shm-size=4g"
@@ -23,8 +23,8 @@
 		"TMPDIR": "/tmp",
 		"NUMBA_CACHE_DIR": "/tmp/"
 	},
-	"postCreateCommand": "./.devcontainer/postCreateCommand.sh",
-	"initializeCommand": "./.devcontainer/initializeCommand.sh",
+	"postCreateCommand": "./.devcontainer/framework/postCreateCommand.sh",
+	"initializeCommand": "./.devcontainer/framework/initializeCommand.sh",
 	"remoteUser": "ubuntu",
 	"customizations": {
 		"vscode": {
@@ -55,8 +55,7 @@
 				"python.defaultInterpreterPath": "/usr/bin/python",
 				"python.testing.pytestEnabled": true,
 				"python.testing.pytestArgs": [
-					"sub-packages/",
-					"scripts/"
+					"sub-packages/"
 				],
 				"python.analysis.typeCheckingMode": "standard"
 			}

.devcontainer/initializeCommand.sh …container/framework/initializeCommand.sh.devcontainer/initializeCommand.sh renamed to .devcontainer/framework/initializeCommand.sh .devcontainer/initializeCommand.sh renamed to .devcontainer/framework/initializeCommand.sh

@@ -0,0 +1,4 @@
+FROM nvcr.io/nvidia/pytorch:25.06-py3
+RUN --mount=type=cache,target=/root/.cache/pip \
+    --mount=type=bind,source=requirements.txt,target=/workspace/requirements.txt \
+    PIP_CONSTRAINT= pip install -r /workspace/requirements.txt

.devcontainer/postCreateCommand.sh …container/framework/postCreateCommand.sh.devcontainer/postCreateCommand.sh renamed to .devcontainer/framework/postCreateCommand.sh .devcontainer/postCreateCommand.sh renamed to .devcontainer/framework/postCreateCommand.sh

@@ -0,0 +1,3 @@
+# Dev Container Setup
+
+General-purpose dev container for local recipe development.

.devcontainer/recipes/Dockerfile

@@ -0,0 +1,32 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile
+{
+    "name": "BioNeMo Recipes Dev Container",
+    "build": {
+        "dockerfile": "Dockerfile"
+    },
+    "mounts": [
+        "source=${localEnv:HOME}/.cache,target=/home/ubuntu/.cache,type=bind,consistency=cached",
+        "source=${localEnv:HOME}/.netrc,target=/home/ubuntu/.netrc,type=bind,consistency=cached",
+        "source=${localEnv:HOME}/.bash_history_devcontainer,target=/home/ubuntu/.bash_history,type=bind,consistency=cached"
+    ],
+    "postCreateCommand": ".devcontainer/recipes/postCreateCommand.sh",
+    "remoteUser": "ubuntu",
+    "runArgs": [
+        "--gpus=all",
+        "--shm-size=4g"
+    ],
+    "customizations": {
+        "vscode": {
+            "extensions": [
+                "ms-python.python",
+                "charliermarsh.ruff",
+                "redhat.vscode-yaml",
+                "ms-toolsai.jupyter",
+                "eamodio.gitlens",
+                "tamasfe.even-better-toml",
+                "streetsidesoftware.code-spell-checker"
+            ]
+        }
+    }
+}

.devcontainer/recipes/README.md

@@ -0,0 +1,12 @@
+accelerate
+datasets
+deepspeed
+hydra-core
+torch
+megatron-fsdp==0.1.0rc0
+torchmetrics
+tqdm
+transformer_engine
+transformers @ git+https://github.com/huggingface/transformers.git
+typer
+wandb

.devcontainer/recipes/devcontainer.json

@@ -20,7 +20,7 @@ Configure CI behavior by applying the relevant labels:
 - [INCLUDE_NOTEBOOKS_TESTS](https://github.com/NVIDIA/bionemo-framework/blob/main/docs/docs/user-guide/contributing/contributing.md#include_notebooks_tests) - Execute notebook validation tests in pytest
 - [INCLUDE_SLOW_TESTS](https://github.com/NVIDIA/bionemo-framework/blob/main/docs/docs/user-guide/contributing/contributing.md#include_slow_tests) - Execute tests labelled as slow in pytest for extensive testing
 
-> \[!NOTE\]
+> [!NOTE]
 > By default, the notebooks validation tests are skipped unless explicitly enabled.
 
 #### Authorizing CI Runs

.devcontainer/recipes/postCreateCommand.sh

@@ -1,4 +1,4 @@
-name: "BioNemo Image Build and Unit Tests"
+name: "BioNeMo Framework CI"
 
 on:
   push:
@@ -20,18 +20,54 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  pre-commit:
+  changed-files:
     runs-on: ubuntu-latest
+    outputs:
+      any_changed: ${{ steps.changed-files.outputs.any_changed }}
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
           submodules: "recursive"
+
+      - uses: step-security/changed-files@v46
+        id: changed-files
+        with:
+          base_sha: main
+          files: |
+            **
+            !models/**
+            !recipes/**
+            !**.md
+            !.github/**
+            !.gitignore
+            !.devcontainer/**
+            .github/workflows/unit-tests-framework.yml
+
+      - name: Show output
+        run: |
+          echo '${{ toJSON(steps.changed-files.outputs) }}'
+        shell:
+          bash
+
+
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.12"
-          cache: "pip"
-      - run: pip install -r requirements-dev.txt
+          python-version: "3.13"
+      - name: Setup UV
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+      - run: |
+          uv tool install pre-commit --with pre-commit-uv --force-reinstall
+          uv tool install tach>=0.9.0
+          uv tool update-shell
       - run: ./ci/scripts/static_checks.sh
 
   # With copy-pr-bot, we need to get the PR labels from the PR API rather than from the event metadata.
@@ -65,8 +101,10 @@ jobs:
     needs:
       - pre-commit
       - get-pr-labels
+      - changed-files
     runs-on: linux-amd64-cpu16
-    if: ${{ !contains(fromJSON(needs.get-pr-labels.outputs.labels || '[]'), 'SKIP_CI') }}
+    # We want this to run on schedule events even if no files have changed
+    if: ${{ github.event_name == 'schedule' || (!contains(fromJSON(needs.get-pr-labels.outputs.labels || '[]'), 'SKIP_CI') && needs.changed-files.outputs.any_changed == 'true') }}
     steps:
       - name: Login to Docker Hub
         uses: docker/login-action@v3
@@ -77,7 +115,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
-          submodules: "recursive"
+          submodules: true
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3