Jwilber/update conv schedule (#1559)

Update schedule to run less frequent, and update esm2 jobs:

- `.github/workflows/scdl-performance-tests.yml` — scdl now runs weekly
on Fridays 2pm PST (0 22 * * 5) instead of daily.

- `.github/workflows/convergence-tests.yml` — single cron 0 8 * * 0 (Sun
00:00 PST = Sat night midnight), a new gate-biweekly job that skips odd
ISO weeks, and the matrix runs all three configs (esm2_650m, esm2_15b,
codonfm_ptl_te) on every scheduled run.

- `ci/lepton/.../esm2_native_te_15b.yaml` — num_train_steps: 20_000 →
500.

- `ci/lepton/.../esm2_native_te_650m.yaml` — added top-level
fp8_recipe/fp8_format, two new FP8 products (bshd+fp8, thd+fp8)
mirroring the 15b config, and extended run_script to pass both fp8
params.

---------

Signed-off-by: jwilber <jwilber@nvidia.com>

Author: jwilber

.github/workflows/convergence-tests.yml

@@ -32,16 +32,32 @@ on:
         required: false
         type: string
   schedule:
-    - cron: "0 8 * * 1,3" # Mon/Wed at 1am PST (esm2)
-    - cron: "0 8 * * 2,4" # Tue/Thu at 1am PST (codonfm)
+    - cron: "0 8 * * 0" # Sun 00:00 PST (Sat night midnight), bi-weekly gated in job
 
 jobs:
+  # GitHub cron has no native bi-weekly, so gate on even ISO week number.
+  gate-biweekly:
+    runs-on: ubuntu-latest
+    outputs:
+      proceed: ${{ steps.check.outputs.proceed }}
+    steps:
+      - id: check
+        run: |
+          if [ "${{ github.event_name }}" != "schedule" ]; then
+            echo "proceed=true" >> "$GITHUB_OUTPUT"
+          elif [ $((10#$(date -u +%V) % 2)) -eq 0 ]; then
+            echo "proceed=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "proceed=false" >> "$GITHUB_OUTPUT"
+          fi
+
   submit-lepton-jobs:
+    needs: gate-biweekly
+    if: needs.gate-biweekly.outputs.proceed == 'true'
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        # Mon/Wed runs esm2, Tue/Thu runs codonfm
-        model_config: ${{ github.event_name == 'schedule' && github.event.schedule == '0 8 * * 2,4' && fromJSON('["codonfm_ptl_te"]') || github.event_name == 'schedule' && github.event.schedule == '0 8 * * 1,3' && fromJSON('["esm2_native_te_650m", "esm2_native_te_15b"]') || fromJSON(format('["{0}"]', github.event.inputs.model_config)) }}
+        model_config: ${{ github.event_name == 'schedule' && fromJSON('["esm2_native_te_650m", "esm2_native_te_15b", "codonfm_ptl_te"]') || fromJSON(format('["{0}"]', github.event.inputs.model_config)) }}
       fail-fast: false
     steps:
       - name: Checkout

.github/workflows/scdl-performance-tests.yml

@@ -13,7 +13,7 @@ on:
         required: false
         type: string
   schedule:
-    - cron: "0 9 * * *" # everyday at 1am PST
+    - cron: "0 22 * * 5" # Fridays at 2pm PST (weekly)
 
 jobs:
   submit-lepton-jobs:

ci/lepton/model_convergence/configs/recipes/esm2_native_te_15b.yaml

@@ -61,7 +61,7 @@ use_torch_compile: false
 # these should match the keys in the recipe's config file
 model_tag: nvidia/esm2_t48_15B_UR50D
 # task_cmd: train_fsdp2 # mfsdp
-num_train_steps: 20_000
+num_train_steps: 500
 # dataset commands
 micro_batch_size: 8
 load_dataset_kwargs_path: nvidia/esm2_uniref_pretraining_data

ci/lepton/model_convergence/configs/recipes/esm2_native_te_650m.yaml

@@ -32,6 +32,8 @@ framework: native # native, accelerate
 precision: bf16 # likely bf16 or fp8
 te_enabled: true
 fp8_enabled: false
+fp8_recipe: ""
+fp8_format: ""
 # thd_enabled: false
 
 # Catchall for additional features/configs
@@ -90,6 +92,28 @@ products:
     micro_batch_size: 48
     wandb_name: "esm2_native_650m__fsdp2__thd__${now:%Y%m%d-%H%M%S}__${gitsha:}"
     job_name: "esm2-native-650m-fsdp2-thd"
+  # TE bshd perf, FSDP2, FP8
+  - config: L1_650M
+    task_cmd: train_fsdp2
+    parallelism_strategy: fsdp2
+    thd_enabled: false
+    fp8_enabled: true
+    fp8_recipe: transformer_engine.common.recipe.Float8BlockScaling
+    fp8_format: E4M3
+    micro_batch_size: 48
+    wandb_name: "esm2_native_650m__fsdp2__fp8__${now:%Y%m%d-%H%M%S}__${gitsha:}"
+    job_name: "esm2-native-650m-fsdp2-fp8"
+  # TE thd perf, FSDP2, FP8
+  - config: L1_650M
+    task_cmd: train_fsdp2
+    parallelism_strategy: fsdp2
+    thd_enabled: true
+    fp8_enabled: true
+    fp8_recipe: transformer_engine.common.recipe.Float8BlockScaling
+    fp8_format: E4M3
+    micro_batch_size: 48
+    wandb_name: "esm2_native_650m__fsdp2__thd__fp8__${now:%Y%m%d-%H%M%S}__${gitsha:}"
+    job_name: "esm2-native-650m-fsdp2-thd-fp8"
   # OSS Convergence Baseline
   # - config: L1_650M
   #   model_tag: facebook/esm2_t33_650M_UR50D
@@ -137,4 +161,6 @@ run_script: |
     checkpoint.resume_from_checkpoint=${resume_from_checkpoint} \
     +checkpoint.save_checkpoints=${save_checkpoints} \
     +checkpoint.use_distributed_checkpoint_fsdp2=${use_distributed_checkpoint_fsdp2} \
-    fp8_config.enabled=${fp8_enabled}
+    fp8_config.enabled=${fp8_enabled} \
+    fp8_config.fp8_recipe=${fp8_recipe} \
+    fp8_config.fp8_format=${fp8_format}