[None][test] Promote disagg perf sanity tests to pre-merge for functional verification (NVIDIA#16351)

Signed-off-by: chenfeiz0326 <chenfeiz@nvidia.com>

Author: chenfeiz0326

jenkins/L0_Test.groovy

@@ -4883,13 +4883,23 @@ def launchTestJobs(pipeline, testFilter)
         "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-5": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 5, 6, 8, 1, true],
         "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-6": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 6, 6, 8, 1, true],
     ]
+    // B200 PerfSanity pre-merge disaggregated (functional-only: perf regressions do not fail CI)
+    // 2 Nodes
+    x86SlurmTestConfigs += buildStageConfigs(
+        "DGX_B200-16_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-FUNCTIONAL-ONLY-CTX1-NODE1-GPU4-GEN1-NODE1-GPU8",
+        "auto:dgx-b200-flex",
+        "l0_b200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu8",
+        1,
+        16,
+        2
+    )
     // B200 PerfSanity post-merge disaggregated
     // 2 Nodes
     x86SlurmTestConfigs += buildStageConfigs(
         "DGX_B200-16_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN1-NODE1-GPU8-Post-Merge",
         "auto:dgx-b200-flex",
         "l0_b200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu8",
-        6,
+        5,
         16,
         2
     )
@@ -4986,6 +4996,16 @@ def launchTestJobs(pipeline, testFilter)
         8,
         2
     )
+    // PerfSanity pre-merge disaggregated (functional-only: perf regressions do not fail CI)
+    // 2 Nodes
+    multiNodesSBSAConfigs += buildStageConfigs(
+        "GB200-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-FUNCTIONAL-ONLY-CTX1-NODE1-GPU1-GEN1-NODE1-GPU4",
+        "auto:gb200-flex",
+        "l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4",
+        1,
+        8,
+        2
+    )
     // PerfSanity post-merge disaggregated
     // 2 Nodes
     multiNodesSBSAConfigs += buildStageConfigs(
@@ -5000,7 +5020,7 @@ def launchTestJobs(pipeline, testFilter)
         "GB200-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU1-GEN1-NODE1-GPU4-Post-Merge",
         "auto:gb200-flex",
         "l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4",
-        7,
+        6,
         8,
         2
     )
@@ -5102,12 +5122,21 @@ def launchTestJobs(pipeline, testFilter)
         8,
         2
     )
+    // 3 Nodes (pre-merge, functional-only)
+    multiNodesSBSAConfigs += buildStageConfigs(
+        "GB300-12_GPUs-3_Nodes-PyTorch-Disagg-PerfSanity-FUNCTIONAL-ONLY-CTX1-NODE1-GPU4-GEN1-NODE2-GPU8",
+        "auto:gb300-flex",
+        "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8",
+        1,
+        12,
+        3
+    )
     // 3 Nodes
     multiNodesSBSAConfigs += buildStageConfigs(
         "GB300-12_GPUs-3_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN1-NODE2-GPU8-Post-Merge",
         "auto:gb300-flex",
         "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8",
-        5,
+        4,
         12,
         3
     )
@@ -5139,12 +5168,21 @@ def launchTestJobs(pipeline, testFilter)
         8,
         2
     )
+    // 3 Nodes (pre-merge, functional-only)
+    multiNodesSBSAConfigs += buildStageConfigs(
+        "GB300-12_GPUs-3_Nodes-PyTorch-Disagg-PerfSanity-FUNCTIONAL-ONLY-CTX1-NODE1-GPU2-GEN1-NODE2-GPU8",
+        "auto:gb300-flex",
+        "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8",
+        1,
+        12,
+        3
+    )
     // 3 Nodes
     multiNodesSBSAConfigs += buildStageConfigs(
         "GB300-12_GPUs-3_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU2-GEN1-NODE2-GPU8-Post-Merge",
         "auto:gb300-flex",
         "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8",
-        5,
+        4,
         12,
         3
     )

tests/integration/defs/perf/README_test_perf_sanity.md

@@ -229,6 +229,16 @@ Tests are defined in `jenkins/L0_Test.groovy` under the `launchTestJobs` functio
 
 **Important**: Pre-merge and post-merge tests must be in separate stages.
 
+### Pre-merge vs Post-merge Perf-Regression Gating
+
+By default, `test_perf_sanity.py` fails CI on perf regression for pre-merge stages and only warns for post-merge stages. This is auto-detected from the Jenkins job URL (`PostMerge` substring), not the stage name.
+
+**`FUNCTIONAL-ONLY` stage-name flag**: A pre-merge stage whose name contains `FUNCTIONAL-ONLY` (e.g. `GB200-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-FUNCTIONAL-ONLY-CTX1-NODE1-GPU4-GEN1-NODE1-GPU4`) still runs the full perf harness — benchmarks execute, metrics are uploaded to OpenSearch, dashboards update — but perf regressions **do not fail CI**. Only functional failures (build errors, crashes, empty output) fail the stage.
+
+Use this for pre-merge stages whose goal is to catch functional regressions on paths that only had post-merge coverage before. It preserves the data-continuity benefit of running in pre-merge (baselines still update from PR data points) without the flakiness cost of gating on the noisier disagg perf numbers.
+
+Detection is by substring match on `os.environ["stageName"]` inside `test_perf_sanity.py`; no changes to `perf_regression_utils.py`.
+
 ### GPU Hours Calculation
 
 - Each CI stage runtime is approximately **1 hour**

tests/integration/defs/perf/test_perf_sanity.py

@@ -2273,11 +2273,20 @@ def add_dict_prefix(config_dict: dict, prefix_name: str) -> dict:
         else:
             return
 
+        stage_name = os.environ.get("stageName", "")
         extra_fields = {
-            "s_stage_name": os.environ.get("stageName", ""),
+            "s_stage_name": stage_name,
             "s_test_list": self._test_param_labels,
         }
 
+        # Stages tagged "FUNCTIONAL-ONLY" run the full perf harness (numbers are
+        # still uploaded to OpenSearch and dashboards) but do not fail CI on perf
+        # regression -- same behavior as post-merge. Used for pre-merge disagg
+        # coverage where the goal is functional-failure detection, not gating on
+        # perf. Explicit False (not None) so the auto-detect in
+        # process_and_upload_test_results does not flip it back on for pre-merge.
+        fail_on_regression = False if "FUNCTIONAL-ONLY" in stage_name else None
+
         # gen_only tests are gated solely on per-iter prev_device_step_time, not
         # token throughput (token-based numbers are dominated by KV cache transfer
         # time in gen_only mode and are not a useful regression signal there).
@@ -2304,6 +2313,7 @@ def add_dict_prefix(config_dict: dict, prefix_name: str) -> dict:
             regression_metrics=regression_metrics,
             extra_fields=extra_fields,
             upload_to_db=self.upload_to_db,
+            fail_on_regression=fail_on_regression,
         )
 
 

tests/integration/test_lists/test-db/l0_b200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu8.yml

@@ -1,5 +1,21 @@
 version: 0.0.1
 l0_b200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu8:
+- condition:
+    ranges:
+      # 1 ctx worker with each 1 node and 4 GPUs
+      # 1 gen worker with each 1 node and 8 GPUs
+      system_gpu_count:
+        gte: 16
+        lte: 16
+    wildcards:
+      gpu:
+      - '*b200*'
+    terms:
+      stage: pre_merge
+      backend: pytorch
+  tests:
+  - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (120)
+
 - condition:
     ranges:
       # 1 ctx worker with each 1 node and 4 GPUs
@@ -17,7 +33,6 @@ l0_b200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu8:
   - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120)
   - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_1k1k_con2048_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (120)
   - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_1k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120)
-  - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (120)
   - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120)
   - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_8k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (120)
   # - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-b200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120)

tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml

@@ -1,5 +1,21 @@
 version: 0.0.1
 l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4:
+- condition:
+    ranges:
+      # 1 ctx worker with each 1 node and 1 GPUs
+      # 1 gen worker with each 1 node and 4 GPUs
+      system_gpu_count:
+        gte: 8
+        lte: 8
+    wildcards:
+      gpu:
+      - '*gb200*'
+    terms:
+      stage: pre_merge
+      backend: pytorch
+  tests:
+  - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90)
+
 - condition:
     ranges:
       # 1 ctx worker with each 1 node and 1 GPUs
@@ -20,4 +36,3 @@ l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4:
   - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90)
   - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_qwen3-235b-fp4_8k1k_con64_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90)
   - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90)
-  - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] TIMEOUT (90)

tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml

@@ -1,5 +1,22 @@
 version: 0.0.1
 l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8:
+- condition:
+    ranges:
+      # 1 ctx worker with each 1 node and 2 GPUs
+      # 1 gen worker with each 2 node and 8 GPUs
+      system_gpu_count:
+        gte: 12
+        lte: 12
+    wildcards:
+      gpu:
+      - '*gb300*'
+    terms:
+      stage: pre_merge
+      backend: pytorch
+  tests:
+  # glm-5-fp4
+  - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (90)
+
 - condition:
     ranges:
       # 1 ctx worker with each 1 node and 2 GPUs
@@ -19,4 +36,3 @@ l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8:
   - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90)
   - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (90)
   - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (90)
-  - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (90)

tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml

@@ -1,5 +1,22 @@
 version: 0.0.1
 l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8:
+- condition:
+    ranges:
+      # 1 ctx worker with each 1 node and 4 GPUs
+      # 1 gen worker with each 2 node and 8 GPUs
+      system_gpu_count:
+        gte: 12
+        lte: 12
+    wildcards:
+      gpu:
+      - '*gb300*'
+    terms:
+      stage: pre_merge
+      backend: pytorch
+  tests:
+  # kimi-k25-thinking-fp4 8k1k
+  - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90)
+
 - condition:
     ranges:
       # 1 ctx worker with each 1 node and 4 GPUs
@@ -18,7 +35,5 @@ l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8:
   - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (180)
   # kimi-k25-thinking-fp4 1k1k
   - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] TIMEOUT (90)
-  # kimi-k25-thinking-fp4 8k1k
-  - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (90)
   - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (180)
   - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] TIMEOUT (90)