separate prefill and decode gpu tpt

Author: Jatin Gangani

.github/workflows/benchmark-multinode-tmpl.yml

@@ -93,10 +93,14 @@ jobs:
           for result_file in ${RESULT_FILENAME}_*.json; do
             if [ -f "$result_file" ]; then
               echo "Processing $result_file"
-              # Extract GPU count from filename for tp_size calculation
-              gpus=$(echo "$result_file" | sed "s/.*_gpus\([0-9]*\)\.json/\1/")
+              # Extract GPU count, prefill_gpus and decode_gpus from filename for tp_size calculation
+              gpus=$(echo "$result_file" | sed -n "s/.*_gpus_\([0-9]*\).*\.json/\1/p")
+              prefill_gpus=$(echo "$result_file" | sed -n "s/.*_ctx_\([0-9]*\).*\.json/\1/p")
+              decode_gpus=$(echo "$result_file" | sed -n "s/.*_gen_\([0-9]*\).*\.json/\1/p")
+              
               if [ -n "$gpus" ]; then
-                TP=$gpus RESULT_FILENAME=${result_file%.json} EP_SIZE=1 DP_ATTENTION=false python3 utils/process_result.py
+                echo "Extracted: gpus=$gpus, prefill_gpus=$prefill_gpus, decode_gpus=$decode_gpus"
+                TP=$gpus RESULT_FILENAME=${result_file%.json} EP_SIZE=1 DP_ATTENTION=false PREFILL_GPUS="$prefill_gpus" DECODE_GPUS="$decode_gpus" python3 utils/process_result.py
               fi
             fi
           done

runners/launch_gb200-nv.sh

@@ -49,7 +49,7 @@ if [[ $FRAMEWORK == "dynamo-trtllm" ]]; then
     rm -rf "$DYNAMO_PATH"
     git clone https://github.com/ai-dynamo/dynamo.git "$DYNAMO_PATH"
     cd "$DYNAMO_PATH"
-    git checkout release/0.5.1-rc0.pre1
+    git checkout tanmayv-sa
     git submodule update --init --recursive
 
     # Navigate to performance sweeps directory
@@ -163,7 +163,7 @@ else # if statement at the top - search for "FRAMEWORK_DIFF_IF_STATEMENT #2"
     # Always clone and setup Dynamo
     echo "Cloning Dynamo repository..."
     rm -rf "$DYNAMO_PATH"
-    git clone --branch update-wait-for-model https://github.com/Elnifio/dynamo.git $DYNAMO_PATH
+    git clone --branch update-result-file-name https://github.com/Elnifio/dynamo.git $DYNAMO_PATH
     cd "$DYNAMO_PATH"
 
     # Navigate to corresponding directory
@@ -272,7 +272,7 @@ else # search for "FRAMEWORK_DIFF_IF_STATEMENT #3" for this if-statement
 
     # Result JSON are contained within the result directory
     for result_file in $(find $LOGS_DIR -type f); do
-        # result_file should directly be isl_ISL_osl_OSL_concurrency_CONC_req_rate_R_gpusN.json
+        # result_file should directly be isl_ISL_osl_OSL_concurrency_CONC_req_rate_R_gpus_N_ctx_M_gen_N.json
         file_name=$(basename $result_file)
         if [ -f $result_file ]; then
             # Copy the result file to workspace with a unique name

utils/process_result.py

@@ -7,6 +7,12 @@
 hw = os.environ.get('RUNNER_TYPE')
 tp_size = int(os.environ.get('TP'))
 ep_size = int(os.environ.get('EP_SIZE'))
+prefill_gpus_str = os.environ.get('PREFILL_GPUS', '')
+decode_gpus_str = os.environ.get('DECODE_GPUS', '')
+
+# If empty string (aggregated runs), assign to tp_size (total gpus), otherwise convert to int
+prefill_gpus = tp_size if not prefill_gpus_str else int(prefill_gpus_str)
+decode_gpus = tp_size if not decode_gpus_str else int(decode_gpus_str)
 dp_attention = os.environ.get('DP_ATTENTION')
 result_filename = os.environ.get('RESULT_FILENAME')
 framework = os.environ.get('FRAMEWORK')
@@ -26,7 +32,8 @@
     'framework': framework,
     'precision': precision,
     'tput_per_gpu': float(bmk_result['total_token_throughput']) / tp_size,
-    'output_tput_per_gpu': float(bmk_result['output_throughput']) / tp_size
+    'output_tput_per_gpu': float(bmk_result['output_throughput']) / decode_gpus,
+    'input_tput_per_gpu': (float(bmk_result['total_token_throughput']) - float(bmk_result['output_throughput']) )/ prefill_gpus
 }
 
 if mtp_mode:  # MTP