More chagnes to mlperf setup to support llama2 70b and mixtral 8x7b (#185)

Author: vipannalla

benchmarks/mlperf/main.py

@@ -23,7 +23,7 @@
 
 import mlperf_loadgen as lg
 
-_MLPERF_ID = "mixtral-8x7b"
+_MLPERF_ID = "llama2-70b"
 
 sys.path.insert(0, os.getcwd())
 
@@ -135,7 +135,12 @@ def get_args():
           'eg. {"tok_input_len": "tok_input_length"}'
       ),
   )
-
+  parser.add_argument(
+      "--mlperf-conf-id",
+      type=str,
+      default=_MLPERF_ID,
+      help="When given overrides the default user.conf path",
+ )
   args = parser.parse_args()
   return args
 
@@ -156,8 +161,8 @@ def main():
   else:
     user_conf = args.user_conf
 
-  settings.FromConfig(args.mlperf_conf, _MLPERF_ID, args.scenario)
-  settings.FromConfig(user_conf, _MLPERF_ID, args.scenario)
+  settings.FromConfig(args.mlperf_conf, args.mlperf_conf_id, args.scenario)
+  settings.FromConfig(user_conf, args.mlperf_conf_id, args.scenario)
   log.info("Mlperf config: %s", args.mlperf_conf)
   log.info("User config: %s", user_conf)
 

benchmarks/mlperf/scripts/generate_server_accuracy_run.sh

@@ -14,6 +14,9 @@
 
 source run_utils.sh
 
+export TOKENIZER_PATH=meta-llama/Llama-2-70b-chat-hf
+export DATASET_PREFIX=""
+export MODEL_ID="llama2-70b"
 DATASET_NAME=$(get_dataset_name ${DATASET_TYPE})
 export DATASET_PATH=${DATA_DISK_DIR}/${DATASET_NAME}.pkl
 export API_URL=${API_URL}
@@ -33,6 +36,7 @@ echo "OUTPUT_ACCURACY_JSON_PATH: ${OUTPUT_ACCURACY_JSON_PATH}"
 echo "USER_CONFIG: ${USER_CONFIG}"
 
 mkdir -p ${OUTPUT_LOG_DIR} && cp ../${USER_CONFIG} ${OUTPUT_LOG_DIR}
+MIXTRAL_COLS_RENAME="{\"tok_input_len\": \"tok_input_length\", \"tok_ref_output_len\": \"tok_output_length\"}"
 
 # Accuracy Run
 cd ../ && python3 main.py \
@@ -53,6 +57,8 @@ cd ../ && python3 main.py \
 	--tokenizer-path ${TOKENIZER_PATH} \
 	--log-interval ${LOG_INTERVAL} \
 	--num-client-threads ${NUM_CLIENT_THREADS} \
+	--mlperf-conf-id "${MODEL_ID}" \
+        --rename-dataset-cols "${MIXTRAL_COLS_RENAME}" \
 	--output-log-dir ${OUTPUT_LOG_DIR} 2>&1 | tee ${OUTPUT_LOG_DIR}/server_accuracy_log.log
 
 # Eval Run

benchmarks/mlperf/scripts/generate_server_audit_run.sh

@@ -13,7 +13,9 @@
 # limitations under the License.
 
 source run_utils.sh
-
+export TOKENIZER_PATH=meta-llama/Llama-2-70b-chat-hf
+export DATASET_PREFIX=""
+export MODEL_ID="llama2-70b"
 DATASET_NAME=$(get_dataset_name ${DATASET_TYPE})
 export DATASET_PATH=${DATA_DISK_DIR}/${DATASET_NAME}.pkl
 export API_URL=${API_URL}
@@ -50,4 +52,5 @@ cd ../ && python3 main.py \
 	--tokenizer-path ${TOKENIZER_PATH} \
 	--log-interval ${LOG_INTERVAL} \
 	--num-client-threads ${NUM_CLIENT_THREADS} \
+	--mlperf-conf-id "${MODEL_ID}" \
 	--output-log-dir ${OUTPUT_LOG_DIR} 2>&1 | tee ${OUTPUT_LOG_DIR}/server_audit_log.log

benchmarks/mlperf/scripts/generate_server_performance_run.sh

@@ -14,6 +14,9 @@
 
 source run_utils.sh
 
+export TOKENIZER_PATH=meta-llama/Llama-2-70b-chat-hf
+export DATASET_PREFIX=""
+export MODEL_ID="llama2-70b"
 DATASET_NAME=$(get_dataset_name ${DATASET_TYPE})
 export DATASET_PATH=${DATA_DISK_DIR}/${DATASET_NAME}.pkl
 export API_URL=${API_URL}
@@ -52,4 +55,5 @@ cd ../ && python3 main.py \
 	--log-interval ${LOG_INTERVAL} \
 	--num-client-threads ${NUM_CLIENT_THREADS} \
 	--rename-dataset-cols "${MIXTRAL_COLS_RENAME}" \
+	--mlperf-conf-id "${MODEL_ID}" \
 	--output-log-dir ${OUTPUT_LOG_DIR} 2>&1 | tee ${OUTPUT_LOG_DIR}/server_performance_log.log

benchmarks/mlperf/scripts/run_utils.sh

@@ -15,7 +15,7 @@
 
 # Tokenizer
 # export TOKENIZER_PATH=meta-llama/Llama-2-70b-chat-hf
-export DATASET_PREFIX=mixtral
+export DATASET_PREFIX="mixtral-"
 export TOKENIZER_PATH=mistralai/Mixtral-8x7B-Instruct-v0.1
 export NUM_CLIENT_THREADS=${NUM_CLIENT_THREADS:=600}
 
@@ -25,7 +25,7 @@ export LOADGEN_RUN_TIMESTAMP=$(TZ=America/Los_Angeles date +%Y%m%d%H%M%S%Z)
 get_dataset_name() {
   dataset_type=$1
 	if [ ${dataset_type} = "full" ]
-		then echo "${DATASET_PREFIX}-processed-data"
+		then echo "${DATASET_PREFIX}processed-data"
 	elif [ ${dataset_type} = "calibration" ]
 		then echo "${DATASET_PREFIX}-processed-calibration-data"
 	fi

benchmarks/mlperf/user.conf

@@ -22,7 +22,7 @@ mixtral-8x7b.Server.min_query_count = 15000
 
 # These fields should be defined and overridden by user.conf.
 *.Offline.target_qps = 5.0
-llama2-70b.Server.target_qps = 1.0
+llama2-70b.Server.target_qps = 14.5
 mixtral-8x7b.Server.target_qps = 11.0