Skip to content

Commit d274607

Browse files
yinggehyinggeh
authored
ci: Update TRT-LLM tests with new config "prompt_embedding_table_data_type" (#8421)
1 parent e0c5dc9 commit d274607

3 files changed

Lines changed: 162 additions & 260 deletions

File tree

‎qa/L0_orca/test.sh‎

Lines changed: 1 addition & 134 deletions
Original file line numberDiff line numberDiff line change
@@ -46,140 +46,7 @@ SERVER_LOG="${NAME}_server.log"
4646
SERVER_TIMEOUT=${SERVER_TIMEOUT:=120}
4747
CLIENT_PY=${BASE_DIR}/orca_http_test.py
4848
CLIENT_LOG="${NAME}_orca_http_test.log"
49-
source ../common/util.sh
50-
51-
function clone_tensorrt_llm_backend_repo {
52-
rm -rf $TENSORRTLLM_BACKEND_DIR && mkdir $TENSORRTLLM_BACKEND_DIR
53-
apt-get update && apt-get install git-lfs -y --no-install-recommends
54-
git clone --single-branch --depth=1 -b ${TENSORRTLLM_BACKEND_REPO_TAG} ${TRITON_REPO_ORG}/tensorrtllm_backend.git $TENSORRTLLM_BACKEND_DIR
55-
cd $TENSORRTLLM_BACKEND_DIR && git lfs install && git submodule update --init --recursive
56-
}
57-
58-
function build_gpt2_base_model {
59-
# Download weights from HuggingFace Transformers
60-
cd ${GPT_DIR} && rm -rf gpt2 && git clone https://huggingface.co/gpt2-medium gpt2 && cd gpt2
61-
rm pytorch_model.bin model.safetensors
62-
if ! wget -q https://huggingface.co/gpt2-medium/resolve/main/pytorch_model.bin; then
63-
echo "Downloading pytorch_model.bin failed."
64-
exit 1
65-
fi
66-
cd ${GPT_DIR}
67-
68-
# Convert weights from HF Tranformers to FT format
69-
python3 convert_checkpoint.py --model_dir gpt2 --dtype float16 --tp_size ${NUM_GPUS} --output_dir "./c-model/gpt2/${NUM_GPUS}-gpu/"
70-
cd ${BASE_DIR}
71-
}
72-
73-
function build_gpt2_tensorrt_engine {
74-
# Build TensorRT engines
75-
cd ${GPT_DIR}
76-
trtllm-build --checkpoint_dir "./c-model/gpt2/${NUM_GPUS}-gpu/" \
77-
--gpt_attention_plugin float16 \
78-
--remove_input_padding enable \
79-
--paged_kv_cache enable \
80-
--gemm_plugin float16 \
81-
--workers "${NUM_GPUS}" \
82-
--output_dir "${ENGINES_DIR}"
83-
84-
cd ${BASE_DIR}
85-
}
86-
87-
function replace_config_tags {
88-
tag_to_replace="${1}"
89-
new_value="${2}"
90-
config_file_path="${3}"
91-
sed -i "s|${tag_to_replace}|${new_value}|g" ${config_file_path}
92-
}
93-
94-
function prepare_model_repository {
95-
rm -rf ${MODEL_REPOSITORY} && mkdir ${MODEL_REPOSITORY}
96-
cp -r ${TENSORRTLLM_BACKEND_DIR}/tensorrt_llm/triton_backend/all_models/inflight_batcher_llm/* ${MODEL_REPOSITORY}
97-
rm -rf ${MODEL_REPOSITORY}/tensorrt_llm_bls
98-
mv "${MODEL_REPOSITORY}/ensemble" "${MODEL_REPOSITORY}/${MODEL_NAME}"
99-
100-
replace_config_tags "model_version: -1" "model_version: 1" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt"
101-
replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt"
102-
replace_config_tags 'name: "ensemble"' "name: \"$MODEL_NAME\"" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt"
103-
replace_config_tags '${logits_datatype}' "TYPE_FP32" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt"
104-
105-
replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt"
106-
replace_config_tags '${preprocessing_instance_count}' '1' "${MODEL_REPOSITORY}/preprocessing/config.pbtxt"
107-
replace_config_tags '${tokenizer_dir}' "${TOKENIZER_DIR}/" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt"
108-
replace_config_tags '${logits_datatype}' "TYPE_FP32" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt"
109-
replace_config_tags '${max_queue_delay_microseconds}' "1000000" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt"
110-
replace_config_tags '${max_queue_size}' "0" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt"
111-
112-
replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/postprocessing/config.pbtxt"
113-
replace_config_tags '${postprocessing_instance_count}' '1' "${MODEL_REPOSITORY}/postprocessing/config.pbtxt"
114-
replace_config_tags '${tokenizer_dir}' "${TOKENIZER_DIR}/" "${MODEL_REPOSITORY}/postprocessing/config.pbtxt"
115-
replace_config_tags '${logits_datatype}' "TYPE_FP32" "${MODEL_REPOSITORY}/postprocessing/config.pbtxt"
116-
117-
replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt"
118-
replace_config_tags '${decoupled_mode}' 'true' "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt"
119-
replace_config_tags '${max_queue_delay_microseconds}' "1000000" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt"
120-
replace_config_tags '${batching_strategy}' 'inflight_fused_batching' "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt"
121-
replace_config_tags '${engine_dir}' "${ENGINES_DIR}" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt"
122-
replace_config_tags '${triton_backend}' "tensorrtllm" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt"
123-
replace_config_tags '${max_queue_size}' "0" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt"
124-
replace_config_tags '${logits_datatype}' "TYPE_FP32" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt"
125-
replace_config_tags '${encoder_input_features_data_type}' "TYPE_FP32" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt"
126-
}
127-
128-
# Wait until server health endpoint shows ready. Sets WAIT_RET to 0 on
129-
# success, 1 on failure
130-
function wait_for_server_ready() {
131-
local wait_time_secs="${1:-30}"
132-
shift
133-
local spids=("$@")
134-
135-
WAIT_RET=0
136-
137-
for _ in $(seq "$wait_time_secs"); do
138-
for pid in "${spids[@]}"; do
139-
if ! kill -0 "$pid" >/dev/null 2>&1; then
140-
echo "=== Server not running."
141-
WAIT_RET=1
142-
return
143-
fi
144-
done
145-
146-
sleep 1
147-
148-
if curl -s --fail localhost:8000/v2/health/ready &&
149-
curl -s --fail -w "%{http_code}" -o /dev/null -d '{"log_verbose_level":1}' localhost:8000/v2/logging; then
150-
return
151-
fi
152-
done
153-
154-
echo "=== Timeout $wait_time_secs secs. Server not ready."
155-
WAIT_RET=1
156-
}
157-
158-
function run_server {
159-
python3 ${TENSORRTLLM_BACKEND_DIR}/tensorrt_llm/triton_backend/scripts/launch_triton_server.py --world_size="${NUM_GPUS}" --model_repo="${MODEL_REPOSITORY}" >${SERVER_LOG} 2>&1 &
160-
sleep 2 # allow time to obtain the pid(s)
161-
# Read PIDs into an array, trimming whitespaces
162-
readarray -t SERVER_PID < <(pgrep "tritonserver")
163-
164-
wait_for_server_ready ${SERVER_TIMEOUT} "${SERVER_PID[@]}"
165-
if [ "$WAIT_RET" != "0" ]; then
166-
# Cleanup
167-
kill "${SERVER_PID[@]}" >/dev/null 2>&1 || true
168-
echo -e "\n***\n*** Failed to start $SERVER\n***"
169-
cat $SERVER_LOG
170-
exit 1
171-
fi
172-
}
173-
174-
function kill_server {
175-
pgrep tritonserver | xargs kill -SIGINT
176-
for pid in "${SERVER_PID[@]}"; do
177-
echo "Waiting for proc ${pid} to terminate..."
178-
while kill -0 $pid >/dev/null 2>&1; do
179-
sleep 1
180-
done
181-
done
182-
}
49+
source ../common/trtllm_util.sh
18350

18451
clone_tensorrt_llm_backend_repo
18552
build_gpt2_base_model

‎qa/L0_perf_tensorrt_llm/test.sh‎

Lines changed: 1 addition & 126 deletions
Original file line numberDiff line numberDiff line change
@@ -43,13 +43,7 @@ SERVER=${TRITON_DIR}/bin/tritonserver
4343
BACKEND_DIR=${TRITON_DIR}/backends
4444
SERVER_LOG="${NAME}_server.log"
4545
SERVER_TIMEOUT=${SERVER_TIMEOUT:=120}
46-
47-
function clone_tensorrt_llm_backend_repo {
48-
rm -rf $TENSORRTLLM_BACKEND_DIR && mkdir $TENSORRTLLM_BACKEND_DIR
49-
apt-get update && apt-get install git-lfs -y --no-install-recommends
50-
git clone --single-branch --depth=1 -b ${TENSORRTLLM_BACKEND_REPO_TAG} ${TRITON_REPO_ORG}/tensorrtllm_backend.git $TENSORRTLLM_BACKEND_DIR
51-
cd $TENSORRTLLM_BACKEND_DIR && git lfs install && git submodule update --init --recursive
52-
}
46+
source ../common/trtllm_util.sh
5347

5448
# Update Open MPI to a version compatible with SLURM.
5549
function upgrade_openmpi {
@@ -95,125 +89,6 @@ function upgrade_openmpi {
9589
mpirun --version
9690
}
9791

98-
function build_gpt2_base_model {
99-
# Download weights from HuggingFace Transformers
100-
cd ${GPT_DIR} && rm -rf gpt2 && git clone https://huggingface.co/gpt2-medium gpt2 && cd gpt2
101-
rm pytorch_model.bin model.safetensors
102-
if ! wget -q https://huggingface.co/gpt2-medium/resolve/main/pytorch_model.bin; then
103-
echo "Downloading pytorch_model.bin failed."
104-
exit 1
105-
fi
106-
cd ${GPT_DIR}
107-
108-
# Convert weights from HF Tranformers to FT format
109-
python3 convert_checkpoint.py --model_dir gpt2 --dtype float16 --tp_size ${NUM_GPUS} --output_dir "./c-model/gpt2/${NUM_GPUS}-gpu/"
110-
cd ${BASE_DIR}
111-
}
112-
113-
function build_gpt2_tensorrt_engine {
114-
# Build TensorRT engines
115-
cd ${GPT_DIR}
116-
trtllm-build --checkpoint_dir "./c-model/gpt2/${NUM_GPUS}-gpu/" \
117-
--gpt_attention_plugin float16 \
118-
--remove_input_padding enable \
119-
--paged_kv_cache enable \
120-
--gemm_plugin float16 \
121-
--workers "${NUM_GPUS}" \
122-
--output_dir "${ENGINES_DIR}"
123-
124-
cd ${BASE_DIR}
125-
}
126-
127-
function replace_config_tags {
128-
tag_to_replace="${1}"
129-
new_value="${2}"
130-
config_file_path="${3}"
131-
sed -i "s|${tag_to_replace}|${new_value}|g" ${config_file_path}
132-
}
133-
134-
function prepare_model_repository {
135-
rm -rf ${MODEL_REPOSITORY} && mkdir ${MODEL_REPOSITORY}
136-
cp -r ${TENSORRTLLM_BACKEND_DIR}/tensorrt_llm/triton_backend/all_models/inflight_batcher_llm/* ${MODEL_REPOSITORY}
137-
rm -rf ${MODEL_REPOSITORY}/tensorrt_llm_bls
138-
mv "${MODEL_REPOSITORY}/ensemble" "${MODEL_REPOSITORY}/${MODEL_NAME}"
139-
140-
replace_config_tags "model_version: -1" "model_version: 1" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt"
141-
replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt"
142-
replace_config_tags 'name: "ensemble"' "name: \"$MODEL_NAME\"" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt"
143-
144-
replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt"
145-
replace_config_tags '${preprocessing_instance_count}' '1' "${MODEL_REPOSITORY}/preprocessing/config.pbtxt"
146-
replace_config_tags '${tokenizer_dir}' "${TOKENIZER_DIR}/" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt"
147-
148-
replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/postprocessing/config.pbtxt"
149-
replace_config_tags '${postprocessing_instance_count}' '1' "${MODEL_REPOSITORY}/postprocessing/config.pbtxt"
150-
replace_config_tags '${tokenizer_dir}' "${TOKENIZER_DIR}/" "${MODEL_REPOSITORY}/postprocessing/config.pbtxt"
151-
152-
replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt"
153-
replace_config_tags '${decoupled_mode}' 'true' "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt"
154-
replace_config_tags '${max_queue_delay_microseconds}' "1000000" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt"
155-
replace_config_tags '${batching_strategy}' 'inflight_fused_batching' "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt"
156-
replace_config_tags '${engine_dir}' "${ENGINES_DIR}" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt"
157-
replace_config_tags '${triton_backend}' "tensorrtllm" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt"
158-
replace_config_tags '${max_queue_size}' "0" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt"
159-
}
160-
161-
# Wait until server health endpoint shows ready. Sets WAIT_RET to 0 on
162-
# success, 1 on failure
163-
function wait_for_server_ready() {
164-
local wait_time_secs="${1:-30}"
165-
shift
166-
local spids=("$@")
167-
168-
WAIT_RET=0
169-
170-
for _ in $(seq "$wait_time_secs"); do
171-
for pid in "${spids[@]}"; do
172-
if ! kill -0 "$pid" >/dev/null 2>&1; then
173-
echo "=== Server not running."
174-
WAIT_RET=1
175-
return
176-
fi
177-
done
178-
179-
sleep 1
180-
181-
if curl -s --fail localhost:8000/v2/health/ready &&
182-
curl -s --fail -w "%{http_code}" -o /dev/null -d '{"log_verbose_level":1}' localhost:8000/v2/logging; then
183-
return
184-
fi
185-
done
186-
187-
echo "=== Timeout $wait_time_secs secs. Server not ready."
188-
WAIT_RET=1
189-
}
190-
191-
function run_server {
192-
python3 ${TENSORRTLLM_BACKEND_DIR}/tensorrt_llm/triton_backend/scripts/launch_triton_server.py --world_size="${NUM_GPUS}" --model_repo="${MODEL_REPOSITORY}" >${SERVER_LOG} 2>&1 &
193-
sleep 2 # allow time to obtain the pid(s)
194-
# Read PIDs into an array, trimming whitespaces
195-
readarray -t SERVER_PID < <(pgrep "tritonserver")
196-
197-
wait_for_server_ready ${SERVER_TIMEOUT} "${SERVER_PID[@]}"
198-
if [ "$WAIT_RET" != "0" ]; then
199-
# Cleanup
200-
kill "${SERVER_PID[@]}" >/dev/null 2>&1 || true
201-
echo -e "\n***\n*** Failed to start $SERVER\n***"
202-
cat $SERVER_LOG
203-
exit 1
204-
fi
205-
}
206-
207-
function kill_server {
208-
pgrep tritonserver | xargs kill -SIGINT
209-
for pid in "${SERVER_PID[@]}"; do
210-
echo "Waiting for proc ${pid} to terminate..."
211-
while kill -0 $pid >/dev/null 2>&1; do
212-
sleep 1
213-
done
214-
done
215-
}
216-
21792
upgrade_openmpi
21893
clone_tensorrt_llm_backend_repo
21994
build_gpt2_base_model

0 commit comments

Comments
 (0)