From d3d9d5f350837d1edf7f77c609c5ccb958e46199 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 14 May 2025 15:50:23 +0800 Subject: [PATCH 1/6] Integrate VisualQnA set_env to ut scripts. Signed-off-by: ZePan110 --- .../amd/gpu/rocm/set_env_vllm.sh | 6 +-- VisualQnA/tests/README.md | 45 +++++++++++++++++++ VisualQnA/tests/test_compose_on_rocm.sh | 18 +------- VisualQnA/tests/test_compose_tgi_on_gaudi.sh | 11 +---- VisualQnA/tests/test_compose_tgi_on_xeon.sh | 14 +----- VisualQnA/tests/test_compose_vllm_on_rocm.sh | 19 +------- 6 files changed, 55 insertions(+), 58 deletions(-) create mode 100644 VisualQnA/tests/README.md diff --git a/VisualQnA/docker_compose/amd/gpu/rocm/set_env_vllm.sh b/VisualQnA/docker_compose/amd/gpu/rocm/set_env_vllm.sh index 7786abe497..68a1bb0b9d 100644 --- a/VisualQnA/docker_compose/amd/gpu/rocm/set_env_vllm.sh +++ b/VisualQnA/docker_compose/amd/gpu/rocm/set_env_vllm.sh @@ -6,7 +6,7 @@ export HOST_IP=${host_ip} export EXTERNAL_HOST_IP=${host_ip} export VISUALQNA_VLLM_SERVICE_PORT="8081" -export VISUALQNA_HUGGINGFACEHUB_API_TOKEN=${Your_HUGGINGFACEHUB_API_TOKEN} +export VISUALQNA_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} export VISUALQNA_CARD_ID="card1" export VISUALQNA_RENDER_ID="renderD136" export VISUALQNA_LVM_MODEL_ID="Xkev/Llama-3.2V-11B-cot" @@ -16,8 +16,8 @@ export MEGA_SERVICE_HOST_IP=${HOST_IP} export LVM_SERVICE_HOST_IP=${HOST_IP} export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:${BACKEND_SERVICE_PORT}/v1/visualqna" export FRONTEND_SERVICE_IP=${HOST_IP} -export FRONTEND_SERVICE_PORT=18001 +export FRONTEND_SERVICE_PORT=5173 export BACKEND_SERVICE_NAME=visualqna export BACKEND_SERVICE_IP=${HOST_IP} -export BACKEND_SERVICE_PORT=18002 +export BACKEND_SERVICE_PORT=8888 export NGINX_PORT=18003 diff --git a/VisualQnA/tests/README.md b/VisualQnA/tests/README.md new file mode 100644 index 0000000000..8d07371b51 --- /dev/null +++ b/VisualQnA/tests/README.md @@ -0,0 +1,45 @@ +# VisualQnA E2E test scripts + +## Set the required environment variable + +```bash +export HUGGINGFACEHUB_API_TOKEN="Your_Huggingface_API_Token" +``` + +## Run test + +On Intel Xeon with Vllm: + +```bash +bash test_compose_on_xeon.sh +``` + +On Intel Gaudi with Vllm: + +```bash +bash test_compose_on_gaudi.sh +``` + +On Intel Xeon with TGI: + +```bash +bash test_compose_tgi_on_xeon.sh +``` + +On Intel Gaudi with TGI: + +```bash +bash test_compose_tgi_on_gaudi.sh +``` + +On AMD ROCm with TGI: + +```bash +bash test_compose_on_rocm.sh +``` + +On AMD ROCm with vLLM: + +```bash +bash test_compose_vllm_on_rocm.sh +``` diff --git a/VisualQnA/tests/test_compose_on_rocm.sh b/VisualQnA/tests/test_compose_on_rocm.sh index dcdf4e4719..6e2213e8db 100644 --- a/VisualQnA/tests/test_compose_on_rocm.sh +++ b/VisualQnA/tests/test_compose_on_rocm.sh @@ -15,26 +15,12 @@ ip_address=$(hostname -I | awk '{print $1}') export MODEL_CACHE=${model_cache:-"/var/lib/GenAI/data"} export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} -export HOST_IP=${ip_address} -export VISUALQNA_TGI_SERVICE_PORT="8399" -export VISUALQNA_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} -export VISUALQNA_CARD_ID="card1" -export VISUALQNA_RENDER_ID="renderD136" -export LVM_MODEL_ID="Xkev/Llama-3.2V-11B-cot" +export host_ip=${ip_address} export MODEL="llava-hf/llava-v1.6-mistral-7b-hf" -export LVM_ENDPOINT="http://${HOST_IP}:8399" -export LVM_SERVICE_PORT=9399 -export MEGA_SERVICE_HOST_IP=${HOST_IP} -export LVM_SERVICE_HOST_IP=${HOST_IP} -export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${BACKEND_SERVICE_PORT}/v1/visualqna" -export FRONTEND_SERVICE_IP=${HOST_IP} export FRONTEND_SERVICE_PORT=5173 -export BACKEND_SERVICE_NAME=visualqna -export BACKEND_SERVICE_IP=${HOST_IP} export BACKEND_SERVICE_PORT=8888 -export NGINX_PORT=18003 export PATH="~/miniconda3/bin:$PATH" -export MODEL_CACHE=${model_cache:-"/var/opea/multimodalqna-service/data"} +source $WORKPATH/docker_compose/amd/gpu/rocm/set_env.sh function build_docker_images() { opea_branch=${opea_branch:-"main"} diff --git a/VisualQnA/tests/test_compose_tgi_on_gaudi.sh b/VisualQnA/tests/test_compose_tgi_on_gaudi.sh index 913d6ed527..9e3c8f0f72 100644 --- a/VisualQnA/tests/test_compose_tgi_on_gaudi.sh +++ b/VisualQnA/tests/test_compose_tgi_on_gaudi.sh @@ -42,19 +42,10 @@ function start_services() { cd $WORKPATH/docker_compose/intel/hpu/gaudi export LVM_MODEL_ID="llava-hf/llava-v1.6-mistral-7b-hf" - export LVM_ENDPOINT="http://${ip_address}:8399" export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} - export LVM_SERVICE_PORT=9399 - export MEGA_SERVICE_HOST_IP=${ip_address} - export LVM_SERVICE_HOST_IP=${ip_address} - export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:8888/v1/visualqna" - export FRONTEND_SERVICE_IP=${ip_address} - export FRONTEND_SERVICE_PORT=5173 - export BACKEND_SERVICE_NAME=visualqna - export BACKEND_SERVICE_IP=${ip_address} - export BACKEND_SERVICE_PORT=8888 export NGINX_PORT=80 export host_ip=${ip_address} + source ./set_env.sh sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env diff --git a/VisualQnA/tests/test_compose_tgi_on_xeon.sh b/VisualQnA/tests/test_compose_tgi_on_xeon.sh index d6311719d0..1c5e2a35f0 100644 --- a/VisualQnA/tests/test_compose_tgi_on_xeon.sh +++ b/VisualQnA/tests/test_compose_tgi_on_xeon.sh @@ -40,21 +40,11 @@ function build_docker_images() { function start_services() { cd $WORKPATH/docker_compose/intel/cpu/xeon/ - - export LVM_MODEL_ID="llava-hf/llava-v1.6-mistral-7b-hf" - export LVM_ENDPOINT="http://${ip_address}:8399" + export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} - export LVM_SERVICE_PORT=9399 - export MEGA_SERVICE_HOST_IP=${ip_address} - export LVM_SERVICE_HOST_IP=${ip_address} - export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:8888/v1/visualqna" - export FRONTEND_SERVICE_IP=${ip_address} - export FRONTEND_SERVICE_PORT=5173 - export BACKEND_SERVICE_NAME=visualqna - export BACKEND_SERVICE_IP=${ip_address} - export BACKEND_SERVICE_PORT=8888 export NGINX_PORT=80 export host_ip=${ip_address} + source ./set_env.sh sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env diff --git a/VisualQnA/tests/test_compose_vllm_on_rocm.sh b/VisualQnA/tests/test_compose_vllm_on_rocm.sh index abf014631c..4bdd0f72d0 100644 --- a/VisualQnA/tests/test_compose_vllm_on_rocm.sh +++ b/VisualQnA/tests/test_compose_vllm_on_rocm.sh @@ -14,26 +14,11 @@ ip_address=$(hostname -I | awk '{print $1}') export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} -export HOST_IP=${ip_address} -export VISUALQNA_VLLM_SERVICE_PORT="8081" -export VISUALQNA_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} -export VISUALQNA_CARD_ID="card1" -export VISUALQNA_RENDER_ID="renderD136" -export VISUALQNA_LVM_MODEL_ID="Xkev/Llama-3.2V-11B-cot" +export host_ip=${ip_address} export MODEL="llava-hf/llava-v1.6-mistral-7b-hf" -export LVM_ENDPOINT="http://${HOST_IP}:${VISUALQNA_VLLM_SERVICE_PORT}" -export LVM_SERVICE_PORT=9399 -export MEGA_SERVICE_HOST_IP=${HOST_IP} -export LVM_SERVICE_HOST_IP=${HOST_IP} -export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${BACKEND_SERVICE_PORT}/v1/visualqna" -export FRONTEND_SERVICE_IP=${HOST_IP} -export FRONTEND_SERVICE_PORT=5173 -export BACKEND_SERVICE_NAME=visualqna -export BACKEND_SERVICE_IP=${HOST_IP} -export BACKEND_SERVICE_PORT=8888 -export NGINX_PORT=18003 export PATH="~/miniconda3/bin:$PATH" export MODEL_CACHE=${model_cache:-"/var/opea/multimodalqna-service/data"} +source $WORKPATH/docker_compose/amd/gpu/rocm/set_env_vllm.sh function build_docker_images() { opea_branch=${opea_branch:-"main"} From 9a92bc84c07fe7ecac35cdfc69f51e4054f3623f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 14 May 2025 07:56:22 +0000 Subject: [PATCH 2/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- VisualQnA/tests/test_compose_tgi_on_xeon.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VisualQnA/tests/test_compose_tgi_on_xeon.sh b/VisualQnA/tests/test_compose_tgi_on_xeon.sh index 1c5e2a35f0..5e591b8ea0 100644 --- a/VisualQnA/tests/test_compose_tgi_on_xeon.sh +++ b/VisualQnA/tests/test_compose_tgi_on_xeon.sh @@ -40,7 +40,7 @@ function build_docker_images() { function start_services() { cd $WORKPATH/docker_compose/intel/cpu/xeon/ - + export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} export NGINX_PORT=80 export host_ip=${ip_address} From fc322dc69bf819d88a069dfbcf62a39557d252f7 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 14 May 2025 16:26:53 +0800 Subject: [PATCH 3/6] Switch vllm to v0.6.6.post1+Gaudi-1.20.0 Signed-off-by: ZePan110 --- VisualQnA/tests/test_compose_on_gaudi.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/VisualQnA/tests/test_compose_on_gaudi.sh b/VisualQnA/tests/test_compose_on_gaudi.sh index 3fbc8e0adc..3832829f6c 100644 --- a/VisualQnA/tests/test_compose_on_gaudi.sh +++ b/VisualQnA/tests/test_compose_on_gaudi.sh @@ -24,6 +24,8 @@ function build_docker_images() { git clone https://github.com/HabanaAI/vllm-fork.git cd ./vllm-fork/ + VLLM_FORK_VER=v0.6.6.post1+Gaudi-1.20.0 + git checkout ${VLLM_FORK_VER} &> /dev/null docker build -f Dockerfile.hpu -t opea/vllm-gaudi:${TAG} --shm-size=128g . --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy cd .. rm -rf vllm-fork From 094a3f746e7cf6609aea12d49da179d0bd319cde Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 14 May 2025 16:39:35 +0800 Subject: [PATCH 4/6] Fix Signed-off-by: ZePan110 --- VisualQnA/docker_compose/amd/gpu/rocm/set_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VisualQnA/docker_compose/amd/gpu/rocm/set_env.sh b/VisualQnA/docker_compose/amd/gpu/rocm/set_env.sh index eecc6d0575..e0a0f95042 100644 --- a/VisualQnA/docker_compose/amd/gpu/rocm/set_env.sh +++ b/VisualQnA/docker_compose/amd/gpu/rocm/set_env.sh @@ -6,7 +6,7 @@ export HOST_IP=${host_ip} export EXTERNAL_HOST_IP=${host_ip} export VISUALQNA_TGI_SERVICE_PORT="8399" -export VISUALQNA_HUGGINGFACEHUB_API_TOKEN=${Your_HUGGINGFACEHUB_API_TOKEN} +export VISUALQNA_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} export VISUALQNA_CARD_ID="card1" export VISUALQNA_RENDER_ID="renderD136" export LVM_MODEL_ID="Xkev/Llama-3.2V-11B-cot" From 87c8c60375fb1880831b0d443c176fa917c76a64 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Thu, 15 May 2025 08:51:48 +0800 Subject: [PATCH 5/6] Fix Signed-off-by: ZePan110 --- VisualQnA/tests/test_compose_tgi_on_gaudi.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VisualQnA/tests/test_compose_tgi_on_gaudi.sh b/VisualQnA/tests/test_compose_tgi_on_gaudi.sh index 9e3c8f0f72..f13924ab2c 100644 --- a/VisualQnA/tests/test_compose_tgi_on_gaudi.sh +++ b/VisualQnA/tests/test_compose_tgi_on_gaudi.sh @@ -41,11 +41,11 @@ function build_docker_images() { function start_services() { cd $WORKPATH/docker_compose/intel/hpu/gaudi - export LVM_MODEL_ID="llava-hf/llava-v1.6-mistral-7b-hf" export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} export NGINX_PORT=80 export host_ip=${ip_address} source ./set_env.sh + export LVM_MODEL_ID="llava-hf/llava-v1.6-mistral-7b-hf" sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env From a748826ffd9c81ed6f21c4237701742ad08b173d Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Thu, 15 May 2025 16:00:22 +0800 Subject: [PATCH 6/6] Fix Signed-off-by: ZePan110 --- VisualQnA/docker_compose/amd/gpu/rocm/set_env.sh | 4 ++-- VisualQnA/tests/test_compose_on_rocm.sh | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/VisualQnA/docker_compose/amd/gpu/rocm/set_env.sh b/VisualQnA/docker_compose/amd/gpu/rocm/set_env.sh index e0a0f95042..1cdf88a262 100644 --- a/VisualQnA/docker_compose/amd/gpu/rocm/set_env.sh +++ b/VisualQnA/docker_compose/amd/gpu/rocm/set_env.sh @@ -16,8 +16,8 @@ export MEGA_SERVICE_HOST_IP=${HOST_IP} export LVM_SERVICE_HOST_IP=${HOST_IP} export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:${BACKEND_SERVICE_PORT}/v1/visualqna" export FRONTEND_SERVICE_IP=${HOST_IP} -export FRONTEND_SERVICE_PORT=18001 +export FRONTEND_SERVICE_PORT=5173 export BACKEND_SERVICE_NAME=visualqna export BACKEND_SERVICE_IP=${HOST_IP} -export BACKEND_SERVICE_PORT=18002 +export BACKEND_SERVICE_PORT=8888 export NGINX_PORT=18003 diff --git a/VisualQnA/tests/test_compose_on_rocm.sh b/VisualQnA/tests/test_compose_on_rocm.sh index 6e2213e8db..b587295c59 100644 --- a/VisualQnA/tests/test_compose_on_rocm.sh +++ b/VisualQnA/tests/test_compose_on_rocm.sh @@ -17,8 +17,6 @@ export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} export host_ip=${ip_address} export MODEL="llava-hf/llava-v1.6-mistral-7b-hf" -export FRONTEND_SERVICE_PORT=5173 -export BACKEND_SERVICE_PORT=8888 export PATH="~/miniconda3/bin:$PATH" source $WORKPATH/docker_compose/amd/gpu/rocm/set_env.sh