Skip to content

Commit 66540e6

Browse files
chensuyuechensuyue
authored
[CICD enhance] SearchQnA run CI with latest base image, group logs in GHA outputs. (#2073)
Signed-off-by: chensuyue <suyue.chen@intel.com>
1 parent a1bbc69 commit 66540e6

6 files changed

Lines changed: 93 additions & 40 deletions

File tree

‎SearchQnA/Dockerfile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
# Copyright (C) 2024 Intel Corporation
22
# SPDX-License-Identifier: Apache-2.0
33

4+
ARG IMAGE_REPO=opea
45
ARG BASE_TAG=latest
5-
FROM opea/comps-base:$BASE_TAG
6+
FROM $IMAGE_REPO/comps-base:$BASE_TAG
67

78
COPY ./searchqna.py $HOME/searchqna.py
89

‎SearchQnA/docker_image_build/build.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ services:
55
searchqna:
66
build:
77
args:
8+
IMAGE_REPO: ${REGISTRY:-opea}
9+
BASE_TAG: ${TAG:-latest}
810
http_proxy: ${http_proxy}
911
https_proxy: ${https_proxy}
1012
no_proxy: ${no_proxy}

‎SearchQnA/tests/test_compose_on_gaudi.sh‎

Lines changed: 21 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -17,27 +17,17 @@ ip_address=$(hostname -I | awk '{print $1}')
1717

1818
function build_docker_images() {
1919
opea_branch=${opea_branch:-"main"}
20-
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
21-
if [[ "${opea_branch}" != "main" ]]; then
22-
cd $WORKPATH
23-
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
24-
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
25-
find . -type f -name "Dockerfile*" | while read -r file; do
26-
echo "Processing file: $file"
27-
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
28-
done
29-
fi
30-
3120
cd $WORKPATH/docker_image_build
3221
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
22+
pushd GenAIComps
23+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
24+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
25+
popd && sleep 1s
3326

3427
echo "Build all the images with --no-cache, check docker_image_build.log for details..."
3528
service_list="searchqna searchqna-ui embedding web-retriever reranking llm-textgen nginx"
3629
docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
3730

38-
docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
39-
docker pull ghcr.io/huggingface/tei-gaudi:1.5.0
40-
docker pull ghcr.io/huggingface/tgi-gaudi:2.3.1
4131
docker images && sleep 1s
4232
}
4333

@@ -114,15 +104,31 @@ function stop_docker() {
114104

115105
function main() {
116106

107+
echo "::group::stop_docker"
117108
stop_docker
109+
echo "::endgroup::"
110+
111+
echo "::group::build_docker_images"
118112
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
113+
echo "::endgroup::"
114+
115+
echo "::group::start_services"
119116
start_services
117+
echo "::endgroup::"
120118

119+
echo "::group::validate_megaservice"
121120
validate_megaservice
121+
echo "::endgroup::"
122+
123+
echo "::group::validate_frontend"
122124
validate_frontend
125+
echo "::endgroup::"
123126

127+
echo "::group::stop_docker"
124128
stop_docker
125-
echo y | docker system prune
129+
echo "::endgroup::"
130+
131+
docker system prune -f
126132

127133
}
128134

‎SearchQnA/tests/test_compose_on_rocm.sh‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,18 @@ LOG_PATH="$WORKPATH/tests"
1616
ip_address=$(hostname -I | awk '{print $1}')
1717

1818
function build_docker_images() {
19+
opea_branch=${opea_branch:-"main"}
1920
cd $WORKPATH/docker_image_build
20-
git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout "${opea_branch:-"main"}" && cd ../
21+
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
22+
pushd GenAIComps
23+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
24+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
25+
popd && sleep 1s
2126

2227
echo "Build all the images with --no-cache, check docker_image_build.log for details..."
2328
service_list="searchqna searchqna-ui embedding web-retriever reranking llm-textgen nginx"
2429
docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
2530

26-
docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
27-
docker pull ghcr.io/huggingface/text-generation-inference:2.4.1-rocm
2831
docker images && sleep 1s
2932
}
3033

@@ -38,7 +41,7 @@ function start_services() {
3841
docker compose up -d > ${LOG_PATH}/start_services_with_compose.log
3942
n=0
4043
until [[ "$n" -ge 100 ]]; do
41-
docker logs search-tgi-service > $LOG_PATH/search-tgi-service_start.log
44+
docker logs search-tgi-service > $LOG_PATH/search-tgi-service_start.log 2>&1
4245
if grep -q Connected $LOG_PATH/search-tgi-service_start.log; then
4346
break
4447
fi
@@ -101,15 +104,31 @@ function stop_docker() {
101104

102105
function main() {
103106

107+
echo "::group::stop_docker"
104108
stop_docker
109+
echo "::endgroup::"
110+
111+
echo "::group::build_docker_images"
105112
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
113+
echo "::endgroup::"
114+
115+
echo "::group::start_services"
106116
start_services
117+
echo "::endgroup::"
107118

119+
echo "::group::validate_megaservice"
108120
validate_megaservice
121+
echo "::endgroup::"
122+
123+
echo "::group::validate_frontend"
109124
validate_frontend
125+
echo "::endgroup::"
110126

127+
echo "::group::stop_docker"
111128
stop_docker
112-
echo y | docker system prune
129+
echo "::endgroup::"
130+
131+
docker system prune -f
113132

114133
}
115134

‎SearchQnA/tests/test_compose_on_xeon.sh‎

Lines changed: 21 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -17,26 +17,17 @@ ip_address=$(hostname -I | awk '{print $1}')
1717

1818
function build_docker_images() {
1919
opea_branch=${opea_branch:-"main"}
20-
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
21-
if [[ "${opea_branch}" != "main" ]]; then
22-
cd $WORKPATH
23-
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
24-
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
25-
find . -type f -name "Dockerfile*" | while read -r file; do
26-
echo "Processing file: $file"
27-
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
28-
done
29-
fi
30-
3120
cd $WORKPATH/docker_image_build
3221
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
22+
pushd GenAIComps
23+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
24+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
25+
popd && sleep 1s
3326

3427
echo "Build all the images with --no-cache, check docker_image_build.log for details..."
3528
service_list="searchqna searchqna-ui embedding web-retriever reranking llm-textgen nginx"
3629
docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
3730

38-
docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
39-
docker pull ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu
4031
docker images && sleep 1s
4132
}
4233

@@ -109,15 +100,31 @@ function stop_docker() {
109100

110101
function main() {
111102

103+
echo "::group::stop_docker"
112104
stop_docker
105+
echo "::endgroup::"
106+
107+
echo "::group::build_docker_images"
113108
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
109+
echo "::endgroup::"
110+
111+
echo "::group::start_services"
114112
start_services
113+
echo "::endgroup::"
115114

115+
echo "::group::validate_megaservice"
116116
validate_megaservice
117+
echo "::endgroup::"
118+
119+
echo "::group::validate_frontend"
117120
validate_frontend
121+
echo "::endgroup::"
118122

123+
echo "::group::stop_docker"
119124
stop_docker
120-
echo y | docker system prune
125+
echo "::endgroup::"
126+
127+
docker system prune -f
121128

122129
}
123130

‎SearchQnA/tests/test_compose_vllm_on_rocm.sh‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,18 @@ LOG_PATH="$WORKPATH/tests"
1616
ip_address=$(hostname -I | awk '{print $1}')
1717

1818
function build_docker_images() {
19+
opea_branch=${opea_branch:-"main"}
1920
cd $WORKPATH/docker_image_build
20-
git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout "${opea_branch:-"main"}" && cd ../
21+
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
22+
pushd GenAIComps
23+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
24+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
25+
popd && sleep 1s
2126

2227
echo "Build all the images with --no-cache, check docker_image_build.log for details..."
2328
service_list="searchqna searchqna-ui embedding web-retriever reranking llm-textgen vllm-rocm nginx"
2429
docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
2530

26-
docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
2731
docker images && sleep 3s
2832
}
2933

@@ -102,17 +106,31 @@ function stop_docker() {
102106

103107
function main() {
104108

109+
echo "::group::stop_docker"
105110
stop_docker
111+
echo "::endgroup::"
112+
113+
echo "::group::build_docker_images"
106114
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
107-
start_services
115+
echo "::endgroup::"
108116

109-
sleep 20
117+
echo "::group::start_services"
118+
start_services
119+
echo "::endgroup::"
110120

121+
echo "::group::validate_megaservice"
111122
validate_megaservice
123+
echo "::endgroup::"
124+
125+
echo "::group::validate_frontend"
112126
validate_frontend
127+
echo "::endgroup::"
113128

129+
echo "::group::stop_docker"
114130
stop_docker
115-
echo y | docker system prune
131+
echo "::endgroup::"
132+
133+
docker system prune -f
116134

117135
}
118136

0 commit comments

Comments
 (0)