Skip to content

Commit 6f7daf8

Browse files
chensuyuechensuyue
authored
[CICD enhance] run CI with latest base image, group logs in GHA outputs. (#2075)
Signed-off-by: chensuyue <suyue.chen@intel.com>
1 parent 8d75f0a commit 6f7daf8

8 files changed

Lines changed: 132 additions & 64 deletions

File tree

‎Text2Image/tests/test_compose_on_gaudi.sh‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ function build_docker_images() {
2626
docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
2727
}
2828

29-
function start_service() {
29+
function start_services() {
3030
cd $WORKPATH/docker_compose/intel/hpu/gaudi
3131
export no_proxy="localhost,127.0.0.1,"${ip_address}
3232
# Start Docker Containers
@@ -69,15 +69,27 @@ function stop_docker() {
6969

7070
function main() {
7171

72+
echo "::group::stop_docker"
7273
stop_docker
74+
echo "::endgroup::"
7375

74-
build_docker_images
75-
start_service
76+
echo "::group::build_docker_images"
77+
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
78+
echo "::endgroup::"
7679

80+
echo "::group::start_services"
81+
start_services
82+
echo "::endgroup::"
83+
84+
echo "::group::validate_microservice"
7785
validate_microservice
86+
echo "::endgroup::"
7887

88+
echo "::group::stop_docker"
7989
stop_docker
80-
echo y | docker system prune
90+
echo "::endgroup::"
91+
92+
docker system prune -f
8193

8294
}
8395

‎Text2Image/tests/test_compose_on_xeon.sh‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ function build_docker_images() {
2626
docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
2727
}
2828

29-
function start_service() {
29+
function start_services() {
3030
cd $WORKPATH/docker_compose/intel/cpu/xeon
3131
export no_proxy="localhost,127.0.0.1,"${ip_address}
3232
docker compose -f compose.yaml up -d > ${LOG_PATH}/start_services_with_compose.log
@@ -68,15 +68,27 @@ function stop_docker() {
6868

6969
function main() {
7070

71+
echo "::group::stop_docker"
7172
stop_docker
73+
echo "::endgroup::"
7274

73-
build_docker_images
74-
start_service
75+
echo "::group::build_docker_images"
76+
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
77+
echo "::endgroup::"
7578

79+
echo "::group::start_services"
80+
start_services
81+
echo "::endgroup::"
82+
83+
echo "::group::validate_microservice"
7684
validate_microservice
85+
echo "::endgroup::"
7786

87+
echo "::group::stop_docker"
7888
stop_docker
79-
echo y | docker system prune
89+
echo "::endgroup::"
90+
91+
docker system prune -f
8092

8193
}
8294

‎Translation/Dockerfile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
# Copyright (C) 2024 Intel Corporation
22
# SPDX-License-Identifier: Apache-2.0
33

4+
ARG IMAGE_REPO=opea
45
ARG BASE_TAG=latest
5-
FROM opea/comps-base:$BASE_TAG
6+
FROM $IMAGE_REPO/comps-base:$BASE_TAG
67

78
COPY ./translation.py $HOME/translation.py
89

‎Translation/docker_image_build/build.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ services:
55
translation:
66
build:
77
args:
8+
IMAGE_REPO: ${REGISTRY:-opea}
9+
BASE_TAG: ${TAG:-latest}
810
http_proxy: ${http_proxy}
911
https_proxy: ${https_proxy}
1012
no_proxy: ${no_proxy}

‎Translation/tests/test_compose_on_gaudi.sh‎

Lines changed: 24 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -17,25 +17,17 @@ ip_address=$(hostname -I | awk '{print $1}')
1717

1818
function build_docker_images() {
1919
opea_branch=${opea_branch:-"main"}
20-
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
21-
if [[ "${opea_branch}" != "main" ]]; then
22-
cd $WORKPATH
23-
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
24-
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
25-
find . -type f -name "Dockerfile*" | while read -r file; do
26-
echo "Processing file: $file"
27-
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
28-
done
29-
fi
30-
3120
cd $WORKPATH/docker_image_build
3221
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
22+
pushd GenAIComps
23+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
24+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
25+
popd && sleep 1s
3326

3427
echo "Build all the images with --no-cache, check docker_image_build.log for details..."
3528
service_list="translation translation-ui llm-textgen nginx"
3629
docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
3730

38-
docker pull ghcr.io/huggingface/tgi-gaudi:2.3.1
3931
docker images && sleep 1s
4032
}
4133

@@ -52,7 +44,7 @@ function start_services() {
5244

5345
n=0
5446
until [[ "$n" -ge 100 ]]; do
55-
docker logs tgi-gaudi-server > ${LOG_PATH}/tgi_service_start.log
47+
docker logs tgi-gaudi-server > ${LOG_PATH}/tgi_service_start.log 2>&1
5648
if grep -q Connected ${LOG_PATH}/tgi_service_start.log; then
5749
break
5850
fi
@@ -162,17 +154,35 @@ function stop_docker() {
162154

163155
function main() {
164156

157+
echo "::group::stop_docker"
165158
stop_docker
159+
echo "::endgroup::"
166160

161+
echo "::group::build_docker_images"
167162
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
163+
echo "::endgroup::"
164+
165+
echo "::group::start_services"
168166
start_services
167+
echo "::endgroup::"
169168

169+
echo "::group::validate_microservices"
170170
validate_microservices
171+
echo "::endgroup::"
172+
173+
echo "::group::validate_megaservice"
171174
validate_megaservice
175+
echo "::endgroup::"
176+
177+
# echo "::group::validate_frontend"
172178
# validate_frontend
179+
# echo "::endgroup::"
173180

181+
echo "::group::stop_docker"
174182
stop_docker
175-
echo y | docker system prune
183+
echo "::endgroup::"
184+
185+
docker system prune -f
176186

177187
}
178188

‎Translation/tests/test_compose_on_rocm.sh‎

Lines changed: 24 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -17,25 +17,17 @@ ip_address=$(hostname -I | awk '{print $1}')
1717

1818
function build_docker_images() {
1919
opea_branch=${opea_branch:-"main"}
20-
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
21-
if [[ "${opea_branch}" != "main" ]]; then
22-
cd $WORKPATH
23-
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
24-
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
25-
find . -type f -name "Dockerfile*" | while read -r file; do
26-
echo "Processing file: $file"
27-
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
28-
done
29-
fi
30-
3120
cd $WORKPATH/docker_image_build
3221
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
22+
pushd GenAIComps
23+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
24+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
25+
popd && sleep 1s
3326

3427
echo "Build all the images with --no-cache, check docker_image_build.log for details..."
3528
service_list="translation translation-ui llm-textgen nginx"
3629
docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
3730

38-
docker pull ghcr.io/huggingface/text-generation-inference:2.4.1-rocm
3931
docker images && sleep 1s
4032
}
4133

@@ -53,7 +45,7 @@ function start_services() {
5345
n=0
5446
# wait long for llm model download
5547
until [[ "$n" -ge 500 ]]; do
56-
docker logs translation-tgi-service > ${LOG_PATH}/translation-tgi-service_start.log
48+
docker logs translation-tgi-service > ${LOG_PATH}/translation-tgi-service_start.log 2>&1
5749
if grep -q Connected ${LOG_PATH}/translation-tgi-service_start.log; then
5850
break
5951
fi
@@ -163,17 +155,35 @@ function stop_docker() {
163155

164156
function main() {
165157

158+
echo "::group::stop_docker"
166159
stop_docker
160+
echo "::endgroup::"
167161

162+
echo "::group::build_docker_images"
168163
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
164+
echo "::endgroup::"
165+
166+
echo "::group::start_services"
169167
start_services
168+
echo "::endgroup::"
170169

170+
echo "::group::validate_microservices"
171171
validate_microservices
172+
echo "::endgroup::"
173+
174+
echo "::group::validate_megaservice"
172175
validate_megaservice
176+
echo "::endgroup::"
177+
178+
echo "::group::validate_frontend"
173179
validate_frontend
180+
echo "::endgroup::"
174181

182+
echo "::group::stop_docker"
175183
stop_docker
176-
echo y | docker system prune
184+
echo "::endgroup::"
185+
186+
docker system prune -f
177187

178188
}
179189

‎Translation/tests/test_compose_on_xeon.sh‎

Lines changed: 24 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -17,25 +17,17 @@ ip_address=$(hostname -I | awk '{print $1}')
1717

1818
function build_docker_images() {
1919
opea_branch=${opea_branch:-"main"}
20-
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
21-
if [[ "${opea_branch}" != "main" ]]; then
22-
cd $WORKPATH
23-
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
24-
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
25-
find . -type f -name "Dockerfile*" | while read -r file; do
26-
echo "Processing file: $file"
27-
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
28-
done
29-
fi
30-
3120
cd $WORKPATH/docker_image_build
3221
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
22+
pushd GenAIComps
23+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
24+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
25+
popd && sleep 1s
3326

3427
echo "Build all the images with --no-cache, check docker_image_build.log for details..."
3528
service_list="translation translation-ui llm-textgen nginx"
3629
docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
3730

38-
docker pull ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu
3931
docker images && sleep 1s
4032
}
4133

@@ -53,7 +45,7 @@ function start_services() {
5345
n=0
5446
# wait long for llm model download
5547
until [[ "$n" -ge 500 ]]; do
56-
docker logs tgi-service > ${LOG_PATH}/tgi_service_start.log
48+
docker logs tgi-service > ${LOG_PATH}/tgi_service_start.log 2>&1
5749
if grep -q Connected ${LOG_PATH}/tgi_service_start.log; then
5850
break
5951
fi
@@ -163,17 +155,35 @@ function stop_docker() {
163155

164156
function main() {
165157

158+
echo "::group::stop_docker"
166159
stop_docker
160+
echo "::endgroup::"
167161

162+
echo "::group::build_docker_images"
168163
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
164+
echo "::endgroup::"
165+
166+
echo "::group::start_services"
169167
start_services
168+
echo "::endgroup::"
170169

170+
echo "::group::validate_microservices"
171171
validate_microservices
172+
echo "::endgroup::"
173+
174+
echo "::group::validate_megaservice"
172175
validate_megaservice
176+
echo "::endgroup::"
177+
178+
echo "::group::validate_frontend"
173179
validate_frontend
180+
echo "::endgroup::"
174181

182+
echo "::group::stop_docker"
175183
stop_docker
176-
echo y | docker system prune
184+
echo "::endgroup::"
185+
186+
docker system prune -f
177187

178188
}
179189

‎Translation/tests/test_compose_vllm_on_rocm.sh‎

Lines changed: 24 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -16,19 +16,12 @@ ip_address=$(hostname -I | awk '{print $1}')
1616

1717
function build_docker_images() {
1818
opea_branch=${opea_branch:-"main"}
19-
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
20-
if [[ "${opea_branch}" != "main" ]]; then
21-
cd $WORKPATH
22-
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
23-
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
24-
find . -type f -name "Dockerfile*" | while read -r file; do
25-
echo "Processing file: $file"
26-
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
27-
done
28-
fi
29-
3019
cd $WORKPATH/docker_image_build
3120
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
21+
pushd GenAIComps
22+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
23+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
24+
popd && sleep 1s
3225

3326
echo "Build all the images with --no-cache, check docker_image_build.log for details..."
3427
service_list="translation translation-ui llm-textgen nginx vllm-rocm"
@@ -50,7 +43,7 @@ function start_services() {
5043
n=0
5144
# wait long for llm model download
5245
until [[ "$n" -ge 500 ]]; do
53-
docker logs translation-vllm-service >& ${LOG_PATH}/translation-vllm-service_start.log
46+
docker logs translation-vllm-service > ${LOG_PATH}/translation-vllm-service_start.log 2>&1
5447
if grep -q "Application startup complete" ${LOG_PATH}/translation-vllm-service_start.log; then
5548
echo "vLLM check successful"
5649
break
@@ -162,17 +155,35 @@ function stop_docker() {
162155

163156
function main() {
164157

158+
echo "::group::stop_docker"
165159
stop_docker
160+
echo "::endgroup::"
166161

162+
echo "::group::build_docker_images"
167163
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
164+
echo "::endgroup::"
165+
166+
echo "::group::start_services"
168167
start_services
168+
echo "::endgroup::"
169169

170+
echo "::group::validate_microservices"
170171
validate_microservices
172+
echo "::endgroup::"
173+
174+
echo "::group::validate_megaservice"
171175
validate_megaservice
176+
echo "::endgroup::"
177+
178+
echo "::group::validate_frontend"
172179
validate_frontend
180+
echo "::endgroup::"
173181

182+
echo "::group::stop_docker"
174183
stop_docker
175-
echo y | docker system prune
184+
echo "::endgroup::"
185+
186+
docker system prune -f
176187

177188
}
178189

0 commit comments

Comments
 (0)