Skip to content

Commit 09d93ec

Browse files
chensuyuechensuyue
authored
[CICD enhance] CodeGen run CI with latest base image, group logs in GHA outputs. (#1928)
Signed-off-by: chensuyue <suyue.chen@intel.com>
1 parent ed918bc commit 09d93ec

6 files changed

Lines changed: 86 additions & 48 deletions

File tree

‎CodeGen/Dockerfile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
# Copyright (C) 2024 Intel Corporation
22
# SPDX-License-Identifier: Apache-2.0
33

4+
ARG IMAGE_REPO=opea
45
ARG BASE_TAG=latest
5-
FROM opea/comps-base:$BASE_TAG
6+
FROM $IMAGE_REPO/comps-base:$BASE_TAG
67

78
COPY ./codegen.py $HOME/codegen.py
89

‎CodeGen/docker_image_build/build.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ services:
55
codegen:
66
build:
77
args:
8+
IMAGE_REPO: ${REGISTRY}
9+
BASE_TAG: ${TAG}
810
http_proxy: ${http_proxy}
911
https_proxy: ${https_proxy}
1012
no_proxy: ${no_proxy}
@@ -39,6 +41,7 @@ services:
3941
build:
4042
context: GenAIComps
4143
dockerfile: comps/third_parties/vllm/src/Dockerfile.amd_gpu
44+
extends: codegen
4245
image: ${REGISTRY:-opea}/vllm-rocm:${TAG:-latest}
4346
vllm:
4447
build:

‎CodeGen/tests/test_compose_on_gaudi.sh‎

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -27,19 +27,13 @@ export no_proxy=${no_proxy},${ip_address}
2727

2828
function build_docker_images() {
2929
opea_branch=${opea_branch:-"main"}
30-
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
31-
if [[ "${opea_branch}" != "main" ]]; then
32-
cd $WORKPATH
33-
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
34-
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
35-
find . -type f -name "Dockerfile*" | while read -r file; do
36-
echo "Processing file: $file"
37-
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
38-
done
39-
fi
4030

4131
cd $WORKPATH/docker_image_build
4232
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
33+
pushd GenAIComps
34+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
35+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
36+
popd && sleep 1s
4337

4438
# Download Gaudi vllm of latest tag
4539
git clone https://github.com/HabanaAI/vllm-fork.git && cd vllm-fork
@@ -250,24 +244,36 @@ function main() {
250244
stop_docker "${docker_compose_profiles[${i}]}"
251245
done
252246

253-
# build docker images
247+
echo "::group::build_docker_images"
254248
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
249+
echo "::endgroup::"
255250

256251
# loop all profiles
257252
for ((i = 0; i < len_profiles; i++)); do
258253
echo "Process [${i}]: ${docker_compose_profiles[$i]}, ${docker_llm_container_names[${i}]}"
254+
255+
echo "::group::start_services"
259256
start_services "${docker_compose_profiles[${i}]}" "${docker_llm_container_names[${i}]}"
257+
echo "::endgroup::"
260258
docker ps -a
261259

260+
echo "::group::validate_microservices"
262261
validate_microservices "${docker_llm_container_names[${i}]}"
262+
echo "::endgroup::"
263+
264+
echo "::group::validate_megaservice"
263265
validate_megaservice
266+
echo "::endgroup::"
267+
268+
echo "::group::validate_gradio"
264269
validate_gradio
270+
echo "::endgroup::"
265271

266272
stop_docker "${docker_compose_profiles[${i}]}"
267273
sleep 5s
268274
done
269275

270-
echo y | docker system prune
276+
docker system prune -f
271277
}
272278

273279
main

‎CodeGen/tests/test_compose_on_rocm.sh‎

Lines changed: 23 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,13 @@ ip_address=$(hostname -I | awk '{print $1}')
1717

1818
function build_docker_images() {
1919
opea_branch=${opea_branch:-"main"}
20-
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
21-
if [[ "${opea_branch}" != "main" ]]; then
22-
cd $WORKPATH
23-
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
24-
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
25-
find . -type f -name "Dockerfile*" | while read -r file; do
26-
echo "Processing file: $file"
27-
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
28-
done
29-
fi
3020

3121
cd $WORKPATH/docker_image_build
3222
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
23+
pushd GenAIComps
24+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
25+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
26+
popd && sleep 1s
3327

3428
echo "Build all the images with --no-cache, check docker_image_build.log for details..."
3529
service_list="codegen codegen-ui llm-textgen"
@@ -164,18 +158,35 @@ function stop_docker() {
164158

165159
function main() {
166160

161+
echo "::group::stop_docker"
167162
stop_docker
163+
echo "::endgroup::"
168164

165+
echo "::group::build_docker_images"
169166
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
167+
echo "::endgroup::"
168+
169+
echo "::group::start_services"
170170
start_services
171+
echo "::endgroup::"
171172

173+
echo "::group::validate_microservices"
172174
validate_microservices
175+
echo "::endgroup::"
176+
177+
echo "::group::validate_megaservice"
173178
validate_megaservice
179+
echo "::endgroup::"
180+
181+
echo "::group::validate_frontend"
174182
validate_frontend
183+
echo "::endgroup::"
175184

185+
echo "::group::stop_docker"
176186
stop_docker
177-
echo y | docker system prune
178-
cd $WORKPATH
187+
echo "::endgroup::"
188+
189+
docker system prune -f
179190

180191
}
181192

‎CodeGen/tests/test_compose_on_xeon.sh‎

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -27,19 +27,13 @@ export no_proxy=${no_proxy},${ip_address}
2727

2828
function build_docker_images() {
2929
opea_branch=${opea_branch:-"main"}
30-
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
31-
if [[ "${opea_branch}" != "main" ]]; then
32-
cd $WORKPATH
33-
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
34-
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
35-
find . -type f -name "Dockerfile*" | while read -r file; do
36-
echo "Processing file: $file"
37-
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
38-
done
39-
fi
4030

4131
cd $WORKPATH/docker_image_build
4232
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
33+
pushd GenAIComps
34+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
35+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
36+
popd && sleep 1s
4337

4438
git clone https://github.com/vllm-project/vllm.git && cd vllm
4539
VLLM_VER="v0.8.3"
@@ -256,17 +250,28 @@ function main() {
256250
for ((i = 0; i < len_profiles; i++)); do
257251
echo "Process [${i}]: ${docker_compose_profiles[$i]}, ${docker_llm_container_names[${i}]}"
258252
docker ps -a
253+
254+
echo "::group::start_services"
259255
start_services "${docker_compose_profiles[${i}]}" "${docker_llm_container_names[${i}]}"
256+
echo "::endgroup::"
260257

258+
echo "::group::validate_microservices"
261259
validate_microservices "${docker_llm_container_names[${i}]}"
260+
echo "::endgroup::"
261+
262+
echo "::group::validate_megaservice"
262263
validate_megaservice
264+
echo "::endgroup::"
265+
266+
echo "::group::validate_gradio"
263267
validate_gradio
268+
echo "::endgroup::"
264269

265270
stop_docker "${docker_compose_profiles[${i}]}"
266271
sleep 5s
267272
done
268273

269-
echo y | docker system prune
274+
docker system prune -f
270275
}
271276

272277
main

‎CodeGen/tests/test_compose_vllm_on_rocm.sh‎

Lines changed: 24 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,13 @@ ip_address=$(hostname -I | awk '{print $1}')
1717

1818
function build_docker_images() {
1919
opea_branch=${opea_branch:-"main"}
20-
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
21-
if [[ "${opea_branch}" != "main" ]]; then
22-
cd $WORKPATH
23-
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
24-
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
25-
find . -type f -name "Dockerfile*" | while read -r file; do
26-
echo "Processing file: $file"
27-
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
28-
done
29-
fi
3020

3121
cd $WORKPATH/docker_image_build
3222
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
23+
pushd GenAIComps
24+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
25+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
26+
popd && sleep 1s
3327

3428
echo "Build all the images with --no-cache, check docker_image_build.log for details..."
3529
service_list="vllm-rocm llm-textgen codegen codegen-ui"
@@ -164,17 +158,35 @@ function stop_docker() {
164158

165159
function main() {
166160

161+
echo "::group::stop_docker"
167162
stop_docker
163+
echo "::endgroup::"
164+
165+
echo "::group::build_docker_images"
168166
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
167+
echo "::endgroup::"
168+
169+
echo "::group::start_services"
169170
start_services
171+
echo "::endgroup::"
170172

173+
echo "::group::validate_microservices"
171174
validate_microservices
175+
echo "::endgroup::"
176+
177+
echo "::group::validate_megaservice"
172178
validate_megaservice
179+
echo "::endgroup::"
180+
181+
echo "::group::validate_frontend"
173182
validate_frontend
183+
echo "::endgroup::"
174184

185+
echo "::group::stop_docker"
175186
stop_docker
176-
echo y | docker system prune
177-
cd $WORKPATH
187+
echo "::endgroup::"
188+
189+
docker system prune -f
178190

179191
}
180192

0 commit comments

Comments
 (0)