Skip to content

Commit 26d0701

Browse files
chensuyuechensuyue
authored
[CICD enhance] CodeTrans run CI with latest base image, group logs in GHA outputs. (#1929)
Signed-off-by: chensuyue <suyue.chen@intel.com>
1 parent bd6726c commit 26d0701

8 files changed

Lines changed: 145 additions & 67 deletions

‎CodeTrans/Dockerfile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
# Copyright (C) 2024 Intel Corporation
22
# SPDX-License-Identifier: Apache-2.0
33

4+
ARG IMAGE_REPO=opea
45
ARG BASE_TAG=latest
5-
FROM opea/comps-base:$BASE_TAG
6+
FROM $IMAGE_REPO/comps-base:$BASE_TAG
67

78
COPY ./code_translation.py $HOME/code_translation.py
89

‎CodeTrans/docker_image_build/build.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ services:
55
codetrans:
66
build:
77
args:
8+
IMAGE_REPO: ${REGISTRY:-opea}
9+
BASE_TAG: ${TAG:-latest}
810
http_proxy: ${http_proxy}
911
https_proxy: ${https_proxy}
1012
no_proxy: ${no_proxy}
@@ -45,4 +47,5 @@ services:
4547
build:
4648
context: GenAIComps
4749
dockerfile: comps/third_parties/vllm/src/Dockerfile.amd_gpu
50+
extends: codetrans
4851
image: ${REGISTRY:-opea}/vllm-rocm:${TAG:-latest}

‎CodeTrans/tests/test_compose_on_gaudi.sh‎

Lines changed: 24 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,14 @@ ip_address=$(hostname -I | awk '{print $1}')
1717

1818
function build_docker_images() {
1919
opea_branch=${opea_branch:-"main"}
20-
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
21-
if [[ "${opea_branch}" != "main" ]]; then
22-
cd $WORKPATH
23-
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
24-
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
25-
find . -type f -name "Dockerfile*" | while read -r file; do
26-
echo "Processing file: $file"
27-
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
28-
done
29-
fi
3020

3121
cd $WORKPATH/docker_image_build
3222
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
23+
pushd GenAIComps
24+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
25+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
26+
popd && sleep 1s
27+
3328
git clone https://github.com/HabanaAI/vllm-fork.git && cd vllm-fork
3429
VLLM_FORK_VER=v0.6.6.post1+Gaudi-1.20.0
3530
git checkout ${VLLM_FORK_VER} &> /dev/null && cd ../
@@ -160,17 +155,35 @@ function stop_docker() {
160155

161156
function main() {
162157

158+
echo "::group::stop_docker"
163159
stop_docker
160+
echo "::endgroup::"
164161

162+
echo "::group::build_docker_images"
165163
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
164+
echo "::endgroup::"
165+
166+
echo "::group::start_services"
166167
start_services
168+
echo "::endgroup::"
167169

170+
echo "::group::validate_microservices"
168171
validate_microservices
172+
echo "::endgroup::"
173+
174+
echo "::group::validate_megaservice"
169175
validate_megaservice
176+
echo "::endgroup::"
177+
178+
echo "::group::validate_frontend"
170179
validate_frontend
180+
echo "::endgroup::"
171181

182+
echo "::group::stop_docker"
172183
stop_docker
173-
echo y | docker system prune
184+
echo "::endgroup::"
185+
186+
docker system prune -f
174187

175188
}
176189

‎CodeTrans/tests/test_compose_on_rocm.sh‎

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -18,19 +18,13 @@ ip_address=$(hostname -I | awk '{print $1}')
1818

1919
function build_docker_images() {
2020
opea_branch=${opea_branch:-"main"}
21-
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
22-
if [[ "${opea_branch}" != "main" ]]; then
23-
cd $WORKPATH
24-
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
25-
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
26-
find . -type f -name "Dockerfile*" | while read -r file; do
27-
echo "Processing file: $file"
28-
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
29-
done
30-
fi
3121

3222
cd $WORKPATH/docker_image_build
3323
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
24+
pushd GenAIComps
25+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
26+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
27+
popd && sleep 1s
3428

3529
echo "Build all the images with --no-cache, check docker_image_build.log for details..."
3630
service_list="codetrans codetrans-ui llm-textgen nginx"
@@ -161,17 +155,35 @@ function stop_docker() {
161155

162156
function main() {
163157

158+
echo "::group::stop_docker"
164159
stop_docker
160+
echo "::endgroup::"
165161

162+
echo "::group::build_docker_images"
166163
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
164+
echo "::endgroup::"
165+
166+
echo "::group::start_services"
167167
start_services
168+
echo "::endgroup::"
168169

170+
echo "::group::validate_microservices"
169171
validate_microservices
172+
echo "::endgroup::"
173+
174+
echo "::group::validate_megaservice"
170175
validate_megaservice
176+
echo "::endgroup::"
177+
178+
echo "::group::validate_frontend"
171179
validate_frontend
180+
echo "::endgroup::"
172181

182+
echo "::group::stop_docker"
173183
stop_docker
174-
echo y | docker system prune
184+
echo "::endgroup::"
185+
186+
docker system prune -f
175187

176188
}
177189

‎CodeTrans/tests/test_compose_on_xeon.sh‎

Lines changed: 24 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,14 @@ ip_address=$(hostname -I | awk '{print $1}')
1717

1818
function build_docker_images() {
1919
opea_branch=${opea_branch:-"main"}
20-
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
21-
if [[ "${opea_branch}" != "main" ]]; then
22-
cd $WORKPATH
23-
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
24-
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
25-
find . -type f -name "Dockerfile*" | while read -r file; do
26-
echo "Processing file: $file"
27-
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
28-
done
29-
fi
3020

3121
cd $WORKPATH/docker_image_build
3222
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
23+
pushd GenAIComps
24+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
25+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
26+
popd && sleep 1s
27+
3328
git clone https://github.com/vllm-project/vllm.git && cd vllm
3429
VLLM_VER="v0.8.3"
3530
echo "Check out vLLM tag ${VLLM_VER}"
@@ -163,17 +158,35 @@ function stop_docker() {
163158

164159
function main() {
165160

161+
echo "::group::stop_docker"
166162
stop_docker
163+
echo "::endgroup::"
167164

165+
echo "::group::build_docker_images"
168166
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
167+
echo "::endgroup::"
168+
169+
echo "::group::start_services"
169170
start_services
171+
echo "::endgroup::"
170172

173+
echo "::group::validate_microservices"
171174
validate_microservices
175+
echo "::endgroup::"
176+
177+
echo "::group::validate_megaservice"
172178
validate_megaservice
179+
echo "::endgroup::"
180+
181+
echo "::group::validate_frontend"
173182
validate_frontend
183+
echo "::endgroup::"
174184

185+
echo "::group::stop_docker"
175186
stop_docker
176-
echo y | docker system prune
187+
echo "::endgroup::"
188+
189+
docker system prune -f
177190

178191
}
179192

‎CodeTrans/tests/test_compose_tgi_on_gaudi.sh‎

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,13 @@ ip_address=$(hostname -I | awk '{print $1}')
1717

1818
function build_docker_images() {
1919
opea_branch=${opea_branch:-"main"}
20-
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
21-
if [[ "${opea_branch}" != "main" ]]; then
22-
cd $WORKPATH
23-
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
24-
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
25-
find . -type f -name "Dockerfile*" | while read -r file; do
26-
echo "Processing file: $file"
27-
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
28-
done
29-
fi
3020

3121
cd $WORKPATH/docker_image_build
3222
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
23+
pushd GenAIComps
24+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
25+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
26+
popd && sleep 1s
3327

3428
echo "Build all the images with --no-cache, check docker_image_build.log for details..."
3529
service_list="codetrans codetrans-ui llm-textgen nginx"
@@ -167,17 +161,35 @@ function stop_docker() {
167161

168162
function main() {
169163

164+
echo "::group::stop_docker"
170165
stop_docker
166+
echo "::endgroup::"
171167

168+
echo "::group::build_docker_images"
172169
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
170+
echo "::endgroup::"
171+
172+
echo "::group::start_services"
173173
start_services
174+
echo "::endgroup::"
174175

176+
echo "::group::validate_microservices"
175177
validate_microservices
178+
echo "::endgroup::"
179+
180+
echo "::group::validate_megaservice"
176181
validate_megaservice
182+
echo "::endgroup::"
183+
184+
echo "::group::validate_frontend"
177185
validate_frontend
186+
echo "::endgroup::"
178187

188+
echo "::group::stop_docker"
179189
stop_docker
180-
echo y | docker system prune
190+
echo "::endgroup::"
191+
192+
docker system prune -f
181193

182194
}
183195

‎CodeTrans/tests/test_compose_tgi_on_xeon.sh‎

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,13 @@ ip_address=$(hostname -I | awk '{print $1}')
1717

1818
function build_docker_images() {
1919
opea_branch=${opea_branch:-"main"}
20-
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
21-
if [[ "${opea_branch}" != "main" ]]; then
22-
cd $WORKPATH
23-
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
24-
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
25-
find . -type f -name "Dockerfile*" | while read -r file; do
26-
echo "Processing file: $file"
27-
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
28-
done
29-
fi
3020

3121
cd $WORKPATH/docker_image_build
3222
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
23+
pushd GenAIComps
24+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
25+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
26+
popd && sleep 1s
3327

3428
echo "Build all the images with --no-cache, check docker_image_build.log for details..."
3529
service_list="codetrans codetrans-ui llm-textgen nginx"
@@ -167,17 +161,35 @@ function stop_docker() {
167161

168162
function main() {
169163

164+
echo "::group::stop_docker"
170165
stop_docker
166+
echo "::endgroup::"
171167

168+
echo "::group::build_docker_images"
172169
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
170+
echo "::endgroup::"
171+
172+
echo "::group::start_services"
173173
start_services
174+
echo "::endgroup::"
174175

176+
echo "::group::validate_microservices"
175177
validate_microservices
178+
echo "::endgroup::"
179+
180+
echo "::group::validate_megaservice"
176181
validate_megaservice
182+
echo "::endgroup::"
183+
184+
echo "::group::validate_frontend"
177185
validate_frontend
186+
echo "::endgroup::"
178187

188+
echo "::group::stop_docker"
179189
stop_docker
180-
echo y | docker system prune
190+
echo "::endgroup::"
191+
192+
docker system prune -f
181193

182194
}
183195

‎CodeTrans/tests/test_compose_vllm_on_rocm.sh‎

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,13 @@ ip_address=$(hostname -I | awk '{print $1}')
1717

1818
function build_docker_images() {
1919
opea_branch=${opea_branch:-"main"}
20-
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
21-
if [[ "${opea_branch}" != "main" ]]; then
22-
cd $WORKPATH
23-
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
24-
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
25-
find . -type f -name "Dockerfile*" | while read -r file; do
26-
echo "Processing file: $file"
27-
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
28-
done
29-
fi
3020

3121
cd $WORKPATH/docker_image_build
3222
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
23+
pushd GenAIComps
24+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
25+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
26+
popd && sleep 1s
3327

3428
echo "Build all the images with --no-cache, check docker_image_build.log for details..."
3529
service_list="vllm-rocm llm-textgen codetrans codetrans-ui nginx"
@@ -160,17 +154,35 @@ function stop_docker() {
160154

161155
function main() {
162156

157+
echo "::group::stop_docker"
163158
stop_docker
159+
echo "::endgroup::"
164160

161+
echo "::group::build_docker_images"
165162
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
163+
echo "::endgroup::"
164+
165+
echo "::group::start_services"
166166
start_services
167+
echo "::endgroup::"
167168

169+
echo "::group::validate_microservices"
168170
validate_microservices
171+
echo "::endgroup::"
172+
173+
echo "::group::validate_megaservice"
169174
validate_megaservice
175+
echo "::endgroup::"
176+
177+
echo "::group::validate_frontend"
170178
validate_frontend
179+
echo "::endgroup::"
171180

181+
echo "::group::stop_docker"
172182
stop_docker
173-
echo y | docker system prune
183+
echo "::endgroup::"
184+
185+
docker system prune -f
174186

175187
}
176188

0 commit comments

Comments
 (0)