Skip to content

Commit 7ee6f36

Browse files
chensuyuechensuyue
authored
[CICD enhance] DocIndexRetriever run CI with latest base image, group logs in GHA outputs. (#1932)
Signed-off-by: chensuyue <suyue.chen@intel.com>
1 parent 11b04b3 commit 7ee6f36

7 files changed

Lines changed: 102 additions & 56 deletions

File tree

‎DocIndexRetriever/Dockerfile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
# Copyright (C) 2024 Intel Corporation
22
# SPDX-License-Identifier: Apache-2.0
33

4+
ARG IMAGE_REPO=opea
45
ARG BASE_TAG=latest
5-
FROM opea/comps-base:$BASE_TAG
6+
FROM $IMAGE_REPO/comps-base:$BASE_TAG
67

78
COPY ./retrieval_tool.py $HOME/retrieval_tool.py
89

‎DocIndexRetriever/docker_image_build/build.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ services:
55
doc-index-retriever:
66
build:
77
args:
8+
IMAGE_REPO: ${REGISTRY:-opea}
9+
BASE_TAG: ${TAG:-latest}
810
http_proxy: ${http_proxy}
911
https_proxy: ${https_proxy}
1012
no_proxy: ${no_proxy}

‎DocIndexRetriever/tests/test_compose_milvus_on_gaudi.sh‎

Lines changed: 20 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -20,17 +20,15 @@ function build_docker_images() {
2020
if [ ! -d "GenAIComps" ] ; then
2121
git clone --single-branch --branch "${opea_branch:-"main"}" https://github.com/opea-project/GenAIComps.git
2222
fi
23+
pushd GenAIComps
24+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
25+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
26+
popd && sleep 1s
27+
2328
service_list="dataprep embedding retriever reranking doc-index-retriever"
2429
docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
2530

26-
docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
27-
docker pull ghcr.io/huggingface/tei-gaudi:1.5.0
28-
docker pull quay.io/coreos/etcd:v3.5.5
29-
docker pull minio/minio:RELEASE.2023-03-20T20-16-18Z
30-
docker pull milvusdb/milvus:v2.4.6
3131
docker images && sleep 1s
32-
33-
echo "Docker images built!"
3432
}
3533

3634
function start_services() {
@@ -112,19 +110,27 @@ function stop_docker() {
112110

113111
function main() {
114112

113+
echo "::group::stop_docker"
115114
stop_docker
115+
echo "::endgroup::"
116+
117+
echo "::group::build_docker_images"
116118
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
117-
echo "Dump current docker ps"
118-
docker ps
119-
start_time=$(date +%s)
119+
echo "::endgroup::"
120+
121+
echo "::group::start_services"
120122
start_services
121-
end_time=$(date +%s)
122-
duration=$((end_time-start_time))
123-
echo "Mega service start duration is $duration s"
123+
echo "::endgroup::"
124+
125+
echo "::group::validate_megaservice"
124126
validate_megaservice
127+
echo "::endgroup::"
125128

129+
echo "::group::stop_docker"
126130
stop_docker
127-
echo y | docker system prune
131+
echo "::endgroup::"
132+
133+
docker system prune -f
128134

129135
}
130136

‎DocIndexRetriever/tests/test_compose_milvus_on_xeon.sh‎

Lines changed: 20 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,16 +20,15 @@ function build_docker_images() {
2020
if [ ! -d "GenAIComps" ] ; then
2121
git clone --single-branch --branch "${opea_branch:-"main"}" https://github.com/opea-project/GenAIComps.git
2222
fi
23+
pushd GenAIComps
24+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
25+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
26+
popd && sleep 1s
27+
2328
service_list="dataprep embedding retriever reranking doc-index-retriever"
2429
docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
2530

26-
docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
27-
docker pull quay.io/coreos/etcd:v3.5.5
28-
docker pull minio/minio:RELEASE.2023-03-20T20-16-18Z
29-
docker pull milvusdb/milvus:v2.4.6
3031
docker images && sleep 1s
31-
32-
echo "Docker images built!"
3332
}
3433

3534
function start_services() {
@@ -111,19 +110,27 @@ function stop_docker() {
111110

112111
function main() {
113112

113+
echo "::group::stop_docker"
114114
stop_docker
115+
echo "::endgroup::"
116+
117+
echo "::group::build_docker_images"
115118
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
116-
echo "Dump current docker ps"
117-
docker ps
118-
start_time=$(date +%s)
119+
echo "::endgroup::"
120+
121+
echo "::group::start_services"
119122
start_services
120-
end_time=$(date +%s)
121-
duration=$((end_time-start_time))
122-
echo "Mega service start duration is $duration s"
123+
echo "::endgroup::"
124+
125+
echo "::group::validate_megaservice"
123126
validate_megaservice
127+
echo "::endgroup::"
124128

129+
echo "::group::stop_docker"
125130
stop_docker
126-
echo y | docker system prune
131+
echo "::endgroup::"
132+
133+
docker system prune -f
127134

128135
}
129136

‎DocIndexRetriever/tests/test_compose_on_gaudi.sh‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,15 @@ function build_docker_images() {
2121
if [ ! -d "GenAIComps" ] ; then
2222
git clone --single-branch --branch "${opea_branch:-"main"}" https://github.com/opea-project/GenAIComps.git
2323
fi
24+
pushd GenAIComps
25+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
26+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
27+
popd && sleep 1s
2428

2529
echo "Build all the images with --no-cache, check docker_image_build.log for details..."
2630
docker compose -f build.yaml build --no-cache > ${LOG_PATH}/docker_image_build.log
2731

28-
docker pull redis/redis-stack:7.2.0-v9
29-
docker pull ghcr.io/huggingface/tei-gaudi:1.5.0
3032
docker images && sleep 1s
31-
echo "Docker images built!"
3233
}
3334

3435
function start_services() {
@@ -103,19 +104,27 @@ function stop_docker() {
103104

104105
function main() {
105106

107+
echo "::group::stop_docker"
106108
stop_docker
109+
echo "::endgroup::"
110+
111+
echo "::group::build_docker_images"
107112
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
108-
echo "Dump current docker ps"
109-
docker ps
110-
start_time=$(date +%s)
113+
echo "::endgroup::"
114+
115+
echo "::group::start_services"
111116
start_services
112-
end_time=$(date +%s)
113-
duration=$((end_time-start_time))
114-
echo "Mega service start duration is $duration s"
117+
echo "::endgroup::"
118+
119+
echo "::group::validate_megaservice"
115120
validate_megaservice
121+
echo "::endgroup::"
116122

123+
echo "::group::stop_docker"
117124
stop_docker
118-
echo y | docker system prune
125+
echo "::endgroup::"
126+
127+
docker system prune -f
119128

120129
}
121130

‎DocIndexRetriever/tests/test_compose_on_xeon.sh‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,15 @@ function build_docker_images() {
2121
if [ ! -d "GenAIComps" ] ; then
2222
git clone --single-branch --branch "${opea_branch:-"main"}" https://github.com/opea-project/GenAIComps.git
2323
fi
24+
pushd GenAIComps
25+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
26+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
27+
popd && sleep 1s
28+
2429
service_list="dataprep embedding retriever reranking doc-index-retriever"
2530
docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
2631

27-
docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
28-
docker pull redis/redis-stack:7.2.0-v9
2932
docker images && sleep 1s
30-
31-
echo "Docker images built!"
3233
}
3334

3435
function start_services() {
@@ -110,20 +111,27 @@ function stop_docker() {
110111

111112
function main() {
112113

114+
echo "::group::stop_docker"
113115
stop_docker
116+
echo "::endgroup::"
117+
118+
echo "::group::build_docker_images"
114119
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
115-
echo "Dump current docker ps"
116-
docker ps
120+
echo "::endgroup::"
117121

118-
start_time=$(date +%s)
122+
echo "::group::start_services"
119123
start_services
120-
end_time=$(date +%s)
121-
duration=$((end_time-start_time))
122-
echo "Mega service start duration is $duration s"
124+
echo "::endgroup::"
125+
126+
echo "::group::validate_megaservice"
123127
validate_megaservice
128+
echo "::endgroup::"
124129

130+
echo "::group::stop_docker"
125131
stop_docker
126-
echo y | docker system prune
132+
echo "::endgroup::"
133+
134+
docker system prune -f
127135

128136
}
129137

‎DocIndexRetriever/tests/test_compose_without_rerank_on_xeon.sh‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,11 @@ function build_docker_images() {
2121
if [ ! -d "GenAIComps" ] ; then
2222
git clone --single-branch --branch "${opea_branch:-"main"}" https://github.com/opea-project/GenAIComps.git
2323
fi
24+
pushd GenAIComps
25+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
26+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
27+
popd && sleep 1s
28+
2429
service_list="dataprep embedding retriever doc-index-retriever"
2530
docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
2631

@@ -114,19 +119,27 @@ function stop_docker() {
114119

115120
function main() {
116121

122+
echo "::group::stop_docker"
117123
stop_docker
124+
echo "::endgroup::"
125+
126+
echo "::group::build_docker_images"
118127
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
119-
echo "Dump current docker ps"
120-
docker ps
121-
start_time=$(date +%s)
128+
echo "::endgroup::"
129+
130+
echo "::group::start_services"
122131
start_services
123-
end_time=$(date +%s)
124-
duration=$((end_time-start_time))
125-
echo "Mega service start duration is $duration s"
132+
echo "::endgroup::"
133+
134+
echo "::group::validate_megaservice"
126135
validate_megaservice
136+
echo "::endgroup::"
127137

138+
echo "::group::stop_docker"
128139
stop_docker
129-
echo y | docker system prune
140+
echo "::endgroup::"
141+
142+
docker system prune -f
130143

131144
}
132145

0 commit comments

Comments
 (0)