Skip to content

Commit c27d883

Browse files
ZePan110ZePan110
committed
Restore
Signed-off-by: ZePan110 <ze.pan@intel.com>
1 parent 4965282 commit c27d883

1 file changed

Lines changed: 173 additions & 174 deletions

File tree

‎.github/workflows/_run-docker-compose.yml‎

Lines changed: 173 additions & 174 deletions
Original file line numberDiff line numberDiff line change
@@ -70,177 +70,176 @@ jobs:
7070
shell: bash
7171
id: test-case-matrix
7272
run: |
73-
echo "${{ inputs.hardware }}"
74-
# example_l=$(echo ${{ inputs.example }} | tr '[:upper:]' '[:lower:]')
75-
# cd ${{ github.workspace }}/${{ inputs.example }}/tests
76-
# run_test_cases=""
77-
78-
# if [[ "${{ inputs.hardware }}" == "gaudi"* ]]; then
79-
# hardware="gaudi"
80-
# elif [[ "${{ inputs.hardware }}" == "xeon"* ]]; then
81-
# hardware="xeon"
82-
# else
83-
# hardware="${{ inputs.hardware }}"
84-
# fi
85-
# default_test_case=$(find . -type f -name "test_compose_on_$hardware.sh" | cut -d/ -f2)
86-
# if [ "$default_test_case" ]; then run_test_cases="$default_test_case"; fi
87-
# other_test_cases=$(find . -type f -name "test_compose_*_on_$hardware.sh" | cut -d/ -f2)
88-
# echo "default_test_case=$default_test_case"
89-
# echo "other_test_cases=$other_test_cases"
90-
91-
# if [ "${{ inputs.tag }}" == "ci" ]; then
92-
# base_commit=$(curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
93-
# "https://api.github.com/repos/opea-project/GenAIExamples/commits?sha=${{ github.event.pull_request.base.ref }}" | jq -r '.[0].sha')
94-
# merged_commit=$(git log -1 --format='%H')
95-
# changed_files="$(git diff --name-only ${base_commit} ${merged_commit} | grep -vE '${{ inputs.diff_excluded_files }}')" || true
96-
# fi
97-
98-
# for test_case in $other_test_cases; do
99-
# if [ "${{ inputs.tag }}" == "ci" ]; then
100-
# flag=${test_case%_on_*}
101-
# flag=${flag#test_compose_}
102-
# if [[ $(printf '%s\n' "${changed_files[@]}" | grep ${{ inputs.example }} | grep ${flag}) ]]; then
103-
# run_test_cases="$run_test_cases $test_case"
104-
# fi
105-
# else
106-
# run_test_cases="$run_test_cases $test_case"
107-
# fi
108-
# done
109-
110-
# if [ -z "$run_test_cases" ] && [[ $(printf '%s\n' "${changed_files[@]}" | grep ${{ inputs.example }} | grep /tests/) ]]; then
111-
# run_test_cases=$other_test_cases
112-
# fi
113-
114-
# if [[ "${{ hardware }}" == "xeon"* ]]; then
115-
# if [ -f "${{ github.workspace }}/${{ inputs.example }}/tests/test_ui_on_xeon.sh" ]; then
116-
# run_test_cases="$run_test_cases test_ui_on_xeon.sh";
117-
# fi
118-
# fi
119-
# test_cases=$(echo $run_test_cases | tr ' ' '\n' | sort -u | jq -R '.' | jq -sc '.')
120-
# echo "test_cases=$test_cases"
121-
# echo "test_cases=$test_cases" >> $GITHUB_OUTPUT
122-
123-
# compose-test:
124-
# needs: [get-test-case]
125-
# if: ${{ needs.get-test-case.outputs.test_cases != '[""]' }}
126-
# strategy:
127-
# matrix:
128-
# test_case: ${{ fromJSON(needs.get-test-case.outputs.test_cases) }}
129-
# fail-fast: false
130-
# runs-on: ${{ inputs.hardware }}
131-
# continue-on-error: true
132-
# steps:
133-
# - name: Clean up Working Directory
134-
# run: |
135-
# sudo rm -rf ${{github.workspace}}/* || true
136-
137-
# echo "Cleaning up containers using ports..."
138-
# cid=$(docker ps --format '{{.Names}} : {{.Ports}}' | grep -v ' : $' | grep -v 0.0.0.0:5000 | awk -F' : ' '{print $1}')
139-
# if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
140-
# docker system prune -f
141-
142-
# echo "Cleaning up images ..."
143-
# docker images --filter reference="*/*/*:latest" -q | xargs -r docker rmi && sleep 1s
144-
# docker images --filter reference="*/*:ci" -q | xargs -r docker rmi && sleep 1s
145-
# docker images --filter reference="*:5000/*/*" -q | xargs -r docker rmi && sleep 1s
146-
# docker images --filter reference="opea/comps-base" -q | xargs -r docker rmi && sleep 1s
147-
# docker images
148-
149-
# - name: Checkout out Repo
150-
# uses: actions/checkout@v4
151-
# with:
152-
# ref: ${{ needs.get-test-case.outputs.CHECKOUT_REF }}
153-
# fetch-depth: 0
154-
155-
# - name: Clean up container before test
156-
# shell: bash
157-
# run: |
158-
# docker ps
159-
# cd ${{ github.workspace }}/${{ inputs.example }}
160-
# export test_case=${{ matrix.test_case }}
161-
# export hardware=${{ inputs.hardware }}
162-
# bash ${{ github.workspace }}/.github/workflows/scripts/docker_compose_clean_up.sh "containers"
163-
# bash ${{ github.workspace }}/.github/workflows/scripts/docker_compose_clean_up.sh "ports"
164-
# docker ps
165-
166-
# - name: Log in DockerHub
167-
# uses: docker/login-action@v3.2.0
168-
# with:
169-
# username: ${{ secrets.DOCKERHUB_USER }}
170-
# password: ${{ secrets.DOCKERHUB_TOKEN }}
171-
172-
# - name: Run test
173-
# shell: bash
174-
# env:
175-
# HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HF_TOKEN }}
176-
# HF_TOKEN: ${{ secrets.HF_TOKEN }}
177-
# GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }}
178-
# GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
179-
# PINECONE_KEY: ${{ secrets.PINECONE_KEY }}
180-
# PINECONE_KEY_LANGCHAIN_TEST: ${{ secrets.PINECONE_KEY_LANGCHAIN_TEST }}
181-
# SDK_BASE_URL: ${{ secrets.SDK_BASE_URL }}
182-
# SERVING_TOKEN: ${{ secrets.SERVING_TOKEN }}
183-
# OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
184-
# FINNHUB_API_KEY: ${{ secrets.FINNHUB_API_KEY }}
185-
# FINANCIAL_DATASETS_API_KEY: ${{ secrets.FINANCIAL_DATASETS_API_KEY }}
186-
# TAVILY_API_KEY: ${{ secrets.TAVILY_API_KEY }}
187-
# IMAGE_REPO: ${{ inputs.registry }}
188-
# IMAGE_TAG: ${{ inputs.tag }}
189-
# opea_branch: ${{ inputs.opea_branch }}
190-
# example: ${{ inputs.example }}
191-
# hardware: ${{ inputs.hardware }}
192-
# test_case: ${{ matrix.test_case }}
193-
# use_model_cache: ${{ inputs.use_model_cache }}
194-
# run: |
195-
# cd ${{ github.workspace }}/$example/tests
196-
# if [[ "$IMAGE_REPO" == "" ]]; then export IMAGE_REPO="${OPEA_IMAGE_REPO}opea"; fi
197-
# if [[ "$use_model_cache" == "true" ]]; then
198-
# if [ -d "/data2/hf_model" ]; then
199-
# export model_cache="/data2/hf_model"
200-
# else
201-
# echo "Model cache directory /data2/hf_model does not exist"
202-
# export model_cache="$HOME/.cache/huggingface/hub"
203-
# fi
204-
# if [[ "$test_case" == *"rocm"* ]]; then
205-
# export model_cache="/var/lib/GenAI/data"
206-
# fi
207-
# fi
208-
# if [ -f "${test_case}" ]; then timeout 60m bash "${test_case}"; else echo "Test script {${test_case}} not found, skip test!"; fi
209-
210-
# - name: Clean up container after test
211-
# if: always()
212-
# run: |
213-
# set -x
214-
215-
# echo "Cleaning up containers using ports..."
216-
# cid=$(docker ps --format '{{.Names}} : {{.Ports}}' | grep -v ' : $' | grep -v 0.0.0.0:5000 | awk -F' : ' '{print $1}')
217-
# if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
218-
219-
# echo "Cleaning up images ..."
220-
# df -h
221-
# sleep 1
222-
# docker system df
223-
# sleep 1
224-
# if [[ "${{ inputs.hardware }}" == "xeon"* ]]; then
225-
# docker system prune -a -f
226-
# else
227-
# docker images --filter reference="*/*/*:latest" -q | xargs -r docker rmi && sleep 1s
228-
# docker images --filter reference="*/*:ci" -q | xargs -r docker rmi && sleep 1s
229-
# docker images --filter reference="*:5000/*/*" -q | xargs -r docker rmi && sleep 1s
230-
# docker images --filter reference="opea/comps-base" -q | xargs -r docker rmi && sleep 1s
231-
# docker system prune -f
232-
# fi
233-
# sleep 5
234-
# docker images
235-
# sleep 1
236-
# df -h
237-
# sleep 1
238-
# docker system df
239-
# sleep 1
240-
241-
# - name: Publish pipeline artifact
242-
# if: ${{ !cancelled() }}
243-
# uses: actions/upload-artifact@v4
244-
# with:
245-
# name: ${{ inputs.hardware }}_${{ inputs.example }}_${{ matrix.test_case }}
246-
# path: ${{ github.workspace }}/${{ inputs.example }}/tests/*.log
73+
example_l=$(echo ${{ inputs.example }} | tr '[:upper:]' '[:lower:]')
74+
cd ${{ github.workspace }}/${{ inputs.example }}/tests
75+
run_test_cases=""
76+
77+
if [[ "${{ inputs.hardware }}" == "gaudi"* ]]; then
78+
hardware="gaudi"
79+
elif [[ "${{ inputs.hardware }}" == "xeon"* ]]; then
80+
hardware="xeon"
81+
else
82+
hardware="${{ inputs.hardware }}"
83+
fi
84+
default_test_case=$(find . -type f -name "test_compose_on_$hardware.sh" | cut -d/ -f2)
85+
if [ "$default_test_case" ]; then run_test_cases="$default_test_case"; fi
86+
other_test_cases=$(find . -type f -name "test_compose_*_on_$hardware.sh" | cut -d/ -f2)
87+
echo "default_test_case=$default_test_case"
88+
echo "other_test_cases=$other_test_cases"
89+
90+
if [ "${{ inputs.tag }}" == "ci" ]; then
91+
base_commit=$(curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
92+
"https://api.github.com/repos/opea-project/GenAIExamples/commits?sha=${{ github.event.pull_request.base.ref }}" | jq -r '.[0].sha')
93+
merged_commit=$(git log -1 --format='%H')
94+
changed_files="$(git diff --name-only ${base_commit} ${merged_commit} | grep -vE '${{ inputs.diff_excluded_files }}')" || true
95+
fi
96+
97+
for test_case in $other_test_cases; do
98+
if [ "${{ inputs.tag }}" == "ci" ]; then
99+
flag=${test_case%_on_*}
100+
flag=${flag#test_compose_}
101+
if [[ $(printf '%s\n' "${changed_files[@]}" | grep ${{ inputs.example }} | grep ${flag}) ]]; then
102+
run_test_cases="$run_test_cases $test_case"
103+
fi
104+
else
105+
run_test_cases="$run_test_cases $test_case"
106+
fi
107+
done
108+
109+
if [ -z "$run_test_cases" ] && [[ $(printf '%s\n' "${changed_files[@]}" | grep ${{ inputs.example }} | grep /tests/) ]]; then
110+
run_test_cases=$other_test_cases
111+
fi
112+
113+
if [[ "${{ hardware }}" == "xeon"* ]]; then
114+
if [ -f "${{ github.workspace }}/${{ inputs.example }}/tests/test_ui_on_xeon.sh" ]; then
115+
run_test_cases="$run_test_cases test_ui_on_xeon.sh";
116+
fi
117+
fi
118+
test_cases=$(echo $run_test_cases | tr ' ' '\n' | sort -u | jq -R '.' | jq -sc '.')
119+
echo "test_cases=$test_cases"
120+
echo "test_cases=$test_cases" >> $GITHUB_OUTPUT
121+
122+
compose-test:
123+
needs: [get-test-case]
124+
if: ${{ needs.get-test-case.outputs.test_cases != '[""]' }}
125+
strategy:
126+
matrix:
127+
test_case: ${{ fromJSON(needs.get-test-case.outputs.test_cases) }}
128+
fail-fast: false
129+
runs-on: ${{ inputs.hardware }}
130+
continue-on-error: true
131+
steps:
132+
- name: Clean up Working Directory
133+
run: |
134+
sudo rm -rf ${{github.workspace}}/* || true
135+
136+
echo "Cleaning up containers using ports..."
137+
cid=$(docker ps --format '{{.Names}} : {{.Ports}}' | grep -v ' : $' | grep -v 0.0.0.0:5000 | awk -F' : ' '{print $1}')
138+
if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
139+
docker system prune -f
140+
141+
echo "Cleaning up images ..."
142+
docker images --filter reference="*/*/*:latest" -q | xargs -r docker rmi && sleep 1s
143+
docker images --filter reference="*/*:ci" -q | xargs -r docker rmi && sleep 1s
144+
docker images --filter reference="*:5000/*/*" -q | xargs -r docker rmi && sleep 1s
145+
docker images --filter reference="opea/comps-base" -q | xargs -r docker rmi && sleep 1s
146+
docker images
147+
148+
- name: Checkout out Repo
149+
uses: actions/checkout@v4
150+
with:
151+
ref: ${{ needs.get-test-case.outputs.CHECKOUT_REF }}
152+
fetch-depth: 0
153+
154+
- name: Clean up container before test
155+
shell: bash
156+
run: |
157+
docker ps
158+
cd ${{ github.workspace }}/${{ inputs.example }}
159+
export test_case=${{ matrix.test_case }}
160+
export hardware=${{ inputs.hardware }}
161+
bash ${{ github.workspace }}/.github/workflows/scripts/docker_compose_clean_up.sh "containers"
162+
bash ${{ github.workspace }}/.github/workflows/scripts/docker_compose_clean_up.sh "ports"
163+
docker ps
164+
165+
- name: Log in DockerHub
166+
uses: docker/login-action@v3.2.0
167+
with:
168+
username: ${{ secrets.DOCKERHUB_USER }}
169+
password: ${{ secrets.DOCKERHUB_TOKEN }}
170+
171+
- name: Run test
172+
shell: bash
173+
env:
174+
HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HF_TOKEN }}
175+
HF_TOKEN: ${{ secrets.HF_TOKEN }}
176+
GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }}
177+
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
178+
PINECONE_KEY: ${{ secrets.PINECONE_KEY }}
179+
PINECONE_KEY_LANGCHAIN_TEST: ${{ secrets.PINECONE_KEY_LANGCHAIN_TEST }}
180+
SDK_BASE_URL: ${{ secrets.SDK_BASE_URL }}
181+
SERVING_TOKEN: ${{ secrets.SERVING_TOKEN }}
182+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
183+
FINNHUB_API_KEY: ${{ secrets.FINNHUB_API_KEY }}
184+
FINANCIAL_DATASETS_API_KEY: ${{ secrets.FINANCIAL_DATASETS_API_KEY }}
185+
TAVILY_API_KEY: ${{ secrets.TAVILY_API_KEY }}
186+
IMAGE_REPO: ${{ inputs.registry }}
187+
IMAGE_TAG: ${{ inputs.tag }}
188+
opea_branch: ${{ inputs.opea_branch }}
189+
example: ${{ inputs.example }}
190+
hardware: ${{ inputs.hardware }}
191+
test_case: ${{ matrix.test_case }}
192+
use_model_cache: ${{ inputs.use_model_cache }}
193+
run: |
194+
cd ${{ github.workspace }}/$example/tests
195+
if [[ "$IMAGE_REPO" == "" ]]; then export IMAGE_REPO="${OPEA_IMAGE_REPO}opea"; fi
196+
if [[ "$use_model_cache" == "true" ]]; then
197+
if [ -d "/data2/hf_model" ]; then
198+
export model_cache="/data2/hf_model"
199+
else
200+
echo "Model cache directory /data2/hf_model does not exist"
201+
export model_cache="$HOME/.cache/huggingface/hub"
202+
fi
203+
if [[ "$test_case" == *"rocm"* ]]; then
204+
export model_cache="/var/lib/GenAI/data"
205+
fi
206+
fi
207+
if [ -f "${test_case}" ]; then timeout 60m bash "${test_case}"; else echo "Test script {${test_case}} not found, skip test!"; fi
208+
209+
- name: Clean up container after test
210+
if: always()
211+
run: |
212+
set -x
213+
214+
echo "Cleaning up containers using ports..."
215+
cid=$(docker ps --format '{{.Names}} : {{.Ports}}' | grep -v ' : $' | grep -v 0.0.0.0:5000 | awk -F' : ' '{print $1}')
216+
if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
217+
218+
echo "Cleaning up images ..."
219+
df -h
220+
sleep 1
221+
docker system df
222+
sleep 1
223+
if [[ "${{ inputs.hardware }}" == "xeon"* ]]; then
224+
docker system prune -a -f
225+
else
226+
docker images --filter reference="*/*/*:latest" -q | xargs -r docker rmi && sleep 1s
227+
docker images --filter reference="*/*:ci" -q | xargs -r docker rmi && sleep 1s
228+
docker images --filter reference="*:5000/*/*" -q | xargs -r docker rmi && sleep 1s
229+
docker images --filter reference="opea/comps-base" -q | xargs -r docker rmi && sleep 1s
230+
docker system prune -f
231+
fi
232+
sleep 5
233+
docker images
234+
sleep 1
235+
df -h
236+
sleep 1
237+
docker system df
238+
sleep 1
239+
240+
- name: Publish pipeline artifact
241+
if: ${{ !cancelled() }}
242+
uses: actions/upload-artifact@v4
243+
with:
244+
name: ${{ inputs.hardware }}_${{ inputs.example }}_${{ matrix.test_case }}
245+
path: ${{ github.workspace }}/${{ inputs.example }}/tests/*.log

0 commit comments

Comments
 (0)