Skip to content

Commit c535d8e

Browse files
luarssluarss
authored
ci: parallelise secret CI, cache HF dataset, speed up docker build (The-OpenROAD-Project#291)
- Split monolithic build-backend-docker job into lint-backend, lint-frontend, lint-evaluation (ubuntu-latest), test, and docker-eval jobs; lint jobs run free runners in parallel - Remove redundant HF source_list.json download from test step - Add docker-compose.ci.yml override: skips HF clone via SKIP_HF_DOWNLOAD build arg, bind-mounts pre-downloaded ./data, reduces healthcheck start_period from 1200s to 300s - Add docker-up-ci / docker-down-ci Makefile targets using the CI compose override - Use pytest -n auto in make test to parallelise 349 tests via already-installed pytest-xdist - Add SKIP_HF_DOWNLOAD ARG to backend/Dockerfile so production builds still clone the dataset; CI skips it and mounts instead - Change uv sync --dev to uv sync in Dockerfile to omit dev tools from the production image Signed-off-by: Jack Luar <jluar@precisioninno.com>
1 parent d2fde4d commit c535d8e

5 files changed

Lines changed: 109 additions & 17 deletions

File tree

‎.github/workflows/ci-secret.yaml‎

Lines changed: 77 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,63 @@ on:
1111
- 'evaluation/**'
1212
- 'Makefile'
1313
- 'docker-compose.yml'
14+
- 'docker-compose.ci.yml'
1415

1516
defaults:
1617
run:
1718
shell: bash
1819

1920
jobs:
20-
build-backend-docker:
21+
lint-backend:
22+
runs-on: ubuntu-latest
23+
steps:
24+
- name: Setup python
25+
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
26+
with:
27+
python-version: '3.12'
28+
- name: Install uv
29+
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
30+
- name: Checkout code
31+
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
32+
- name: Install deps
33+
run: cd backend && make init-dev
34+
- name: Lint
35+
run: cd backend && make check
36+
37+
lint-frontend:
38+
runs-on: ubuntu-latest
39+
steps:
40+
- name: Setup python
41+
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
42+
with:
43+
python-version: '3.12'
44+
- name: Install uv
45+
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
46+
- name: Checkout code
47+
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
48+
- name: Install deps
49+
run: cd frontend && make init-dev
50+
- name: Lint
51+
run: cd frontend && make check
52+
53+
lint-evaluation:
54+
runs-on: ubuntu-latest
55+
steps:
56+
- name: Setup python
57+
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
58+
with:
59+
python-version: '3.12'
60+
- name: Install uv
61+
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
62+
- name: Checkout code
63+
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
64+
- name: Install deps
65+
run: cd evaluation && make init-dev
66+
- name: Lint
67+
run: cd evaluation && make check
68+
69+
test:
70+
needs: [lint-backend]
2171
runs-on: self-hosted
2272
steps:
2373
- name: Setup python
@@ -28,19 +78,32 @@ jobs:
2878
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
2979
- name: Checkout code
3080
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
31-
- name: Setup prereqs
32-
run: |
33-
make init-dev
34-
- name: Run formatting checks
35-
run: |
36-
make check
81+
- name: Install deps
82+
run: cd backend && make init-dev
3783
- name: Run unit tests
3884
working-directory: backend
3985
run: |
40-
uv pip install huggingface_hub[cli]
41-
huggingface-cli download --repo-type dataset The-OpenROAD-Project/ORAssistant_RAG_Dataset --include source_list.json --local-dir data/
4286
cp .env.test .env
4387
make test
88+
89+
docker-eval:
90+
needs: [test, lint-frontend, lint-evaluation]
91+
runs-on: self-hosted
92+
steps:
93+
- name: Checkout code
94+
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
95+
96+
- name: Download HF dataset
97+
run: |
98+
if [ ! -d "data" ] || [ -z "$(ls -A data 2>/dev/null)" ]; then
99+
pip install huggingface_hub
100+
huggingface-cli download The-OpenROAD-Project/ORAssistant_RAG_Dataset \
101+
--local-dir ./data
102+
echo "Dataset downloaded"
103+
else
104+
echo "Dataset already cached on runner, skipping download"
105+
fi
106+
44107
- name: Populate environment variables
45108
run: |
46109
cp backend/.env.example backend/.env
@@ -54,14 +117,16 @@ jobs:
54117
sed -i 's|{{GOOGLE_PROJECT_ID}}|${{ secrets.GOOGLE_PROJECT_ID }}|g' evaluation/.env
55118
sed -i 's|{{PATH_TO_GOOGLE_APPLICATION_CREDENTIALS}}|src/secret.json|g' evaluation/.env
56119
sed -i 's|HF_TOKEN=|HF_TOKEN=${{ secrets.HF_TOKEN }}|g' evaluation/.env
120+
57121
- name: Copy Google credentials
58122
env:
59123
GOOGLE_SECRET_JSON: ${{ secrets.PATH_TO_GOOGLE_APPLICATION_CREDENTIALS }}
60124
run: |
61125
make seed-credentials
62-
- name: Build Docker image
126+
127+
- name: Build and start Docker stack
63128
run: |
64-
make docker-up
129+
make docker-up-ci
65130
66131
- name: Wait for graph readiness
67132
run: |
@@ -97,7 +162,6 @@ jobs:
97162
echo "=== COMPLETE POSTGRES LOGS ==="
98163
docker logs postgres 2>&1 || echo "Failed to get postgres logs"
99164
100-
# Save to files for artifacts
101165
docker logs backend > docker-logs/backend.log 2>&1 || echo "Failed to capture backend logs"
102166
docker logs frontend > docker-logs/frontend.log 2>&1 || echo "Failed to capture frontend logs"
103167
docker logs postgres > docker-logs/postgres.log 2>&1 || echo "Failed to capture postgres logs"
@@ -128,4 +192,4 @@ jobs:
128192
- name: Teardown
129193
if: always()
130194
run: |
131-
make docker-down
195+
make docker-down-ci

‎Makefile‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,18 @@ check-ci:
3636
docker-up:
3737
@docker compose -f docker-compose.yml up --build --wait
3838

39+
.PHONY: docker-up-ci
40+
docker-up-ci:
41+
@docker compose -f docker-compose.yml -f docker-compose.ci.yml up --build --wait
42+
3943
.PHONY: docker-down
4044
docker-down:
4145
@docker compose -f docker-compose.yml down --volumes --remove-orphans
4246

47+
.PHONY: docker-down-ci
48+
docker-down-ci:
49+
@docker compose -f docker-compose.yml -f docker-compose.ci.yml down --volumes --remove-orphans
50+
4351
.PHONY: docker-dev
4452
docker-dev:
4553
@docker compose -f docker-compose.yml -f docker-compose.dev.yml up --build --wait

‎backend/Dockerfile‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,15 @@ RUN pip install uv
2020
COPY ./pyproject.toml /ORAssistant-backend/pyproject.toml
2121
COPY . .
2222

23-
RUN uv venv .venv && uv sync --dev && uv run /ORAssistant-backend/src/post_install.py
23+
RUN uv venv .venv && uv sync && uv run /ORAssistant-backend/src/post_install.py
2424

25-
RUN git clone https://huggingface.co/datasets/The-OpenROAD-Project/ORAssistant_RAG_Dataset && \
25+
ARG SKIP_HF_DOWNLOAD=false
26+
RUN if [ "$SKIP_HF_DOWNLOAD" = "false" ]; then \
27+
git clone https://huggingface.co/datasets/The-OpenROAD-Project/ORAssistant_RAG_Dataset && \
2628
mkdir -p data && \
2729
mv ORAssistant_RAG_Dataset/* data/ && \
28-
rm -rf ORAssistant_RAG_Dataset
30+
rm -rf ORAssistant_RAG_Dataset; \
31+
fi
2932

3033
EXPOSE 8000
3134

‎backend/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ build-docs:
2828

2929
.PHONY: test
3030
test:
31-
@uv run pytest
31+
@uv run pytest -n auto
3232

3333
.PHONY: mcp
3434
mcp:

‎docker-compose.ci.yml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
services:
2+
backend:
3+
build:
4+
context: ./backend
5+
args:
6+
SKIP_HF_DOWNLOAD: "true"
7+
volumes:
8+
- faiss_data:/ORAssistant-backend/faiss_db
9+
- ./data:/ORAssistant-backend/data
10+
environment:
11+
HEALTHCHECK_START_PERIOD: "300s"
12+
HEALTHCHECK_INTERVAL: "10s"
13+
HEALTHCHECK_RETRIES: "30"
14+
15+
volumes:
16+
faiss_data:
17+
driver: local

0 commit comments

Comments
 (0)