Skip to content

Commit 2b3a1db

Browse files
NiklasKoehneckeAASebastianNiehusAA
NiklasKoehneckeAA
and
SebastianNiehusAA
authored
refactor: share python setup across test jobs (#1219)
* refactor: remove unused os workflows * feat: add reusable python setup * refactor: DRY for python setup, bump studio --------- Co-authored-by: Sebastian Niehus <165138846+SebastianNiehusAA@users.noreply.github.com>
1 parent d51be8d commit 2b3a1db

7 files changed

Lines changed: 65 additions & 216 deletions

File tree

.github/workflows/get_jfrog_access_token.sh renamed to .github/composites/get_jfrog_access_token.sh

File renamed without changes.

.github/workflows/get_jfrog_access_token_subject.sh renamed to .github/composites/get_jfrog_access_token_subject.sh

File renamed without changes.

‎.github/composites/python-setup/action.yml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
name: Checkout and set up python
2+
description: "Installs python, dependencies and handles venv caching"
3+
runs:
4+
using: composite
5+
steps:
6+
- uses: actions/setup-python@v5
7+
with:
8+
python-version: "3.10"
9+
10+
- name: Install and configure Poetry
11+
uses: snok/install-poetry@v1
12+
with:
13+
virtualenvs-create: true
14+
virtualenvs-in-project: true
15+
installer-parallel: true
16+
virtualenvs-path: .venv
17+
18+
- name: Load cached venv
19+
id: cached-poetry-dependencies
20+
uses: actions/cache@v4
21+
with:
22+
path: .venv
23+
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
24+
25+
- name: Install dependencies
26+
shell: bash
27+
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
28+
run: |
29+
poetry config installer.max-workers 10
30+
poetry install --no-interaction

‎.github/workflows/daily.yml‎

Lines changed: 0 additions & 87 deletions
This file was deleted.

‎.github/workflows/sdk-tests.yml‎

Lines changed: 34 additions & 124 deletions
Original file line numberDiff line numberDiff line change
@@ -4,49 +4,25 @@ on:
44
runner:
55
type: string
66
default: "ubuntu-latest"
7-
secrets:
8-
AA_TOKEN:
9-
required: true
10-
HUGGING_FACE_TOKEN:
11-
required: true
7+
timeout:
8+
type: number
9+
default: 15 # mins
10+
11+
defaults:
12+
run:
13+
shell: bash
14+
1215
jobs:
1316
lint:
14-
defaults:
15-
run:
16-
shell: bash
17-
timeout-minutes: 15
17+
timeout-minutes: ${{inputs.timeout}}
1818
runs-on: ${{inputs.runner}}
1919
steps:
2020
- name: Checkout repository
2121
uses: actions/checkout@v4
22-
23-
- uses: actions/setup-python@v5
24-
with:
25-
python-version: "3.10"
26-
27-
- name: Install and configure Poetry
28-
uses: snok/install-poetry@v1
29-
with:
30-
virtualenvs-create: true
31-
virtualenvs-in-project: true
32-
installer-parallel: true
33-
34-
- name: Load cached venv
35-
id: cached-poetry-dependencies
36-
uses: actions/cache@v4
37-
with:
38-
path: .venv
39-
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
40-
41-
- name: Install dependencies
42-
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
43-
run: |
44-
poetry config installer.max-workers 10
45-
poetry install --no-interaction
22+
- uses: ./.github/composites/python-setup
4623

4724
- name: set PY for pre-commit
4825
run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV
49-
5026
- uses: actions/cache@v4
5127
with:
5228
path: ~/.cache/pre-commit
@@ -56,37 +32,13 @@ jobs:
5632
run: |
5733
./scripts/lint.sh
5834
doctest:
59-
defaults:
60-
run:
61-
shell: bash
62-
timeout-minutes: 15
35+
timeout-minutes: ${{inputs.timeout}}
6336
runs-on: ${{inputs.runner}}
6437
steps:
6538
- name: Checkout repository
6639
uses: actions/checkout@v4
67-
- uses: actions/setup-python@v5
68-
with:
69-
python-version: "3.10"
70-
71-
- name: Install and configure Poetry
72-
uses: snok/install-poetry@v1
73-
with:
74-
virtualenvs-create: true
75-
virtualenvs-in-project: true
76-
installer-parallel: true
40+
- uses: ./.github/composites/python-setup
7741

78-
- name: Load cached venv
79-
id: cached-poetry-dependencies
80-
uses: actions/cache@v4
81-
with:
82-
path: .venv
83-
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
84-
85-
- name: Install dependencies
86-
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
87-
run: |
88-
poetry config installer.max-workers 10
89-
poetry install --no-interaction
9042
- name: Run doctest
9143
env:
9244
AA_TOKEN: ${{ secrets.AA_TOKEN }}
@@ -95,10 +47,7 @@ jobs:
9547
run: ./scripts/doctest.sh
9648

9749
test:
98-
defaults:
99-
run:
100-
shell: bash
101-
timeout-minutes: 15
50+
timeout-minutes: ${{inputs.timeout}}
10251
runs-on: ${{inputs.runner}}
10352
services:
10453
argilla-elastic-search:
@@ -138,15 +87,16 @@ jobs:
13887
POSTGRES_USER: "il_sdk"
13988
POSTGRES_PASSWORD: "test"
14089
studio-backend:
141-
image: registry.gitlab.aleph-alpha.de/product/studio/backend:latest
90+
image: registry.gitlab.aleph-alpha.de/product/studio/backend:v0.0.65
14291
ports:
14392
- "8000:8000"
14493
env:
145-
POSTGRES_HOST: "postgres"
146-
POSTGRES_PORT: "5432"
14794
POSTGRES_DB: "il_sdk"
14895
POSTGRES_USER: "il_sdk"
14996
POSTGRES_PASSWORD: "test"
97+
POSTGRES_HOST: "postgres"
98+
POSTGRES_PORT: "5432"
99+
150100
AUTHORIZATION_SERVICE_URL: ${{ secrets.AUTHORIZATION_SERVICE_URL }}
151101
AA_TOKEN: ${{ secrets.AA_TOKEN }}
152102
API_SCHEDULER_URL: ${{ secrets.CLIENT_URL }}
@@ -157,53 +107,31 @@ jobs:
157107
steps:
158108
- name: Checkout repository
159109
uses: actions/checkout@v4
160-
161-
- uses: actions/setup-python@v5
162-
with:
163-
python-version: "3.10"
164-
165-
- name: Install and configure Poetry
166-
uses: snok/install-poetry@v1
167-
with:
168-
virtualenvs-create: true
169-
virtualenvs-in-project: true
170-
installer-parallel: true
171-
172-
- name: Load cached venv
173-
id: cached-poetry-dependencies
174-
uses: actions/cache@v4
175-
with:
176-
path: .venv
177-
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
178-
179-
- name: Install dependencies
180-
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
181-
run: |
182-
poetry config installer.max-workers 10
183-
poetry install --no-interaction --without docs
110+
- uses: ./.github/composites/python-setup
184111

185112
- name: Run pytest
186113
env:
114+
POSTGRES_DB: "il_sdk"
115+
POSTGRES_USER: "il_sdk"
116+
POSTGRES_PASSWORD: "test"
117+
POSTGRES_HOST: "localhost"
118+
POSTGRES_PORT: "5433"
119+
120+
AUTHORIZATION_SERVICE_URL: ${{ secrets.AUTHORIZATION_SERVICE_URL }}
187121
AA_TOKEN: ${{ secrets.AA_TOKEN }}
122+
DATA_SERVICE_URL: ${{secrets.DATA_SERVICE_URL}}
123+
188124
HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
189125
ARGILLA_API_URL: "http://localhost:6900/"
190126
ARGILLA_API_KEY: "argilla.apikey"
191127
CLIENT_URL: ${{ secrets.CLIENT_URL }}
192128
STUDIO_URL: "http://localhost:8000/"
193129
DOCUMENT_INDEX_URL: ${{secrets.DOCUMENT_INDEX_URL}}
194130
PHARIA_KERNEL_URL: https://pharia-kernel.product.pharia.com
195-
POSTGRES_HOST: "localhost"
196-
POSTGRES_PORT: "5433"
197-
POSTGRES_DB: "il_sdk"
198-
POSTGRES_USER: "il_sdk"
199-
POSTGRES_PASSWORD: "test"
200131
run: |
201132
./scripts/test.sh
202133
run-notebooks:
203-
defaults:
204-
run:
205-
shell: bash
206-
timeout-minutes: 15
134+
timeout-minutes: ${{inputs.timeout}}
207135
runs-on: ${{inputs.runner}}
208136
services:
209137
argilla-elastic-search:
@@ -228,15 +156,16 @@ jobs:
228156
POSTGRES_USER: "il_sdk"
229157
POSTGRES_PASSWORD: "test"
230158
studio-backend:
231-
image: registry.gitlab.aleph-alpha.de/product/studio/backend:latest
159+
image: registry.gitlab.aleph-alpha.de/product/studio/backend:v0.0.65
232160
ports:
233161
- "8001:8000"
234162
env:
235-
POSTGRES_HOST: "postgres"
236-
POSTGRES_PORT: "5432"
237163
POSTGRES_DB: "il_sdk"
238164
POSTGRES_USER: "il_sdk"
239165
POSTGRES_PASSWORD: "test"
166+
POSTGRES_HOST: "postgres"
167+
POSTGRES_PORT: "5432"
168+
240169
AUTHORIZATION_SERVICE_URL: ${{ secrets.AUTHORIZATION_SERVICE_URL }}
241170
AA_TOKEN: ${{ secrets.AA_TOKEN }}
242171
API_SCHEDULER_URL: ${{ secrets.CLIENT_URL }}
@@ -247,28 +176,9 @@ jobs:
247176
steps:
248177
- name: Checkout repository
249178
uses: actions/checkout@v4
250-
- uses: actions/setup-python@v5
251-
with:
252-
python-version: "3.10"
253-
- name: Install and configure Poetry
254-
uses: snok/install-poetry@v1
255-
with:
256-
virtualenvs-create: true
257-
virtualenvs-in-project: true
258-
installer-parallel: true
179+
- uses: ./.github/composites/python-setup
259180

260-
- name: Load cached venv
261-
id: cached-poetry-dependencies
262-
uses: actions/cache@v4
263-
with:
264-
path: .venv
265-
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
266-
- name: Install dependencies
267-
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
268-
run: |
269-
poetry config installer.max-workers 10
270-
poetry install --no-interaction
271-
- name: Configure Poetry for notebooks and run
181+
- name: Run Notebooks
272182
env:
273183
AA_TOKEN: ${{ secrets.AA_TOKEN }}
274184
HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}

‎.github/workflows/test-execution.yml‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,10 @@ on:
99
# manual trigger
1010
workflow_dispatch:
1111

12-
13-
1412
concurrency:
1513
group: ${{ github.workflow }}-${{ github.ref }}
1614
cancel-in-progress: true
1715
jobs:
1816
python-tests:
1917
uses: ./.github/workflows/sdk-tests.yml
20-
with:
21-
runner: 'ubuntu-latest'
2218
secrets: inherit

‎docker-compose.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ services:
6161
# echo $GITLAB_TOKEN | docker login registry.gitlab.aleph-alpha.de -u your_email@for_gitlab --password-stdin
6262
# docker compose pull to update containers
6363
studio-backend:
64-
image: registry.gitlab.aleph-alpha.de/product/studio/backend:v0.0.64
64+
image: registry.gitlab.aleph-alpha.de/product/studio/backend:v0.0.65
6565
ports:
6666
- 8000:8000
6767
depends_on:

0 commit comments

Comments
 (0)