Skip to content

Commit 83caaa0

Browse files
chrisburrchrisburr
authored
Merge pull request #249 from chaen/gubbins
Introduce extensions: Gubbins
2 parents 139b82d + 1704d06 commit 83caaa0

136 files changed

Lines changed: 14812 additions & 879 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.coveragerc‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,10 @@
22
omit =
33
tests/*
44
*/tests/*
5-
**/diracx/client/*
5+
**/diracx/client/generated/*
66
**/diracx/testing/*
7+
**/gubbins/testing/*
8+
**/gubbins/client/generated/*
79

810
[paths]
911
source =

‎.github/workflows/extensions.yml‎

Lines changed: 284 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,284 @@
1+
name: Extensions full test
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
- gubbins
8+
pull_request:
9+
branches:
10+
- main
11+
12+
13+
defaults:
14+
run:
15+
shell: bash -el {0}
16+
17+
jobs:
18+
unittest:
19+
name: Unit test - ${{ matrix.package }}
20+
runs-on: ubuntu-latest
21+
strategy:
22+
fail-fast: false
23+
matrix:
24+
# In principle, the dependencies could be limited to the extension packages.
25+
# However, we want to use the local packages, and not those published on pypi
26+
include:
27+
- package: "./extensions/gubbins/gubbins-core"
28+
dependencies: "./extensions/gubbins/gubbins-testing ./diracx-testing ./diracx-core"
29+
- package: "./extensions/gubbins/gubbins-db"
30+
dependencies: "./extensions/gubbins/gubbins-testing ./extensions/gubbins/gubbins-core ./diracx-testing ./diracx-db ./diracx-core "
31+
- package: "./extensions/gubbins/gubbins-routers"
32+
dependencies: "./extensions/gubbins/gubbins-testing ./extensions/gubbins/gubbins-db ./extensions/gubbins/gubbins-core ./diracx-testing ./diracx-db ./diracx-core ./diracx-routers"
33+
- package: "./extensions/gubbins/gubbins-client"
34+
dependencies: "./extensions/gubbins/gubbins-testing ./diracx-testing ./extensions/gubbins/gubbins-client ./extensions/gubbins/gubbins-core ./diracx-client ./diracx-core "
35+
- package: "./extensions/gubbins/gubbins-cli"
36+
dependencies: "./extensions/gubbins/gubbins-testing ./extensions/gubbins/gubbins-client ./extensions/gubbins/gubbins-core ./diracx-testing ./diracx-cli ./diracx-client ./diracx-core ./diracx-api"
37+
steps:
38+
- name: Checkout code
39+
uses: actions/checkout@v4
40+
- uses: mamba-org/setup-micromamba@v2
41+
with:
42+
# TODO: Use a conda environment file used for the diracx/base container image
43+
environment-name: test-env
44+
create-args: >-
45+
python=3.11
46+
m2crypto
47+
python-gfal2
48+
mypy
49+
pip
50+
init-shell: bash
51+
post-cleanup: 'all'
52+
- name: Set up environment
53+
run: |
54+
pip install pytest-github-actions-annotate-failures
55+
pip install git+https://github.com/DIRACGrid/DIRAC.git@integration
56+
pip install ${{ matrix.dependencies }} ${{ matrix.package }}[types]
57+
- name: Run mypy
58+
run: |
59+
mypy ${{ matrix.package }}/src
60+
- name: Run pytest
61+
run: |
62+
cd ${{ matrix.package }}
63+
pip install .[testing]
64+
export DIRACX_EXTENSIONS=gubbins,diracx
65+
pytest --cov-report=xml:coverage.xml --junitxml=report.xml
66+
- name: Upload coverage report
67+
uses: codecov/codecov-action@v4.6.0
68+
69+
70+
build-wheels:
71+
name: Build wheels
72+
runs-on: "ubuntu-latest"
73+
if: github.event_name != 'push' || github.repository == 'DIRACGrid/diracx'
74+
defaults:
75+
run:
76+
# We need extglob for REFERENCE_BRANCH substitution
77+
shell: bash -l -O extglob {0}
78+
steps:
79+
- name: Checkout
80+
uses: actions/checkout@v4
81+
- uses: actions/setup-python@v5
82+
with:
83+
python-version: '3.11'
84+
- name: Installing dependencies
85+
run: |
86+
python -m pip install \
87+
build \
88+
python-dateutil \
89+
pytz \
90+
readme_renderer[md] \
91+
requests \
92+
setuptools_scm
93+
- name: Build distributions
94+
run: |
95+
for pkg_dir in $PWD/diracx-*; do
96+
echo "Building $pkg_dir"
97+
python -m build --outdir $PWD/dist $pkg_dir
98+
done
99+
# Also build the diracx metapackage
100+
python -m build --outdir $PWD/dist .
101+
# And build the gubbins package
102+
for pkg_dir in $PWD/extensions/gubbins/gubbins-*; do
103+
# Skip the testing package
104+
if [[ "${pkg_dir}" =~ .*testing.* ]];
105+
then
106+
echo "Do not build ${pkg_dir}";
107+
continue;
108+
fi
109+
echo "Building $pkg_dir"
110+
python -m build --outdir $PWD/dist $pkg_dir
111+
done
112+
- name: 'Upload Artifact'
113+
uses: actions/upload-artifact@v4
114+
with:
115+
name: gubbins-whl
116+
path: dist/*.whl
117+
retention-days: 5
118+
119+
# Build to docker image with the code in it
120+
build-image:
121+
needs: build-wheels
122+
timeout-minutes: 30
123+
runs-on: ubuntu-latest
124+
steps:
125+
- name: Checkout
126+
uses: actions/checkout@v4
127+
- name: Set up Docker Buildx
128+
uses: docker/setup-buildx-action@v3
129+
- name: Download gubbins wheels
130+
uses: actions/download-artifact@v4
131+
with:
132+
name: gubbins-whl
133+
- name: "Find wheels"
134+
id: find_wheel
135+
run: |
136+
# We need to copy them there to be able to access them in the RUN --mount
137+
cp diracx*.whl gubbins*.whl extensions/containers/services/
138+
for wheel_fn in *.whl; do
139+
pkg_name=$(basename "${wheel_fn}" | cut -d '-' -f 1)
140+
echo "${pkg_name}-wheel-name=$(ls "${pkg_name}"-*.whl)" >> $GITHUB_OUTPUT
141+
done
142+
- name: Build and export service
143+
uses: docker/build-push-action@v6
144+
with:
145+
context: extensions/containers/services
146+
tags: gubbins/services:dev
147+
outputs: type=docker,dest=/tmp/gubbins_services_image.tar
148+
build-args: |
149+
EXTRA_PACKAGES_TO_INSTALL=git+https://github.com/DIRACGrid/DIRAC.git@integration
150+
EXTENSION_CUSTOM_SOURCES_TO_INSTALL=/bindmount/gubbins_db*.whl,/bindmount/gubbins_routers*.whl,/bindmount/gubbins_client*.whl
151+
- name: Build and export client
152+
uses: docker/build-push-action@v6
153+
with:
154+
context: extensions/containers/client
155+
tags: gubbins/client:dev
156+
outputs: type=docker,dest=/tmp/gubbins_client_image.tar
157+
- name: Upload artifact
158+
uses: actions/upload-artifact@v4
159+
with:
160+
name: gubbins-services-img
161+
path: /tmp/gubbins_services_image.tar
162+
163+
164+
pytest-integration:
165+
needs: build-image
166+
runs-on: ubuntu-latest
167+
steps:
168+
- name: Download gubbins-image
169+
uses: actions/download-artifact@v4
170+
with:
171+
name: gubbins-services-img
172+
path: /tmp
173+
- name: Load image
174+
run: |
175+
docker load --input /tmp/gubbins_services_image.tar
176+
docker image ls -a
177+
- name: Checkout code
178+
uses: actions/checkout@v4
179+
- uses: mamba-org/setup-micromamba@v2
180+
with:
181+
environment-file: environment.yml
182+
init-shell: bash
183+
post-cleanup: 'all'
184+
- name: Set up environment
185+
run: |
186+
pip install pytest-github-actions-annotate-failures
187+
pip install git+https://github.com/DIRACGrid/DIRAC.git@integration
188+
pip install ./diracx-core/[testing] ./diracx-api/[testing] ./diracx-cli/[testing] ./diracx-client/[testing] ./diracx-routers/[testing] ./diracx-db/[testing] ./diracx-testing/[testing] ./extensions/gubbins/gubbins-testing[testing] ./extensions/gubbins/gubbins-db[testing] ./extensions/gubbins/gubbins-routers/[testing] ./extensions/gubbins/gubbins-client/[testing] ./extensions/gubbins/gubbins-cli/[testing] ./extensions/gubbins/gubbins-core/[testing]
189+
- name: Start demo
190+
run: |
191+
git clone https://github.com/DIRACGrid/diracx-charts.git ../diracx-charts
192+
# We have to copy the code to another directory
193+
# and make it a git repository by itself because otherwise the
194+
# root in the pyproject to do not make sense once mounted
195+
# in the containers.
196+
cp -r ./extensions/gubbins /tmp/
197+
sed -i 's@../..@.@g' /tmp/gubbins/pyproject.toml
198+
sed -i 's@../../@@g' /tmp/gubbins/gubbins-*/pyproject.toml
199+
git init /tmp/gubbins/
200+
../diracx-charts/run_demo.sh --enable-open-telemetry --enable-coverage --exit-when-done --set-value developer.autoReload=false --ci-values ../diracx-charts/demo/ci_values.yaml --ci-values ./extensions/gubbins_values.yaml --load-docker-image "gubbins/services:dev" $PWD /tmp/gubbins/
201+
- name: Debugging information
202+
run: |
203+
DIRACX_DEMO_DIR=$PWD/../diracx-charts/.demo
204+
export KUBECONFIG=${DIRACX_DEMO_DIR}/kube.conf
205+
export PATH=${DIRACX_DEMO_DIR}:$PATH
206+
kubectl get pods
207+
for pod_name in $(kubectl get pods -o json | jq -r '.items[] | .metadata.name' | grep -vE '(dex|minio|mysql|rabbitmq|opensearch)'); do
208+
echo "${pod_name}"
209+
kubectl describe pod/"${pod_name}" || true
210+
for container_name in $(kubectl get pods $pod_name -o jsonpath='{.spec.initContainers[*].name} {.spec.containers[*].name}'); do
211+
echo $pod_name $container_name
212+
kubectl logs "${pod_name}" -c "${container_name}" || true
213+
done
214+
done
215+
if [ ! -f "${DIRACX_DEMO_DIR}/.success" ]; then
216+
cat "${DIRACX_DEMO_DIR}/.failed"
217+
exit 1
218+
fi
219+
- name: Run pytest
220+
run: |
221+
cd extensions/gubbins
222+
export DIRACX_EXTENSIONS=gubbins,diracx
223+
pytest --demo-dir=../../../diracx-charts/ --cov-report=xml:coverage-pytest.xml --junitxml=report.xml
224+
- name: Collect demo coverage
225+
run: |
226+
DIRACX_DEMO_DIR=$PWD/../diracx-charts/.demo
227+
export KUBECONFIG=${DIRACX_DEMO_DIR}/kube.conf
228+
export PATH=${DIRACX_DEMO_DIR}:$PATH
229+
# Shutdown the pods so we collect coverage data
230+
for pod_name in $(kubectl get pods -o json | jq -r '.items[] | .metadata.name' | grep -vE '(dex|minio|mysql|rabbitmq|opensearch)'); do
231+
kubectl delete pod/"${pod_name}"
232+
done
233+
set -x
234+
# Combine the coverage data from the demo and make an XML report
235+
coverage_data=$(mktemp)
236+
sudo chown -R $(id -u) "${DIRACX_DEMO_DIR}"/coverage-reports/
237+
coverage combine --keep --data-file "${coverage_data}" "${DIRACX_DEMO_DIR}"/coverage-reports/*
238+
239+
# coverage can't handle having multiple src directories, so we need to make a fake one with symlinks
240+
fake_module=$(mktemp -d)
241+
242+
mkdir -p "${fake_module}/src/diracx"
243+
for fn in "${PWD}"/*/src/diracx/*; do
244+
ln -sf "${fn}" "${fake_module}/src/diracx/$(basename "${fn}")"
245+
done
246+
247+
248+
mkdir -p "${fake_module}/src/gubbins"
249+
for fn in "${PWD}"/extensions/gubbins/*/src/gubbins/*; do
250+
ln -sf "${fn}" "${fake_module}/src/gubbins/$(basename "${fn}")"
251+
done
252+
253+
sed -i "s@source =@source =\n ${fake_module}/src@g" .coveragerc
254+
255+
cat .coveragerc
256+
257+
coverage xml -o coverage-demo.xml --data-file "${coverage_data}"
258+
- name: Upload coverage report
259+
uses: codecov/codecov-action@v4.6.0
260+
with:
261+
files: ./coverage-pytest.xml,./coverage-demo.xml
262+
263+
client-generation:
264+
runs-on: ubuntu-latest
265+
steps:
266+
- name: Checkout code
267+
uses: actions/checkout@v4
268+
- uses: mamba-org/setup-micromamba@v2
269+
with:
270+
environment-file: environment.yml
271+
init-shell: bash
272+
post-cleanup: 'all'
273+
- name: Set up environment
274+
run: |
275+
micromamba install -c conda-forge nodejs pre-commit
276+
pip install git+https://github.com/DIRACGrid/DIRAC.git@integration
277+
pip install ./diracx-core/[testing] ./diracx-api/[testing] ./diracx-cli/[testing] ./diracx-client/[testing] ./diracx-routers/[testing] ./diracx-db/[testing] ./diracx-testing/[testing] ./extensions/gubbins/gubbins-testing[testing] ./extensions/gubbins/gubbins-db[testing] ./extensions/gubbins/gubbins-routers/[testing] ./extensions/gubbins/gubbins-testing/[testing] -e ./extensions/gubbins/gubbins-client/[testing] ./extensions/gubbins/gubbins-core/[testing]
278+
npm install -g autorest
279+
- name: Run autorest
280+
run: |
281+
autorest --python --help
282+
$HOME/.autorest/\@autorest_python\@*/node_modules/\@autorest/python/venv/bin/python -m pip install --upgrade setuptools
283+
export DIRACX_EXTENSIONS=gubbins,diracx
284+
pytest --no-cov --regenerate-client extensions/gubbins/gubbins-client/tests/test_regenerate.py

‎.github/workflows/main.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,8 @@ jobs:
2929
run: |
3030
find -name '*.sh' -print0 | xargs -0 -n1 shellcheck --exclude=SC1090,SC1091 --external-source
3131
32-
pytest:
32+
33+
unittest:
3334
name: Unit test - ${{ matrix.package }}
3435
runs-on: ubuntu-latest
3536
strategy:
@@ -61,6 +62,9 @@ jobs:
6162
- name: Set up environment
6263
run: |
6364
pip install pytest-github-actions-annotate-failures
65+
# Note: DIRAC will install pretty much everything
66+
# from diracx so installing just the dependency may
67+
# be a bit useless
6468
pip install git+https://github.com/DIRACGrid/DIRAC.git@integration
6569
pip install ${{ matrix.dependencies }}
6670
- name: Run pytest

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ parts
2121
bin
2222
develop-eggs
2323
.installed.cfg
24+
*.whl
2425

2526
# Translations
2627
*.mo

‎.pre-commit-config.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,4 +35,4 @@ repos:
3535
- types-requests
3636
- types-aiobotocore[essential]
3737
- boto3-stubs[essential]
38-
exclude: ^(diracx-client/src/diracx/client/|diracx-[a-z]+/tests/|diracx-testing/|build)
38+
exclude: ^(diracx-client/src/diracx/client/generated|diracx-[a-z]+/tests/|diracx-testing/|build|extensions/gubbins/gubbins-client/src/gubbins/client/generated)

‎codecov.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,10 @@ coverage:
1212

1313
codecov:
1414
notify:
15-
after_n_builds: 7
15+
# 6 diracx unit tests
16+
# 1 diracx integration test
17+
# 5 gubbins unit test
18+
# 1 gubbins integration test
19+
after_n_builds: 13
1620

1721
comment: false

‎diracx-cli/pyproject.toml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,15 @@ types = [
3636
[project.scripts]
3737
dirac = "diracx.cli:app"
3838

39+
[project.entry-points."diracx.cli"]
40+
jobs = "diracx.cli.jobs:app"
41+
config = "diracx.cli.config:app"
42+
43+
[project.entry-points."diracx.cli.hidden"]
44+
internal = "diracx.cli.internal:app"
45+
46+
47+
3948
[tool.setuptools.packages.find]
4049
where = ["src"]
4150

@@ -58,3 +67,4 @@ asyncio_mode = "auto"
5867
markers = [
5968
"enabled_dependencies: List of dependencies which should be available to the FastAPI test client",
6069
]
70+
asyncio_default_fixture_loop_scope = "session"

0 commit comments

Comments
 (0)