Merge branch 'main' into drop-config-source-readconfig

Author: chrisburr

.dockerignore

@@ -0,0 +1,9 @@
+# Pixi environments (can be very large)
+.pixi/
+
+# Documentation build output
+site/
+
+# Python build artifacts
+*.egg-info/
+__pycache__/

.github/workflows/deployment.yml

@@ -93,11 +93,11 @@ jobs:
       # https://docs.pypi.org/trusted-publishers/
       - name: Publish package on PyPI
         if: steps.check-tag.outputs.create-release == 'true'
-        uses: pypa/gh-action-pypi-publish@release/v1
+        uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # release/v1
 
   docker:
     needs: deploy-pypi
-    timeout-minutes:  30
+    timeout-minutes: 30
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -112,58 +112,46 @@ jobs:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Download diracx wheels
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
-        with:
-            name: diracx-whl
-      - name: "Find wheels"
-        id: find_wheel
-        run: |
-          # We need to copy them there to be able to access them in the RUN --mount
-          cp diracx*.whl containers/client/
-          cp diracx*.whl containers/services/
-          for wheel_fn in *.whl; do
-            pkg_name=$(basename "${wheel_fn}" | cut -d '-' -f 1)
-            echo "${pkg_name}-wheel-name=$(ls "${pkg_name}"-*.whl)" >> $GITHUB_OUTPUT
-          done
 
-      - name: Build and push client (release)
+      - name: Build and push services (release)
         uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7
         if: ${{ needs.deploy-pypi.outputs.create-release == 'true' }}
         with:
-          context: containers/client/
-          push: ${{ needs.deploy-pypi.outputs.create-release == 'true' }}
-          tags: "ghcr.io/diracgrid/diracx/client:${{ needs.deploy-pypi.outputs.new-version }}"
+          context: .
+          file: containers/Dockerfile
+          build-args: PIXI_ENV=container-services
+          push: true
+          tags: "ghcr.io/diracgrid/diracx/services:${{ needs.deploy-pypi.outputs.new-version }}"
           platforms: linux/amd64,linux/arm64
-          build-args: EXTRA_PACKAGES_TO_INSTALL=DIRACCommon~=9.0.0
-      - name: Build and push services (release)
+      - name: Build and push client (release)
         uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7
         if: ${{ needs.deploy-pypi.outputs.create-release == 'true' }}
         with:
-          context: containers/services/
-          push: ${{ needs.deploy-pypi.outputs.create-release == 'true' }}
-          tags: "ghcr.io/diracgrid/diracx/services:${{ needs.deploy-pypi.outputs.new-version }}"
+          context: .
+          file: containers/Dockerfile
+          build-args: PIXI_ENV=container-client
+          push: true
+          tags: "ghcr.io/diracgrid/diracx/client:${{ needs.deploy-pypi.outputs.new-version }}"
           platforms: linux/amd64,linux/arm64
-          build-args: EXTRA_PACKAGES_TO_INSTALL=DIRACCommon~=9.0.0
 
-      - name: Build and push client (dev)
+      - name: Build and push services (dev)
         uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7
         with:
-          context: containers/client/
+          context: .
+          file: containers/Dockerfile
+          build-args: PIXI_ENV=container-services
           push: ${{ github.event_name != 'pull_request' && github.repository == 'DIRACGrid/diracx' && github.ref_name == 'main' }}
-          tags: ghcr.io/diracgrid/diracx/client:dev
+          tags: ghcr.io/diracgrid/diracx/services:dev
           platforms: linux/amd64,linux/arm64
-          build-args: |
-            EXTRA_PACKAGES_TO_INSTALL=git+https://github.com/DIRACGrid/DIRAC.git@integration#egg=diraccommon\&subdirectory=dirac-common
-      - name: Build and push services (dev)
+      - name: Build and push client (dev)
         uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7
         with:
-          context: containers/services/
+          context: .
+          file: containers/Dockerfile
+          build-args: PIXI_ENV=container-client
           push: ${{ github.event_name != 'pull_request' && github.repository == 'DIRACGrid/diracx' && github.ref_name == 'main' }}
-          tags: ghcr.io/diracgrid/diracx/services:dev
+          tags: ghcr.io/diracgrid/diracx/client:dev
           platforms: linux/amd64,linux/arm64
-          build-args: |
-            EXTRA_PACKAGES_TO_INSTALL=git+https://github.com/DIRACGrid/DIRAC.git@integration#egg=diraccommon\&subdirectory=dirac-common
 
   update-charts:
     name: Update Helm charts

.github/workflows/draft-on-changes-requested.yml

@@ -26,7 +26,7 @@ jobs:
 
   draft:
     needs: get-pr
-    uses: DIRACGrid/.github/.github/workflows/draft-on-changes-requested.yml@main
+    uses: DIRACGrid/.github/.github/workflows/draft-on-changes-requested.yml@eab297d50d33cf7eabf0b388ac57d1f91e36d14c # main
     with:
       pr_number: ${{ fromJSON(needs.get-pr.outputs.pr_number) }}
     secrets:

.github/workflows/integration.yml

@@ -29,21 +29,13 @@ jobs:
       - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
         with:
           python-version: '3.14'
-      - name: Build wheels
-        run: |
-          pip install build
-          for pkg_dir in $PWD/diracx-* .; do
-            echo "Building $pkg_dir"
-            python -m build --outdir "${GITHUB_WORKSPACE}/dist" $pkg_dir
-          done
       - name: Clone DIRAC
         run: |
           pip install typer pyyaml gitpython packaging
           git clone https://github.com/DIRACGrid/DIRAC.git -b "${{ matrix.dirac-branch }}" /tmp/DIRACRepo
           echo "Current revision: $(git -C /tmp/DIRACRepo rev-parse HEAD)"
-          # We need to cd in the directory for the integration_tests.py to work
       - name: Prepare environment
-        run: cd /tmp/DIRACRepo && ./integration_tests.py prepare-environment "TEST_DIRACX=Yes"  --diracx-dist-dir "${GITHUB_WORKSPACE}/dist"
+        run: cd /tmp/DIRACRepo && ./integration_tests.py prepare-environment "TEST_DIRACX=Yes" --diracx-src-dir "${GITHUB_WORKSPACE}"
       - name: Install server
         run:  cd /tmp/DIRACRepo && ./integration_tests.py install-server
       - name: Install client
@@ -64,19 +56,35 @@ jobs:
           if [ -f client-tests-failed ]; then has_error=1; echo "Client tests failed"; fi
           if [ -f pilot-tests-failed ]; then has_error=1; echo "Pilot tests failed"; fi
           if [ ${has_error} = 1 ]; then exit 1; fi
-      - name: diracx filtered requests logs
+      - name: Debugging information
+        if: always()
         run: |
+          mkdir -p /tmp/service-logs
+
+          echo "::group::Container status"
+          docker ps -a --format "table {{.Names}}\t{{.Status}}\t{{.Image}}\t{{.Ports}}"
+          echo "::endgroup::"
+
+          DIRACX_CONTAINERS="diracx diracx-init-keystore diracx-init-cs diracx-init-db diracx-chmod"
+
+          for container in $DIRACX_CONTAINERS; do
+            if docker inspect "$container" &>/dev/null; then
+              echo "::group::Logs: $container"
+              docker logs "$container" 2>&1 | tee -a /tmp/service-logs/diracx.log
+              echo "::endgroup::"
+            fi
+          done
+
+          echo "::group::DiracX API request summary"
           docker logs diracx 2>/dev/null | grep '/api/' \
             | awk -F\" '{print $2, $3}' \
             | awk '{print $1, $2, $4}' \
-            | sort | uniq -c | sort
-      - name: diracx error logs
-        if: ${{ failure() }}
-        run: |
-          mkdir -p /tmp/service-logs
-          docker logs diracx 2>&1 | tee /tmp/service-logs/diracx.log
-          cd /tmp/DIRACRepo
-          ./integration_tests.py logs --no-follow --lines 1000 2>&1 | tee /tmp/service-logs/dirac.log
+            | sort | uniq -c | sort || true
+          echo "::endgroup::"
+
+          echo "::group::DIRAC server logs"
+          cd /tmp/DIRACRepo && ./integration_tests.py logs --no-follow --lines 1000 2>&1 | tee /tmp/service-logs/dirac.log || true
+          echo "::endgroup::"
       - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
         if: ${{ failure() }}
         with:

.github/workflows/main.yml

@@ -73,7 +73,7 @@ jobs:
       - name: Run pytest
         run: pixi run pytest-${{ matrix.package }} --cov-report=xml:coverage.xml --junitxml=report.xml
       - name: Upload coverage report
-        uses: codecov/codecov-action@v5.5.2
+        uses: codecov/codecov-action@1af58845a975a7985b0beb0cbe6fbbb71a41dbad # v5.5.3
 
   pytest-integration:
     runs-on: ubuntu-latest
@@ -121,44 +121,32 @@ jobs:
         with:
           cache: false
           environments: ${{ matrix.extension == 'diracx' && 'default' || 'default-gubbins' }}
-      - name: Build gubbins wheels
-        if: ${{ matrix.extension == 'gubbins' }}
-        run: |
-          for pkg_dir in $PWD/diracx-*; do
-            echo "Building $pkg_dir"
-            pixi exec python-build --outdir $PWD/extensions/containers/services/ $pkg_dir
-          done
-          # Also build the diracx metapackage
-          pixi exec python-build --outdir $PWD/extensions/containers/services/ .
-          # And build the gubbins package
-          for pkg_dir in $PWD/extensions/gubbins/gubbins-*; do
-            # Skip the testing package
-            if [[ "${pkg_dir}" =~ .*testing.* ]];
-            then
-              echo "Do not build ${pkg_dir}";
-              continue;
-            fi
-            echo "Building $pkg_dir"
-            pixi exec python-build --outdir $PWD/extensions/containers/services/ $pkg_dir
-          done
       - name: Set up Docker Buildx
-        if: ${{ matrix.extension == 'gubbins' }}
         uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4
-      - name: Build container for gubbins
-        if: ${{ matrix.extension == 'gubbins' }}
+      - name: Build services image
+        uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7
+        with:
+          context: .
+          file: containers/Dockerfile
+          build-args: |
+            PIXI_ENV=${{ matrix.extension == 'diracx' && 'container-services' || 'gubbins-container-services' }}
+          tags: ghcr.io/${{ matrix.extension == 'diracx' && 'diracgrid/diracx' || 'gubbins' }}/services:dev
+          outputs: type=docker,dest=/tmp/services_image.tar
+      - name: Build client image
         uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7
         with:
-          context: extensions/containers/services
-          tags: gubbins/services:dev
-          outputs: type=docker,dest=/tmp/gubbins_services_image.tar
+          context: .
+          file: containers/Dockerfile
           build-args: |
-            EXTENSION_CUSTOM_SOURCES_TO_INSTALL=/bindmount/gubbins_db*.whl,/bindmount/gubbins_logic*.whl,/bindmount/gubbins_routers*.whl,/bindmount/gubbins_client*.whl
-      - name: Load image
-        if: ${{ matrix.extension == 'gubbins' }}
+            PIXI_ENV=${{ matrix.extension == 'diracx' && 'container-client' || 'gubbins-container-client' }}
+          tags: ghcr.io/${{ matrix.extension == 'diracx' && 'diracgrid/diracx' || 'gubbins' }}/client:dev
+          outputs: type=docker,dest=/tmp/client_image.tar
+      - name: Load images
         run: |
-          docker load --input /tmp/gubbins_services_image.tar
-          # Free up disk space by removing the tarball after loading the image
-          rm -Rf /tmp/gubbins_services_image.tar
+          docker load --input /tmp/services_image.tar
+          rm -f /tmp/services_image.tar
+          docker load --input /tmp/client_image.tar
+          rm -f /tmp/client_image.tar
           docker builder prune -af || true
           docker image ls -a
       - name: Start demo
@@ -171,41 +159,39 @@ jobs:
           demo_args+=("--exit-when-done")
           demo_args+=("--ci-values" "../diracx-charts/demo/ci_values.yaml")
 
-          declare -a demo_source_dirs=("$PWD")
-
           # Download helm/kubectl/kind first
           ../diracx-charts/run_demo.sh --only-download-deps
 
           if [ ${{ matrix.extension }} == 'gubbins' ]; then
             demo_args+=("--set-value" "diracx.developer.autoReload=false")
 
-            # We have to copy the code to another directory
-            # and make it a git repository by itself because otherwise the
-            # root in the pyproject do not make sense once mounted
-            # in the containers.
-            cp -r ./extensions/gubbins /tmp/
-            sed -i 's@../..@.@g' /tmp/gubbins/pyproject.toml
-            sed -i 's@../../@@g' /tmp/gubbins/gubbins-*/pyproject.toml
-            git init /tmp/gubbins/
-
             # Copy gubbins-charts to a temporary location and build dependencies
             cp -r ./extensions/gubbins-charts /tmp/
             ../diracx-charts/.demo/helm dependency build /tmp/gubbins-charts
+            # Replace the downloaded subchart with the locally-cloned one to
+            # ensure we have the pixi-compatible ConfigMap entrypoint
+            rm -f /tmp/gubbins-charts/charts/diracx-*.tgz
+            ../diracx-charts/.demo/helm package ../diracx-charts/diracx -d /tmp/gubbins-charts/charts/
 
             demo_args+=("--extension-chart-path" "/tmp/gubbins-charts")
             demo_args+=("--ci-values" "./extensions/gubbins_values.yaml")
-            demo_args+=("--load-docker-image" "gubbins/services:dev")
-            demo_source_dirs+=("/tmp/gubbins/")
+            demo_args+=("--load-docker-image" "ghcr.io/gubbins/services:dev")
+            demo_args+=("--load-docker-image" "ghcr.io/gubbins/client:dev")
+            demo_args+=("--prune-loaded-images")
           elif [ ${{ matrix.extension }} != 'diracx' ]; then
             echo "Unknown extension: ${{ matrix.extension }}"
             exit 1
           else
             demo_args+=("--set-value" "developer.autoReload=false")
+            demo_args+=("--set-value" "global.imagePullPolicy=IfNotPresent")
+            demo_args+=("--load-docker-image" "ghcr.io/diracgrid/diracx/services:dev")
+            demo_args+=("--load-docker-image" "ghcr.io/diracgrid/diracx/client:dev")
+            demo_args+=("--prune-loaded-images")
           fi
 
           # Run the demo with the provided arguments
           set -x
-          ../diracx-charts/run_demo.sh "${demo_args[@]}" "${demo_source_dirs[@]}"
+          ../diracx-charts/run_demo.sh "${demo_args[@]}"
       - name: Debugging information
         if: always()
         run: .github/scripts/collect-k8s-debug-info.sh "$PWD/../diracx-charts/.demo" "${{ matrix.extension == 'gubbins' && 'gubbins' || 'diracx' }}"
@@ -216,7 +202,7 @@ jobs:
         run: |
           pixi run -e ${{ matrix.extension == 'diracx' && 'default' || 'default-gubbins' }} python -m diracx.testing coverage collect-demo --demo-dir=../diracx-charts/.demo
       - name: Upload coverage report
-        uses: codecov/codecov-action@v5.5.2
+        uses: codecov/codecov-action@1af58845a975a7985b0beb0cbe6fbbb71a41dbad # v5.5.3
         with:
           files: ./coverage-pytest.xml,./coverage-demo.xml
 

.gitignore

@@ -97,7 +97,6 @@ docs/source/_build
 
 # pixi environments
 .pixi
-pixi.lock
 *.egg-info
 docs/templates/_builtin_markdown.jinja
 
@@ -106,3 +105,7 @@ docs/templates/_builtin_markdown.jinja
 
 # docs site
 site
+
+# DiracX specific
+# No point in committing the pixi.lock for gubbins as it cannot work properly within the diracx repo
+extensions/gubbins/pixi.lock

.pre-commit-config.yaml

@@ -28,9 +28,10 @@ repos:
       - id: check-yaml
         args: ["--unsafe"]
       - id: check-added-large-files
+        exclude: pixi\.lock$
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: "v0.15.7"
+    rev: "v0.15.8"
     hooks:
       - id: ruff-check
         args: ["--fix", "--show-fixes"]
@@ -66,6 +67,7 @@ repos:
     hooks:
       - id: codespell
         args: ["-w"]
+        exclude: pixi\.lock$
 
   - repo: https://github.com/adamchainz/blacken-docs
     rev: 1.20.0