feat: replace container base images with pixi-managed environments (#810)

* chore: importlib-metadata should be constrained on diracx-routers

* fix: Force importlib-metadata = "<8.8.0" due to https://pixi.sh/latest/concepts/conda_pypi/#pinned-package-conflicts

* fix: enable OTel log record injection to avoid KeyError with sdk >=1.28.0

Since opentelemetry-sdk v1.28.0 (open-telemetry/opentelemetry-python#4166),
LoggingHandler._translate() calls self.format(record) when a formatter is
set. The existing formatter uses %(otelTraceID)s placeholders but
set_logging_format=False meant the LoggingInstrumentor never injected
those attributes, causing a KeyError at runtime.

* chore: replace container base images with pixi-managed environments

* ci: Simplify running legacy DIRAC integration tests

* test: correct verify_entry_points to handle extensions and duplicate names

* ci: improve debugging output of legacy integration tests

---------

Co-authored-by: Christophe Haen <christophe.haen@cern.ch>

Author: chrisburr

.dockerignore

@@ -0,0 +1,9 @@
+# Pixi environments (can be very large)
+.pixi/
+
+# Documentation build output
+site/
+
+# Python build artifacts
+*.egg-info/
+__pycache__/

.github/workflows/deployment.yml

@@ -97,7 +97,7 @@ jobs:
 
   docker:
     needs: deploy-pypi
-    timeout-minutes:  30
+    timeout-minutes: 30
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -112,58 +112,46 @@ jobs:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Download diracx wheels
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
-        with:
-            name: diracx-whl
-      - name: "Find wheels"
-        id: find_wheel
-        run: |
-          # We need to copy them there to be able to access them in the RUN --mount
-          cp diracx*.whl containers/client/
-          cp diracx*.whl containers/services/
-          for wheel_fn in *.whl; do
-            pkg_name=$(basename "${wheel_fn}" | cut -d '-' -f 1)
-            echo "${pkg_name}-wheel-name=$(ls "${pkg_name}"-*.whl)" >> $GITHUB_OUTPUT
-          done
 
-      - name: Build and push client (release)
+      - name: Build and push services (release)
         uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7
         if: ${{ needs.deploy-pypi.outputs.create-release == 'true' }}
         with:
-          context: containers/client/
-          push: ${{ needs.deploy-pypi.outputs.create-release == 'true' }}
-          tags: "ghcr.io/diracgrid/diracx/client:${{ needs.deploy-pypi.outputs.new-version }}"
+          context: .
+          file: containers/Dockerfile
+          build-args: PIXI_ENV=container-services
+          push: true
+          tags: "ghcr.io/diracgrid/diracx/services:${{ needs.deploy-pypi.outputs.new-version }}"
           platforms: linux/amd64,linux/arm64
-          build-args: EXTRA_PACKAGES_TO_INSTALL=DIRACCommon~=9.0.0
-      - name: Build and push services (release)
+      - name: Build and push client (release)
         uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7
         if: ${{ needs.deploy-pypi.outputs.create-release == 'true' }}
         with:
-          context: containers/services/
-          push: ${{ needs.deploy-pypi.outputs.create-release == 'true' }}
-          tags: "ghcr.io/diracgrid/diracx/services:${{ needs.deploy-pypi.outputs.new-version }}"
+          context: .
+          file: containers/Dockerfile
+          build-args: PIXI_ENV=container-client
+          push: true
+          tags: "ghcr.io/diracgrid/diracx/client:${{ needs.deploy-pypi.outputs.new-version }}"
           platforms: linux/amd64,linux/arm64
-          build-args: EXTRA_PACKAGES_TO_INSTALL=DIRACCommon~=9.0.0
 
-      - name: Build and push client (dev)
+      - name: Build and push services (dev)
         uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7
         with:
-          context: containers/client/
+          context: .
+          file: containers/Dockerfile
+          build-args: PIXI_ENV=container-services
           push: ${{ github.event_name != 'pull_request' && github.repository == 'DIRACGrid/diracx' && github.ref_name == 'main' }}
-          tags: ghcr.io/diracgrid/diracx/client:dev
+          tags: ghcr.io/diracgrid/diracx/services:dev
           platforms: linux/amd64,linux/arm64
-          build-args: |
-            EXTRA_PACKAGES_TO_INSTALL=git+https://github.com/DIRACGrid/DIRAC.git@integration#egg=diraccommon\&subdirectory=dirac-common
-      - name: Build and push services (dev)
+      - name: Build and push client (dev)
         uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7
         with:
-          context: containers/services/
+          context: .
+          file: containers/Dockerfile
+          build-args: PIXI_ENV=container-client
           push: ${{ github.event_name != 'pull_request' && github.repository == 'DIRACGrid/diracx' && github.ref_name == 'main' }}
-          tags: ghcr.io/diracgrid/diracx/services:dev
+          tags: ghcr.io/diracgrid/diracx/client:dev
           platforms: linux/amd64,linux/arm64
-          build-args: |
-            EXTRA_PACKAGES_TO_INSTALL=git+https://github.com/DIRACGrid/DIRAC.git@integration#egg=diraccommon\&subdirectory=dirac-common
 
   update-charts:
     name: Update Helm charts

.github/workflows/integration.yml

@@ -29,21 +29,13 @@ jobs:
       - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
         with:
           python-version: '3.14'
-      - name: Build wheels
-        run: |
-          pip install build
-          for pkg_dir in $PWD/diracx-* .; do
-            echo "Building $pkg_dir"
-            python -m build --outdir "${GITHUB_WORKSPACE}/dist" $pkg_dir
-          done
       - name: Clone DIRAC
         run: |
           pip install typer pyyaml gitpython packaging
           git clone https://github.com/DIRACGrid/DIRAC.git -b "${{ matrix.dirac-branch }}" /tmp/DIRACRepo
           echo "Current revision: $(git -C /tmp/DIRACRepo rev-parse HEAD)"
-          # We need to cd in the directory for the integration_tests.py to work
       - name: Prepare environment
-        run: cd /tmp/DIRACRepo && ./integration_tests.py prepare-environment "TEST_DIRACX=Yes"  --diracx-dist-dir "${GITHUB_WORKSPACE}/dist"
+        run: cd /tmp/DIRACRepo && ./integration_tests.py prepare-environment "TEST_DIRACX=Yes" --diracx-src-dir "${GITHUB_WORKSPACE}"
       - name: Install server
         run:  cd /tmp/DIRACRepo && ./integration_tests.py install-server
       - name: Install client
@@ -64,19 +56,35 @@ jobs:
           if [ -f client-tests-failed ]; then has_error=1; echo "Client tests failed"; fi
           if [ -f pilot-tests-failed ]; then has_error=1; echo "Pilot tests failed"; fi
           if [ ${has_error} = 1 ]; then exit 1; fi
-      - name: diracx filtered requests logs
+      - name: Debugging information
+        if: always()
         run: |
+          mkdir -p /tmp/service-logs
+
+          echo "::group::Container status"
+          docker ps -a --format "table {{.Names}}\t{{.Status}}\t{{.Image}}\t{{.Ports}}"
+          echo "::endgroup::"
+
+          DIRACX_CONTAINERS="diracx diracx-init-keystore diracx-init-cs diracx-init-db diracx-chmod"
+
+          for container in $DIRACX_CONTAINERS; do
+            if docker inspect "$container" &>/dev/null; then
+              echo "::group::Logs: $container"
+              docker logs "$container" 2>&1 | tee -a /tmp/service-logs/diracx.log
+              echo "::endgroup::"
+            fi
+          done
+
+          echo "::group::DiracX API request summary"
           docker logs diracx 2>/dev/null | grep '/api/' \
             | awk -F\" '{print $2, $3}' \
             | awk '{print $1, $2, $4}' \
-            | sort | uniq -c | sort
-      - name: diracx error logs
-        if: ${{ failure() }}
-        run: |
-          mkdir -p /tmp/service-logs
-          docker logs diracx 2>&1 | tee /tmp/service-logs/diracx.log
-          cd /tmp/DIRACRepo
-          ./integration_tests.py logs --no-follow --lines 1000 2>&1 | tee /tmp/service-logs/dirac.log
+            | sort | uniq -c | sort || true
+          echo "::endgroup::"
+
+          echo "::group::DIRAC server logs"
+          cd /tmp/DIRACRepo && ./integration_tests.py logs --no-follow --lines 1000 2>&1 | tee /tmp/service-logs/dirac.log || true
+          echo "::endgroup::"
       - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
         if: ${{ failure() }}
         with:

.github/workflows/main.yml

@@ -121,44 +121,32 @@ jobs:
         with:
           cache: false
           environments: ${{ matrix.extension == 'diracx' && 'default' || 'default-gubbins' }}
-      - name: Build gubbins wheels
-        if: ${{ matrix.extension == 'gubbins' }}
-        run: |
-          for pkg_dir in $PWD/diracx-*; do
-            echo "Building $pkg_dir"
-            pixi exec python-build --outdir $PWD/extensions/containers/services/ $pkg_dir
-          done
-          # Also build the diracx metapackage
-          pixi exec python-build --outdir $PWD/extensions/containers/services/ .
-          # And build the gubbins package
-          for pkg_dir in $PWD/extensions/gubbins/gubbins-*; do
-            # Skip the testing package
-            if [[ "${pkg_dir}" =~ .*testing.* ]];
-            then
-              echo "Do not build ${pkg_dir}";
-              continue;
-            fi
-            echo "Building $pkg_dir"
-            pixi exec python-build --outdir $PWD/extensions/containers/services/ $pkg_dir
-          done
       - name: Set up Docker Buildx
-        if: ${{ matrix.extension == 'gubbins' }}
         uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4
-      - name: Build container for gubbins
-        if: ${{ matrix.extension == 'gubbins' }}
+      - name: Build services image
+        uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7
+        with:
+          context: .
+          file: containers/Dockerfile
+          build-args: |
+            PIXI_ENV=${{ matrix.extension == 'diracx' && 'container-services' || 'gubbins-container-services' }}
+          tags: ghcr.io/${{ matrix.extension == 'diracx' && 'diracgrid/diracx' || 'gubbins' }}/services:dev
+          outputs: type=docker,dest=/tmp/services_image.tar
+      - name: Build client image
         uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7
         with:
-          context: extensions/containers/services
-          tags: gubbins/services:dev
-          outputs: type=docker,dest=/tmp/gubbins_services_image.tar
+          context: .
+          file: containers/Dockerfile
           build-args: |
-            EXTENSION_CUSTOM_SOURCES_TO_INSTALL=/bindmount/gubbins_db*.whl,/bindmount/gubbins_logic*.whl,/bindmount/gubbins_routers*.whl,/bindmount/gubbins_client*.whl
-      - name: Load image
-        if: ${{ matrix.extension == 'gubbins' }}
+            PIXI_ENV=${{ matrix.extension == 'diracx' && 'container-client' || 'gubbins-container-client' }}
+          tags: ghcr.io/${{ matrix.extension == 'diracx' && 'diracgrid/diracx' || 'gubbins' }}/client:dev
+          outputs: type=docker,dest=/tmp/client_image.tar
+      - name: Load images
         run: |
-          docker load --input /tmp/gubbins_services_image.tar
-          # Free up disk space by removing the tarball after loading the image
-          rm -Rf /tmp/gubbins_services_image.tar
+          docker load --input /tmp/services_image.tar
+          rm -f /tmp/services_image.tar
+          docker load --input /tmp/client_image.tar
+          rm -f /tmp/client_image.tar
           docker builder prune -af || true
           docker image ls -a
       - name: Start demo
@@ -191,16 +179,26 @@ jobs:
             # Copy gubbins-charts to a temporary location and build dependencies
             cp -r ./extensions/gubbins-charts /tmp/
             ../diracx-charts/.demo/helm dependency build /tmp/gubbins-charts
+            # Replace the downloaded subchart with the locally-cloned one to
+            # ensure we have the pixi-compatible ConfigMap entrypoint
+            rm -f /tmp/gubbins-charts/charts/diracx-*.tgz
+            ../diracx-charts/.demo/helm package ../diracx-charts/diracx -d /tmp/gubbins-charts/charts/
 
             demo_args+=("--extension-chart-path" "/tmp/gubbins-charts")
             demo_args+=("--ci-values" "./extensions/gubbins_values.yaml")
-            demo_args+=("--load-docker-image" "gubbins/services:dev")
+            demo_args+=("--load-docker-image" "ghcr.io/gubbins/services:dev")
+            demo_args+=("--load-docker-image" "ghcr.io/gubbins/client:dev")
+            demo_args+=("--prune-loaded-images")
             demo_source_dirs+=("/tmp/gubbins/")
           elif [ ${{ matrix.extension }} != 'diracx' ]; then
             echo "Unknown extension: ${{ matrix.extension }}"
             exit 1
           else
             demo_args+=("--set-value" "developer.autoReload=false")
+            demo_args+=("--set-value" "global.imagePullPolicy=IfNotPresent")
+            demo_args+=("--load-docker-image" "ghcr.io/diracgrid/diracx/services:dev")
+            demo_args+=("--load-docker-image" "ghcr.io/diracgrid/diracx/client:dev")
+            demo_args+=("--prune-loaded-images")
           fi
 
           # Run the demo with the provided arguments

.gitignore

@@ -97,7 +97,6 @@ docs/source/_build
 
 # pixi environments
 .pixi
-pixi.lock
 *.egg-info
 docs/templates/_builtin_markdown.jinja
 
@@ -106,3 +105,7 @@ docs/templates/_builtin_markdown.jinja
 
 # docs site
 site
+
+# DiracX specific
+# No point in committing the pixi.lock for gubbins as it cannot work properly within the diracx repo
+extensions/gubbins/pixi.lock

.pre-commit-config.yaml

@@ -28,6 +28,7 @@ repos:
       - id: check-yaml
         args: ["--unsafe"]
       - id: check-added-large-files
+        exclude: pixi\.lock$
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: "v0.15.7"
@@ -66,6 +67,7 @@ repos:
     hooks:
       - id: codespell
         args: ["-w"]
+        exclude: pixi\.lock$
 
   - repo: https://github.com/adamchainz/blacken-docs
     rev: 1.20.0

containers/Dockerfile

@@ -0,0 +1,63 @@
+FROM ghcr.io/prefix-dev/pixi:latest AS build
+
+ARG PIXI_ENV=container-services
+
+WORKDIR /app
+COPY pixi.toml pixi.lock ./
+
+# Copy source directories needed for path-based installs
+COPY .git/ .git/
+COPY diracx-api/ diracx-api/
+COPY diracx-cli/ diracx-cli/
+COPY diracx-client/ diracx-client/
+COPY diracx-core/ diracx-core/
+COPY diracx-db/ diracx-db/
+COPY diracx-logic/ diracx-logic/
+COPY diracx-routers/ diracx-routers/
+COPY diracx-testing/ diracx-testing/
+COPY pyproject.toml ./
+COPY README.md ./
+COPY extensions/gubbins/gubbins-api/ extensions/gubbins/gubbins-api/
+COPY extensions/gubbins/gubbins-cli/ extensions/gubbins/gubbins-cli/
+COPY extensions/gubbins/gubbins-client/ extensions/gubbins/gubbins-client/
+COPY extensions/gubbins/gubbins-core/ extensions/gubbins/gubbins-core/
+COPY extensions/gubbins/gubbins-db/ extensions/gubbins/gubbins-db/
+COPY extensions/gubbins/gubbins-logic/ extensions/gubbins/gubbins-logic/
+COPY extensions/gubbins/gubbins-routers/ extensions/gubbins/gubbins-routers/
+COPY extensions/gubbins/gubbins-testing/ extensions/gubbins/gubbins-testing/
+COPY extensions/gubbins/pyproject.toml extensions/gubbins/
+COPY extensions/gubbins/README.md extensions/gubbins/
+
+# Switch to non-editable installs (same workaround as CI)
+RUN sed -i 's@editable = true@editable = false@g' pixi.toml
+
+RUN pixi install --frozen -e ${PIXI_ENV}
+
+# Generate activation script
+RUN pixi shell-hook -e ${PIXI_ENV} > /activate.sh
+
+FROM ubuntu:24.04
+
+ARG PIXI_ENV=container-services
+ENV PIXI_ENV=${PIXI_ENV}
+
+EXPOSE 8000
+
+# Copy the installed environment
+COPY --from=build /app/.pixi/envs/${PIXI_ENV} /app/.pixi/envs/${PIXI_ENV}
+COPY --from=build /activate.sh /activate.sh
+COPY containers/entrypoint.sh /entrypoint.sh
+
+# In many clusters the container is run as a random uid for security reasons.
+# If we mark the environment directory as group 0 and give it group write
+# permissions then we're still able to manage the environment from inside the
+# container.
+RUN chmod -R g=u /app/.pixi
+
+# Compatibility shim: the Helm chart invokes this micromamba entrypoint path
+RUN printf '#!/bin/bash\nsource /activate.sh\nexec "$@"\n' > /usr/local/bin/_entrypoint.sh && \
+    chmod +x /usr/local/bin/_entrypoint.sh
+
+# Use tini as init (installed by pixi in the env)
+ENTRYPOINT ["/bin/bash", "-c", \
+            "exec /app/.pixi/envs/${PIXI_ENV}/bin/tini -- /entrypoint.sh \"$@\"", "--"]

containers/client/Dockerfile

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+
+source /activate.sh
+
+exec "$@"