From fb3e50e8ef1d9a1931fc01e40ce607bc0f61a688 Mon Sep 17 00:00:00 2001 From: Edoardo Baldi Date: Tue, 6 May 2025 11:03:18 +0200 Subject: [PATCH 01/12] Optimize Docker images & add multi-architecture support - Reduce image size by implementing multi-stage build - Remove torchaudio package and optimize PyTorch installation - Add GPU/CPU variant support via build arguments - Enable multi-architecture builds (amd64/arm64) with proper manifests - Fix Renku compatibility by setting provenance: false - Simplify container environment variables and working directory - Create architecture-specific tags for better image management --- .github/workflows/docker-build.yml | 53 ++++++++++++++++++++-- Dockerfile | 70 ++++++++++++++++++++++-------- docker/environment.yml | 3 -- 3 files changed, 100 insertions(+), 26 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 5762f13b..039fd26d 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -29,6 +29,7 @@ jobs: strategy: matrix: arch: [amd64, arm64] + variant: [cpu, cuda] fail-fast: false steps: - name: Checkout code @@ -54,10 +55,10 @@ jobs: with: images: ghcr.io/${{ github.repository }} tags: | - type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} - type=ref,event=pr - type=ref,event=tag - type=sha,format=short + type=raw,value=${{ matrix.variant }}-${{ matrix.arch }},enable=${{ github.ref == 'refs/heads/main' }} + type=raw,value=${{ matrix.variant }}-${{ matrix.arch }}-pr-${{ github.event.pull_request.number }},enable=${{ github.event_name == 'pull_request' }} + type=raw,value=${{ matrix.variant }}-${{ matrix.arch }}-${{ github.ref_name }},enable=${{ startsWith(github.ref, 'refs/tags/') }} + type=raw,value=${{ matrix.variant }}-${{ matrix.arch }}-sha-${{ github.sha }},prefix= - name: Build and push uses: docker/build-push-action@v6 @@ -67,3 +68,47 @@ jobs: push: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + provenance: false + build-args: | + PYTORCH_VARIANT=${{ matrix.variant }} + outputs: | + type=image,name=ghcr.io/${{ github.repository }},push-by-digest=false,name-canonical=false,push=true,annotation-index.org.opencontainers.image.description=A containerized Python tutorial environment with Jupyter Lab (${{ matrix.variant }} variant, ${{ matrix.arch }} architecture).,annotation-index.org.opencontainers.image.source=https://github.com/${{ github.repository }},annotation-index.org.opencontainers.image.authors=Empa Scientific IT ,annotation-index.org.opencontainers.image.licenses=MIT,annotation-index.org.opencontainers.image.variant=pytorch-${{ matrix.variant }}-${{ matrix.arch }} + + create-manifests: + needs: build-and-push + runs-on: ubuntu-latest + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + permissions: + packages: write + steps: + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Create and push CPU manifest + uses: docker/build-push-action@v6 + with: + context: . + target: builder + platforms: linux/amd64,linux/arm64 + push: true + provenance: false + tags: | + ghcr.io/${{ github.repository }}:cpu + ${{ startsWith(github.ref, 'refs/tags/') && format('ghcr.io/{0}:cpu-{1}', github.repository, github.ref_name) || '' }} + ${{ github.ref == 'refs/heads/main' && format('ghcr.io/{0}:latest', github.repository) || '' }} + + - name: Create and push CUDA manifest + uses: docker/build-push-action@v6 + with: + context: . + target: builder + platforms: linux/amd64,linux/arm64 + push: true + provenance: false + tags: | + ghcr.io/${{ github.repository }}:cuda + ${{ startsWith(github.ref, 'refs/tags/') && format('ghcr.io/{0}:cuda-{1}', github.repository, github.ref_name) || '' }} diff --git a/Dockerfile b/Dockerfile index cf496d34..94c0502d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,8 @@ -# Use the jupyter/minimal-notebook as the base image -FROM quay.io/jupyter/minimal-notebook:latest +# Stage 1: Build environment +FROM quay.io/jupyter/minimal-notebook:latest as builder -# Metadata labels -LABEL org.opencontainers.image.title="Python Tutorial" -LABEL org.opencontainers.image.description="A containerized Python tutorial environment with Jupyter Lab." -LABEL org.opencontainers.image.authors="Empa Scientific IT " -LABEL org.opencontainers.image.url="https://github.com/empa-scientific-it/python-tutorial" -LABEL org.opencontainers.image.source="https://github.com/empa-scientific-it/python-tutorial" -LABEL org.opencontainers.image.version="1.0.0" -LABEL org.opencontainers.image.licenses="MIT" - -# Set environment variables for the tutorial and repository -ENV BASENAME="python-tutorial" -ENV REPO=${HOME}/${BASENAME} -ENV IPYTHONDIR="${HOME}/.ipython" +# Define build argument for PyTorch variant (cpu or cuda) +ARG PYTORCH_VARIANT=cpu # Switch to root user to install additional dependencies USER root @@ -33,16 +22,59 @@ USER ${NB_UID} # Set up the Conda environment COPY docker/environment.yml /tmp/environment.yml RUN mamba env update -n base -f /tmp/environment.yml && \ + # Install PyTorch packages without cache - conditionally based on variant + if [ "$PYTORCH_VARIANT" = "cpu" ]; then \ + echo "Installing CPU-only PyTorch" && \ + pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/cpu; \ + else \ + echo "Installing CUDA-enabled PyTorch" && \ + pip install --no-cache-dir torch torchvision; \ + fi && \ + # Clean up all package caches to reduce image size mamba clean --all -f -y && \ + # Remove pip cache + rm -rf ~/.cache/pip && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" -# Prepare IPython configuration (move earlier in the build) -RUN mkdir -p ${HOME}/.ipython/profile_default +# Stage 2: Runtime environment - creates a lighter final image +FROM quay.io/jupyter/minimal-notebook:latest + +# Inherit build argument for image labeling +ARG PYTORCH_VARIANT=cpu + +# Metadata labels +LABEL org.opencontainers.image.title="Python Tutorial" +LABEL org.opencontainers.image.description="A containerized Python tutorial environment with Jupyter Lab." +LABEL org.opencontainers.image.authors="Empa Scientific IT " +LABEL org.opencontainers.image.url="https://github.com/empa-scientific-it/python-tutorial" +LABEL org.opencontainers.image.source="https://github.com/empa-scientific-it/python-tutorial" +LABEL org.opencontainers.image.version="1.0.0" +LABEL org.opencontainers.image.licenses="MIT" +LABEL org.opencontainers.image.variant="pytorch-${PYTORCH_VARIANT}" + +# Switch to root user to install minimal dependencies +USER root +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + libgl1 && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Switch back to the default notebook user +USER ${NB_UID} + +# Copy the conda environment from the builder stage +COPY --from=builder ${CONDA_DIR} ${CONDA_DIR} + +# Copy home directory with configurations +COPY --from=builder --chown=${NB_UID}:${NB_GID} /home/${NB_USER} /home/${NB_USER} + +# Prepare IPython configuration COPY --chown=${NB_UID}:${NB_GID} binder/ipython_config.py ${HOME}/.ipython/profile_default/ -# Set the working directory to the repository -WORKDIR ${REPO} +# Set the working directory to user's home (repository will be cloned here by Renku) +WORKDIR /home/${NB_USER} # Use the default ENTRYPOINT from the base image to start Jupyter Lab ENTRYPOINT ["tini", "-g", "--", "start.sh"] diff --git a/docker/environment.yml b/docker/environment.yml index 6df2975c..c913a505 100644 --- a/docker/environment.yml +++ b/docker/environment.yml @@ -26,8 +26,5 @@ dependencies: - python-dotenv - pillow - opencv-python - - torch - - torchaudio - - torchvision - albumentations - grad-cam From eabe8f4ffe7724f32069efc67ad1d6f773c633d2 Mon Sep 17 00:00:00 2001 From: Edoardo Baldi Date: Tue, 6 May 2025 11:11:56 +0200 Subject: [PATCH 02/12] Fix problem with "outputs" value --- .github/workflows/docker-build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 039fd26d..eb06c46a 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -71,8 +71,7 @@ jobs: provenance: false build-args: | PYTORCH_VARIANT=${{ matrix.variant }} - outputs: | - type=image,name=ghcr.io/${{ github.repository }},push-by-digest=false,name-canonical=false,push=true,annotation-index.org.opencontainers.image.description=A containerized Python tutorial environment with Jupyter Lab (${{ matrix.variant }} variant, ${{ matrix.arch }} architecture).,annotation-index.org.opencontainers.image.source=https://github.com/${{ github.repository }},annotation-index.org.opencontainers.image.authors=Empa Scientific IT ,annotation-index.org.opencontainers.image.licenses=MIT,annotation-index.org.opencontainers.image.variant=pytorch-${{ matrix.variant }}-${{ matrix.arch }} + outputs: type=image,name=ghcr.io/${{ github.repository }},annotation-index.org.opencontainers.image.description="Python Tutorial (${{ matrix.variant }} variant, ${{ matrix.arch }} architecture)" create-manifests: needs: build-and-push From bcbb3419f24bce74d989ddbdc8f8d6a0c1e63772 Mon Sep 17 00:00:00 2001 From: Edoardo Baldi Date: Tue, 6 May 2025 11:14:37 +0200 Subject: [PATCH 03/12] Fix string formatting --- .github/workflows/docker-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index eb06c46a..0587f05d 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -71,7 +71,7 @@ jobs: provenance: false build-args: | PYTORCH_VARIANT=${{ matrix.variant }} - outputs: type=image,name=ghcr.io/${{ github.repository }},annotation-index.org.opencontainers.image.description="Python Tutorial (${{ matrix.variant }} variant, ${{ matrix.arch }} architecture)" + outputs: type=image,name=ghcr.io/${{ github.repository }},annotation-index.org.opencontainers.image.description=Python Tutorial (${{ matrix.variant }} variant, ${{ matrix.arch }} architecture) create-manifests: needs: build-and-push From e94515dec2be28adde7216c5a381cc662d3ecb73 Mon Sep 17 00:00:00 2001 From: Edoardo Baldi Date: Tue, 6 May 2025 11:18:34 +0200 Subject: [PATCH 04/12] Another fix --- .github/workflows/docker-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 0587f05d..15e1d41c 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -71,7 +71,7 @@ jobs: provenance: false build-args: | PYTORCH_VARIANT=${{ matrix.variant }} - outputs: type=image,name=ghcr.io/${{ github.repository }},annotation-index.org.opencontainers.image.description=Python Tutorial (${{ matrix.variant }} variant, ${{ matrix.arch }} architecture) + outputs: type=image,name=ghcr.io/${{ github.repository }},annotation-index.org.opencontainers.image.description=Python Tutorial ${{ matrix.variant }} variant, ${{ matrix.arch }} architecture create-manifests: needs: build-and-push From 8edb8a58181101eb71b2080f8bb7d3345cf3d8fc Mon Sep 17 00:00:00 2001 From: Edoardo Baldi Date: Tue, 6 May 2025 11:27:34 +0200 Subject: [PATCH 05/12] Another attempt to fix "outputs" --- .github/workflows/docker-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 15e1d41c..fffea998 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -71,7 +71,7 @@ jobs: provenance: false build-args: | PYTORCH_VARIANT=${{ matrix.variant }} - outputs: type=image,name=ghcr.io/${{ github.repository }},annotation-index.org.opencontainers.image.description=Python Tutorial ${{ matrix.variant }} variant, ${{ matrix.arch }} architecture + outputs: type=image,name=ghcr.io/${{ github.repository }},annotation-index.org.opencontainers.image.title=Python Tutorial,annotation-index.org.opencontainers.image.description=A containerized Python tutorial environment with Jupyter Lab. create-manifests: needs: build-and-push From 0e56fd553a4cf6980145a7230e6ce5074487f58e Mon Sep 17 00:00:00 2001 From: Edoardo Baldi Date: Tue, 6 May 2025 11:39:33 +0200 Subject: [PATCH 06/12] Fix error "index annotations not supported for single platform export" --- .github/workflows/docker-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index fffea998..8042b553 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -67,11 +67,9 @@ jobs: platforms: linux/${{ matrix.arch }} push: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} provenance: false build-args: | PYTORCH_VARIANT=${{ matrix.variant }} - outputs: type=image,name=ghcr.io/${{ github.repository }},annotation-index.org.opencontainers.image.title=Python Tutorial,annotation-index.org.opencontainers.image.description=A containerized Python tutorial environment with Jupyter Lab. create-manifests: needs: build-and-push @@ -99,6 +97,7 @@ jobs: ghcr.io/${{ github.repository }}:cpu ${{ startsWith(github.ref, 'refs/tags/') && format('ghcr.io/{0}:cpu-{1}', github.repository, github.ref_name) || '' }} ${{ github.ref == 'refs/heads/main' && format('ghcr.io/{0}:latest', github.repository) || '' }} + outputs: type=image,name=ghcr.io/${{ github.repository }},annotation-index.org.opencontainers.image.title=Python Tutorial,annotation-index.org.opencontainers.image.description=A containerized Python tutorial environment with Jupyter Lab. - name: Create and push CUDA manifest uses: docker/build-push-action@v6 @@ -111,3 +110,4 @@ jobs: tags: | ghcr.io/${{ github.repository }}:cuda ${{ startsWith(github.ref, 'refs/tags/') && format('ghcr.io/{0}:cuda-{1}', github.repository, github.ref_name) || '' }} + outputs: type=image,name=ghcr.io/${{ github.repository }},annotation-index.org.opencontainers.image.title=Python Tutorial,annotation-index.org.opencontainers.image.description=A containerized Python tutorial environment with Jupyter Lab. From c732796931821678a1a40a629a53e744f20e5bb5 Mon Sep 17 00:00:00 2001 From: Edoardo Baldi Date: Tue, 6 May 2025 12:05:03 +0200 Subject: [PATCH 07/12] Changes to Dockerfile * Fix syntax * Force PyTorch uninstall first --- Dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 94c0502d..f9ebfe0b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Stage 1: Build environment -FROM quay.io/jupyter/minimal-notebook:latest as builder +FROM quay.io/jupyter/minimal-notebook:latest AS builder # Define build argument for PyTorch variant (cpu or cuda) ARG PYTORCH_VARIANT=cpu @@ -22,13 +22,15 @@ USER ${NB_UID} # Set up the Conda environment COPY docker/environment.yml /tmp/environment.yml RUN mamba env update -n base -f /tmp/environment.yml && \ + # Force remove any existing PyTorch installations first + pip uninstall -y torch torchvision && \ # Install PyTorch packages without cache - conditionally based on variant if [ "$PYTORCH_VARIANT" = "cpu" ]; then \ echo "Installing CPU-only PyTorch" && \ - pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/cpu; \ + pip install --no-cache-dir --force-reinstall torch torchvision --index-url https://download.pytorch.org/whl/cpu; \ else \ echo "Installing CUDA-enabled PyTorch" && \ - pip install --no-cache-dir torch torchvision; \ + pip install --no-cache-dir --force-reinstall torch torchvision; \ fi && \ # Clean up all package caches to reduce image size mamba clean --all -f -y && \ From accf4a91f1d5c5323d61620d512967b87bde6c90 Mon Sep 17 00:00:00 2001 From: Edoardo Baldi Date: Tue, 6 May 2025 12:06:36 +0200 Subject: [PATCH 08/12] Use "imagetools" to create manifests --- .github/workflows/docker-build.yml | 64 +++++++++++++++++++----------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 8042b553..c8d8fe75 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -78,6 +78,16 @@ jobs: permissions: packages: write steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + driver-opts: | + image=moby/buildkit:latest + network=host + - name: Log in to GHCR uses: docker/login-action@v3 with: @@ -86,28 +96,36 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Create and push CPU manifest - uses: docker/build-push-action@v6 - with: - context: . - target: builder - platforms: linux/amd64,linux/arm64 - push: true - provenance: false - tags: | - ghcr.io/${{ github.repository }}:cpu - ${{ startsWith(github.ref, 'refs/tags/') && format('ghcr.io/{0}:cpu-{1}', github.repository, github.ref_name) || '' }} - ${{ github.ref == 'refs/heads/main' && format('ghcr.io/{0}:latest', github.repository) || '' }} - outputs: type=image,name=ghcr.io/${{ github.repository }},annotation-index.org.opencontainers.image.title=Python Tutorial,annotation-index.org.opencontainers.image.description=A containerized Python tutorial environment with Jupyter Lab. + run: | + # Create and push multi-architecture manifest for CPU variant + docker buildx imagetools create --tag ghcr.io/${{ github.repository }}:cpu \ + ghcr.io/${{ github.repository }}:cpu-amd64 \ + ghcr.io/${{ github.repository }}:cpu-arm64 + + # If on main branch, also tag as latest + if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then + docker buildx imagetools create --tag ghcr.io/${{ github.repository }}:latest \ + ghcr.io/${{ github.repository }}:cpu-amd64 \ + ghcr.io/${{ github.repository }}:cpu-arm64 + fi + + # If this is a tag, add version tag for CPU + if [[ "${{ startsWith(github.ref, 'refs/tags/') }}" == "true" ]]; then + docker buildx imagetools create --tag ghcr.io/${{ github.repository }}:cpu-${{ github.ref_name }} \ + ghcr.io/${{ github.repository }}:cpu-amd64 \ + ghcr.io/${{ github.repository }}:cpu-arm64 + fi - name: Create and push CUDA manifest - uses: docker/build-push-action@v6 - with: - context: . - target: builder - platforms: linux/amd64,linux/arm64 - push: true - provenance: false - tags: | - ghcr.io/${{ github.repository }}:cuda - ${{ startsWith(github.ref, 'refs/tags/') && format('ghcr.io/{0}:cuda-{1}', github.repository, github.ref_name) || '' }} - outputs: type=image,name=ghcr.io/${{ github.repository }},annotation-index.org.opencontainers.image.title=Python Tutorial,annotation-index.org.opencontainers.image.description=A containerized Python tutorial environment with Jupyter Lab. + run: | + # Create and push multi-architecture manifest for CUDA variant + docker buildx imagetools create --tag ghcr.io/${{ github.repository }}:cuda \ + ghcr.io/${{ github.repository }}:cuda-amd64 \ + ghcr.io/${{ github.repository }}:cuda-arm64 + + # If this is a tag, add version tag for CUDA + if [[ "${{ startsWith(github.ref, 'refs/tags/') }}" == "true" ]]; then + docker buildx imagetools create --tag ghcr.io/${{ github.repository }}:cuda-${{ github.ref_name }} \ + ghcr.io/${{ github.repository }}:cuda-amd64 \ + ghcr.io/${{ github.repository }}:cuda-arm64 + fi From d97a2e79504c9e4bb628264ecb0172f293571c4f Mon Sep 17 00:00:00 2001 From: Edoardo Baldi Date: Tue, 6 May 2025 12:42:30 +0200 Subject: [PATCH 09/12] Correctly set tags in manifest using unique SHA --- .github/workflows/docker-build.yml | 70 ++++++++++++++++++++---------- 1 file changed, 47 insertions(+), 23 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index c8d8fe75..73e4388f 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -97,35 +97,59 @@ jobs: - name: Create and push CPU manifest run: | - # Create and push multi-architecture manifest for CPU variant - docker buildx imagetools create --tag ghcr.io/${{ github.repository }}:cpu \ - ghcr.io/${{ github.repository }}:cpu-amd64 \ - ghcr.io/${{ github.repository }}:cpu-arm64 + # Determine the correct tag suffixes based on the event type + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + # PR build - use the commit SHA for more predictable references + AMD64_TAG="cpu-amd64-sha-${{ github.sha }}" + ARM64_TAG="cpu-arm64-sha-${{ github.sha }}" + TARGET_TAG="cpu-sha-${{ github.sha }}" + elif [[ "${{ startsWith(github.ref, 'refs/tags/') }}" == "true" ]]; then + # Tag build - use tag version in the name + AMD64_TAG="cpu-amd64-${{ github.ref_name }}" + ARM64_TAG="cpu-arm64-${{ github.ref_name }}" + TARGET_TAG="cpu-${{ github.ref_name }}" + else + # Main branch build - use simple arch tags + AMD64_TAG="cpu-amd64" + ARM64_TAG="cpu-arm64" + TARGET_TAG="cpu" + fi + + # Create the manifest with the correct tag names + echo "Creating CPU manifest using $AMD64_TAG and $ARM64_TAG" + docker buildx imagetools create --tag ghcr.io/${{ github.repository }}:${TARGET_TAG} \ + ghcr.io/${{ github.repository }}:${AMD64_TAG} \ + ghcr.io/${{ github.repository }}:${ARM64_TAG} # If on main branch, also tag as latest if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then docker buildx imagetools create --tag ghcr.io/${{ github.repository }}:latest \ - ghcr.io/${{ github.repository }}:cpu-amd64 \ - ghcr.io/${{ github.repository }}:cpu-arm64 - fi - - # If this is a tag, add version tag for CPU - if [[ "${{ startsWith(github.ref, 'refs/tags/') }}" == "true" ]]; then - docker buildx imagetools create --tag ghcr.io/${{ github.repository }}:cpu-${{ github.ref_name }} \ - ghcr.io/${{ github.repository }}:cpu-amd64 \ - ghcr.io/${{ github.repository }}:cpu-arm64 + ghcr.io/${{ github.repository }}:${AMD64_TAG} \ + ghcr.io/${{ github.repository }}:${ARM64_TAG} fi - name: Create and push CUDA manifest run: | - # Create and push multi-architecture manifest for CUDA variant - docker buildx imagetools create --tag ghcr.io/${{ github.repository }}:cuda \ - ghcr.io/${{ github.repository }}:cuda-amd64 \ - ghcr.io/${{ github.repository }}:cuda-arm64 - - # If this is a tag, add version tag for CUDA - if [[ "${{ startsWith(github.ref, 'refs/tags/') }}" == "true" ]]; then - docker buildx imagetools create --tag ghcr.io/${{ github.repository }}:cuda-${{ github.ref_name }} \ - ghcr.io/${{ github.repository }}:cuda-amd64 \ - ghcr.io/${{ github.repository }}:cuda-arm64 + # Determine the correct tag suffixes based on the event type + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + # PR build - use the commit SHA for more predictable references + AMD64_TAG="cuda-amd64-sha-${{ github.sha }}" + ARM64_TAG="cuda-arm64-sha-${{ github.sha }}" + TARGET_TAG="cuda-sha-${{ github.sha }}" + elif [[ "${{ startsWith(github.ref, 'refs/tags/') }}" == "true" ]]; then + # Tag build - use tag version in the name + AMD64_TAG="cuda-amd64-${{ github.ref_name }}" + ARM64_TAG="cuda-arm64-${{ github.ref_name }}" + TARGET_TAG="cuda-${{ github.ref_name }}" + else + # Main branch build - use simple arch tags + AMD64_TAG="cuda-amd64" + ARM64_TAG="cuda-arm64" + TARGET_TAG="cuda" fi + + # Create the manifest with the correct tag names + echo "Creating CUDA manifest using $AMD64_TAG and $ARM64_TAG" + docker buildx imagetools create --tag ghcr.io/${{ github.repository }}:${TARGET_TAG} \ + ghcr.io/${{ github.repository }}:${AMD64_TAG} \ + ghcr.io/${{ github.repository }}:${ARM64_TAG} From 163acd4e6df4f98ced7ea3985825652bb6da1056 Mon Sep 17 00:00:00 2001 From: Edoardo Baldi Date: Tue, 6 May 2025 14:27:56 +0200 Subject: [PATCH 10/12] [skip ci] Remove empty param In Docker meta step "prefix=" was empty --- .github/workflows/docker-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 73e4388f..497bde38 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -58,7 +58,7 @@ jobs: type=raw,value=${{ matrix.variant }}-${{ matrix.arch }},enable=${{ github.ref == 'refs/heads/main' }} type=raw,value=${{ matrix.variant }}-${{ matrix.arch }}-pr-${{ github.event.pull_request.number }},enable=${{ github.event_name == 'pull_request' }} type=raw,value=${{ matrix.variant }}-${{ matrix.arch }}-${{ github.ref_name }},enable=${{ startsWith(github.ref, 'refs/tags/') }} - type=raw,value=${{ matrix.variant }}-${{ matrix.arch }}-sha-${{ github.sha }},prefix= + type=raw,value=${{ matrix.variant }}-${{ matrix.arch }}-sha-${{ github.sha }} - name: Build and push uses: docker/build-push-action@v6 From 2a377154ba6f837df2c7c7ea7a0e0f8b34062be4 Mon Sep 17 00:00:00 2001 From: Edoardo Baldi Date: Tue, 6 May 2025 14:42:36 +0200 Subject: [PATCH 11/12] Update README --- README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c4740fd9..5cd1262f 100644 --- a/README.md +++ b/README.md @@ -84,18 +84,30 @@ jupyter lab 3. Open PowerShell: Once Docker Desktop is installed, open PowerShell on your Windows machine. You can do this by pressing the "Windows" key and typing "PowerShell" in the search bar. -4. Pull the Docker image: In PowerShell, run the following command to pull the "empascientificit/python-tutorial" Docker image: +4. Pull the Docker image: In PowerShell, run the following command to pull the Docker image: ```console docker pull ghcr.io/empa-scientific-it/python-tutorial:latest ``` +> **Note:** +> +> The `latest` tag points to the CPU-only variant of the image, which is optimized for size and compatibility. If you have a CUDA-compatible GPU and want to use GPU acceleration for PyTorch operations, you can use the CUDA-enabled variant by replacing `latest` with `cuda`: +> +> ```console +> docker pull ghcr.io/empa-scientific-it/python-tutorial:cuda +> ``` + 5. Run the Docker container: Once the image is downloaded, run the following command to start a Docker container from the image: ```console docker run -p 8888:8888 --name python_tutorial -v /path/to/python-tutorial:/home/jovyan/python-tutorial ghcr.io/empa-scientific-it/python-tutorial:latest jupyter lab --ip 0.0.0.0 --no-browser ``` +> **Note** +> +> If you pulled the CUDA variant, replace `:latest` with `:cuda` in the command above. + Replace `/path/to/python-tutorial` with the path to the folder you created in step 2, for example `C:/Users/yourusername/Desktop/python-tutorial`. > **Note** From 1ca67719a33c90b2e538f3790c71e689acb52e74 Mon Sep 17 00:00:00 2001 From: Edoardo Baldi Date: Tue, 6 May 2025 14:50:52 +0200 Subject: [PATCH 12/12] [skip ci] Update README --- README.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 5cd1262f..f97080b4 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ You should now create a new environment with `conda`: conda env create -f binder/environment.yml ``` -> **Warning** +> [!WARNING] > > If you are on Windows and using Command Prompt or the PowerShell, please make sure to adjust the paths in the commands above accordingly. @@ -74,7 +74,7 @@ jupyter lab ### 2. With Docker -> **Note** +> [!NOTE] > > The following instructions are for Windows. With minor changes, the steps work on macOS or Linux as well. @@ -90,7 +90,7 @@ jupyter lab docker pull ghcr.io/empa-scientific-it/python-tutorial:latest ``` -> **Note:** +> [!NOTE] > > The `latest` tag points to the CPU-only variant of the image, which is optimized for size and compatibility. If you have a CUDA-compatible GPU and want to use GPU acceleration for PyTorch operations, you can use the CUDA-enabled variant by replacing `latest` with `cuda`: > @@ -98,26 +98,30 @@ docker pull ghcr.io/empa-scientific-it/python-tutorial:latest > docker pull ghcr.io/empa-scientific-it/python-tutorial:cuda > ``` +> [!IMPORTANT] +> +> Using the CUDA variant requires a NVIDIA GPU with compatible drivers properly installed and configured for Docker. See [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html) for setup instructions. + 5. Run the Docker container: Once the image is downloaded, run the following command to start a Docker container from the image: ```console docker run -p 8888:8888 --name python_tutorial -v /path/to/python-tutorial:/home/jovyan/python-tutorial ghcr.io/empa-scientific-it/python-tutorial:latest jupyter lab --ip 0.0.0.0 --no-browser ``` -> **Note** +> [!NOTE] > > If you pulled the CUDA variant, replace `:latest` with `:cuda` in the command above. Replace `/path/to/python-tutorial` with the path to the folder you created in step 2, for example `C:/Users/yourusername/Desktop/python-tutorial`. -> **Note** +> [!NOTE] > -> The above command will **mirror** the content of your local folder (e.g., `C:/Users/yourusername/Desktop/python-tutorial`) to the `work/` folder **inside the container**. In this way, every file or folder you copy or create into `work/` will be saved on your machine, and will remain there **even if you stop Docker**. +> The above command will **mirror** the content of your local folder (e.g., `C:/Users/yourusername/Desktop/python-tutorial`) to the `~/python-tutorial` folder **inside the container**. In this way, every file or folder you copy or create into `~/python-tutorial` will be saved on your machine, and will remain there **even if you stop Docker**. 6. Access the Jupyter Notebook: Open a web browser and navigate to `http://localhost:8888/lab`. You should see the Jupyter Notebook interface. Enter the token provided in the PowerShell console to access the notebook. Alternatively, you can directly click on the link that appears in the PowerShell after the container has started. You can now use the Jupyter in the Docker container to run the python-tutorial. When you're done, you can stop the container by pressing `Ctrl+C` in the PowerShell console. -> **Note** +> [!NOTE] > > If you want to restart the container, you can simply run the command `docker container start python_tutorial`.