Skip to content

Commit 0adac7a

Browse files
despadamdespadam
committed
Merge branch 'main' into update_11_functional_programming
2 parents 67d249c + 176e30b commit 0adac7a

File tree

6 files changed

+295
-36
lines changed

6 files changed

+295
-36
lines changed

‎.dockerignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
.git
2+
slides/
3+
.DS_Store
4+
__pycache__/

‎.github/workflows/build-docker-image.yml‎

Lines changed: 0 additions & 28 deletions
This file was deleted.

‎.github/workflows/docker-build.yml‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
---
2+
name: Build Tutorial Container
3+
4+
on:
5+
push:
6+
branches:
7+
- main
8+
paths-ignore:
9+
- "*.md"
10+
- slides/**
11+
- images/**
12+
- .gitignore
13+
tags:
14+
- "v*.*"
15+
- "v*.*.*"
16+
pull_request:
17+
paths-ignore:
18+
- "*.md"
19+
- slides/**
20+
- images/**
21+
- .gitignore
22+
workflow_dispatch:
23+
24+
jobs:
25+
build-and-push:
26+
runs-on: ubuntu-latest
27+
permissions:
28+
packages: write
29+
strategy:
30+
matrix:
31+
arch: [amd64, arm64]
32+
variant: [cpu, cuda]
33+
fail-fast: false
34+
steps:
35+
- name: Checkout code
36+
uses: actions/checkout@v4
37+
38+
- name: Set up QEMU
39+
uses: docker/setup-qemu-action@v3
40+
41+
- name: Setup Docker Buildx
42+
uses: docker/setup-buildx-action@v3
43+
44+
- name: Log in to GHCR
45+
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
46+
uses: docker/login-action@v3
47+
with:
48+
registry: ghcr.io
49+
username: ${{ github.actor }}
50+
password: ${{ secrets.GITHUB_TOKEN }}
51+
52+
- name: Docker meta
53+
id: meta
54+
uses: docker/metadata-action@v5
55+
with:
56+
images: ghcr.io/${{ github.repository }}
57+
tags: |
58+
type=raw,value=${{ matrix.variant }}-${{ matrix.arch }},enable=${{ github.ref == 'refs/heads/main' }}
59+
type=raw,value=${{ matrix.variant }}-${{ matrix.arch }}-pr-${{ github.event.pull_request.number }},enable=${{ github.event_name == 'pull_request' }}
60+
type=raw,value=${{ matrix.variant }}-${{ matrix.arch }}-${{ github.ref_name }},enable=${{ startsWith(github.ref, 'refs/tags/') }}
61+
type=raw,value=${{ matrix.variant }}-${{ matrix.arch }}-sha-${{ github.sha }}
62+
63+
- name: Build and push
64+
uses: docker/build-push-action@v6
65+
with:
66+
context: .
67+
platforms: linux/${{ matrix.arch }}
68+
push: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
69+
tags: ${{ steps.meta.outputs.tags }}
70+
provenance: false
71+
build-args: |
72+
PYTORCH_VARIANT=${{ matrix.variant }}
73+
74+
create-manifests:
75+
needs: build-and-push
76+
runs-on: ubuntu-latest
77+
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
78+
permissions:
79+
packages: write
80+
steps:
81+
- name: Set up QEMU
82+
uses: docker/setup-qemu-action@v3
83+
84+
- name: Setup Docker Buildx
85+
uses: docker/setup-buildx-action@v3
86+
with:
87+
driver-opts: |
88+
image=moby/buildkit:latest
89+
network=host
90+
91+
- name: Log in to GHCR
92+
uses: docker/login-action@v3
93+
with:
94+
registry: ghcr.io
95+
username: ${{ github.actor }}
96+
password: ${{ secrets.GITHUB_TOKEN }}
97+
98+
- name: Create and push CPU manifest
99+
run: |
100+
# Determine the correct tag suffixes based on the event type
101+
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
102+
# PR build - use the commit SHA for more predictable references
103+
AMD64_TAG="cpu-amd64-sha-${{ github.sha }}"
104+
ARM64_TAG="cpu-arm64-sha-${{ github.sha }}"
105+
TARGET_TAG="cpu-sha-${{ github.sha }}"
106+
elif [[ "${{ startsWith(github.ref, 'refs/tags/') }}" == "true" ]]; then
107+
# Tag build - use tag version in the name
108+
AMD64_TAG="cpu-amd64-${{ github.ref_name }}"
109+
ARM64_TAG="cpu-arm64-${{ github.ref_name }}"
110+
TARGET_TAG="cpu-${{ github.ref_name }}"
111+
else
112+
# Main branch build - use simple arch tags
113+
AMD64_TAG="cpu-amd64"
114+
ARM64_TAG="cpu-arm64"
115+
TARGET_TAG="cpu"
116+
fi
117+
118+
# Create the manifest with the correct tag names
119+
echo "Creating CPU manifest using $AMD64_TAG and $ARM64_TAG"
120+
docker buildx imagetools create --tag ghcr.io/${{ github.repository }}:${TARGET_TAG} \
121+
ghcr.io/${{ github.repository }}:${AMD64_TAG} \
122+
ghcr.io/${{ github.repository }}:${ARM64_TAG}
123+
124+
# If on main branch, also tag as latest
125+
if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
126+
docker buildx imagetools create --tag ghcr.io/${{ github.repository }}:latest \
127+
ghcr.io/${{ github.repository }}:${AMD64_TAG} \
128+
ghcr.io/${{ github.repository }}:${ARM64_TAG}
129+
fi
130+
131+
- name: Create and push CUDA manifest
132+
run: |
133+
# Determine the correct tag suffixes based on the event type
134+
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
135+
# PR build - use the commit SHA for more predictable references
136+
AMD64_TAG="cuda-amd64-sha-${{ github.sha }}"
137+
ARM64_TAG="cuda-arm64-sha-${{ github.sha }}"
138+
TARGET_TAG="cuda-sha-${{ github.sha }}"
139+
elif [[ "${{ startsWith(github.ref, 'refs/tags/') }}" == "true" ]]; then
140+
# Tag build - use tag version in the name
141+
AMD64_TAG="cuda-amd64-${{ github.ref_name }}"
142+
ARM64_TAG="cuda-arm64-${{ github.ref_name }}"
143+
TARGET_TAG="cuda-${{ github.ref_name }}"
144+
else
145+
# Main branch build - use simple arch tags
146+
AMD64_TAG="cuda-amd64"
147+
ARM64_TAG="cuda-arm64"
148+
TARGET_TAG="cuda"
149+
fi
150+
151+
# Create the manifest with the correct tag names
152+
echo "Creating CUDA manifest using $AMD64_TAG and $ARM64_TAG"
153+
docker buildx imagetools create --tag ghcr.io/${{ github.repository }}:${TARGET_TAG} \
154+
ghcr.io/${{ github.repository }}:${AMD64_TAG} \
155+
ghcr.io/${{ github.repository }}:${ARM64_TAG}

‎Dockerfile‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
# Stage 1: Build environment
2+
FROM quay.io/jupyter/minimal-notebook:latest AS builder
3+
4+
# Define build argument for PyTorch variant (cpu or cuda)
5+
ARG PYTORCH_VARIANT=cpu
6+
7+
# Switch to root user to install additional dependencies
8+
USER root
9+
RUN apt-get update && \
10+
apt-get install -y --no-install-recommends \
11+
build-essential \
12+
gcc \
13+
g++ \
14+
libffi-dev \
15+
libgl1 && \
16+
apt-get clean && \
17+
rm -rf /var/lib/apt/lists/*
18+
19+
# Switch back to the default notebook user
20+
USER ${NB_UID}
21+
22+
# Set up the Conda environment
23+
COPY docker/environment.yml /tmp/environment.yml
24+
RUN mamba env update -n base -f /tmp/environment.yml && \
25+
# Force remove any existing PyTorch installations first
26+
pip uninstall -y torch torchvision && \
27+
# Install PyTorch packages without cache - conditionally based on variant
28+
if [ "$PYTORCH_VARIANT" = "cpu" ]; then \
29+
echo "Installing CPU-only PyTorch" && \
30+
pip install --no-cache-dir --force-reinstall torch torchvision --index-url https://download.pytorch.org/whl/cpu; \
31+
else \
32+
echo "Installing CUDA-enabled PyTorch" && \
33+
pip install --no-cache-dir --force-reinstall torch torchvision; \
34+
fi && \
35+
# Clean up all package caches to reduce image size
36+
mamba clean --all -f -y && \
37+
# Remove pip cache
38+
rm -rf ~/.cache/pip && \
39+
fix-permissions "${CONDA_DIR}" && \
40+
fix-permissions "/home/${NB_USER}"
41+
42+
# Stage 2: Runtime environment - creates a lighter final image
43+
FROM quay.io/jupyter/minimal-notebook:latest
44+
45+
# Inherit build argument for image labeling
46+
ARG PYTORCH_VARIANT=cpu
47+
48+
# Metadata labels
49+
LABEL org.opencontainers.image.title="Python Tutorial"
50+
LABEL org.opencontainers.image.description="A containerized Python tutorial environment with Jupyter Lab."
51+
LABEL org.opencontainers.image.authors="Empa Scientific IT <scientificit@empa.ch>"
52+
LABEL org.opencontainers.image.url="https://github.com/empa-scientific-it/python-tutorial"
53+
LABEL org.opencontainers.image.source="https://github.com/empa-scientific-it/python-tutorial"
54+
LABEL org.opencontainers.image.version="1.0.0"
55+
LABEL org.opencontainers.image.licenses="MIT"
56+
LABEL org.opencontainers.image.variant="pytorch-${PYTORCH_VARIANT}"
57+
58+
# Switch to root user to install minimal dependencies
59+
USER root
60+
RUN apt-get update && \
61+
apt-get install -y --no-install-recommends \
62+
libgl1 && \
63+
apt-get clean && \
64+
rm -rf /var/lib/apt/lists/*
65+
66+
# Switch back to the default notebook user
67+
USER ${NB_UID}
68+
69+
# Copy the conda environment from the builder stage
70+
COPY --from=builder ${CONDA_DIR} ${CONDA_DIR}
71+
72+
# Copy home directory with configurations
73+
COPY --from=builder --chown=${NB_UID}:${NB_GID} /home/${NB_USER} /home/${NB_USER}
74+
75+
# Prepare IPython configuration
76+
COPY --chown=${NB_UID}:${NB_GID} binder/ipython_config.py ${HOME}/.ipython/profile_default/
77+
78+
# Set the working directory to user's home (repository will be cloned here by Renku)
79+
WORKDIR /home/${NB_USER}
80+
81+
# Use the default ENTRYPOINT from the base image to start Jupyter Lab
82+
ENTRYPOINT ["tini", "-g", "--", "start.sh"]

‎README.md‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
[![Renku](https://renkulab.io/renku-badge.svg)](https://renkulab.io/v2/projects/empa-scientific-it/empa-it-python-tutorial/sessions/01JRT57GABCS15JB5NGNTQVTRB/start)
44
[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/empa-scientific-it/python-tutorial.git/main?labpath=00_index.ipynb)
55

6-
[![Build Docker container](https://github.com/empa-scientific-it/python-tutorial/actions/workflows/build-docker-image.yml/badge.svg)](https://github.com/empa-scientific-it/python-tutorial/actions/workflows/build-docker-image.yml)
6+
[![Build Docker container](https://github.com/empa-scientific-it/python-tutorial/actions/workflows/docker-build.yml/badge.svg)](https://github.com/empa-scientific-it/python-tutorial/actions/workflows/docker-build.yml)
77

88
[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://github.com/codespaces/new?hide_repo_select=true&ref=main&repo=593234387)
99

@@ -48,7 +48,7 @@ You should now create a new environment with `conda`:
4848
conda env create -f binder/environment.yml
4949
```
5050

51-
> **Warning**
51+
> [!WARNING]
5252
>
5353
> If you are on Windows and using Command Prompt or the PowerShell, please make sure to adjust the paths in the commands above accordingly.
5454
@@ -74,7 +74,7 @@ jupyter lab
7474

7575
### 2. With Docker
7676

77-
> **Note**
77+
> [!NOTE]
7878
>
7979
> The following instructions are for Windows. With minor changes, the steps work on macOS or Linux as well.
8080
@@ -84,28 +84,44 @@ jupyter lab
8484

8585
3. Open PowerShell: Once Docker Desktop is installed, open PowerShell on your Windows machine. You can do this by pressing the "Windows" key and typing "PowerShell" in the search bar.
8686

87-
4. Pull the Docker image: In PowerShell, run the following command to pull the "empascientificit/python-tutorial" Docker image:
87+
4. Pull the Docker image: In PowerShell, run the following command to pull the Docker image:
8888

8989
```console
9090
docker pull ghcr.io/empa-scientific-it/python-tutorial:latest
9191
```
9292

93+
> [!NOTE]
94+
>
95+
> The `latest` tag points to the CPU-only variant of the image, which is optimized for size and compatibility. If you have a CUDA-compatible GPU and want to use GPU acceleration for PyTorch operations, you can use the CUDA-enabled variant by replacing `latest` with `cuda`:
96+
>
97+
> ```console
98+
> docker pull ghcr.io/empa-scientific-it/python-tutorial:cuda
99+
> ```
100+
101+
> [!IMPORTANT]
102+
>
103+
> Using the CUDA variant requires a NVIDIA GPU with compatible drivers properly installed and configured for Docker. See [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html) for setup instructions.
104+
93105
5. Run the Docker container: Once the image is downloaded, run the following command to start a Docker container from the image:
94106
95107
```console
96-
docker run -p 8888:8888 --name python_tutorial -v /path/to/python-tutorial:/home/jovyan/work ghcr.io/empa-scientific-it/python-tutorial:latest jupyter lab --ip 0.0.0.0 --no-browser
108+
docker run -p 8888:8888 --name python_tutorial -v /path/to/python-tutorial:/home/jovyan/python-tutorial ghcr.io/empa-scientific-it/python-tutorial:latest jupyter lab --ip 0.0.0.0 --no-browser
97109
```
98110
111+
> [!NOTE]
112+
>
113+
> If you pulled the CUDA variant, replace `:latest` with `:cuda` in the command above.
114+
99115
Replace `/path/to/python-tutorial` with the path to the folder you created in step 2, for example `C:/Users/yourusername/Desktop/python-tutorial`.
100116

101-
> **Note**
117+
> [!NOTE]
102118
>
103-
> The above command will **mirror** the content of your local folder (e.g., `C:/Users/yourusername/Desktop/python-tutorial`) to the `work/` folder **inside the container**. In this way, every file or folder you copy or create into `work/` will be saved on your machine, and will remain there **even if you stop Docker**.
119+
> The above command will **mirror** the content of your local folder (e.g., `C:/Users/yourusername/Desktop/python-tutorial`) to the `~/python-tutorial` folder **inside the container**. In this way, every file or folder you copy or create into `~/python-tutorial` will be saved on your machine, and will remain there **even if you stop Docker**.
104120
105121
6. Access the Jupyter Notebook: Open a web browser and navigate to `http://localhost:8888/lab`. You should see the Jupyter Notebook interface. Enter the token provided in the PowerShell console to access the notebook. Alternatively, you can directly click on the link that appears in the PowerShell after the container has started.
106122

107123
You can now use the Jupyter in the Docker container to run the python-tutorial. When you're done, you can stop the container by pressing `Ctrl+C` in the PowerShell console.
108124

109-
> **Note**
125+
> [!NOTE]
110126
>
111127
> If you want to restart the container, you can simply run the command `docker container start python_tutorial`.

‎docker/environment.yml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
---
2+
name: base
3+
channels:
4+
- conda-forge
5+
dependencies:
6+
- pip
7+
- pip:
8+
- numpy
9+
- matplotlib
10+
- pandas
11+
- ipywidgets
12+
- ipynbname
13+
- jupyterlab
14+
- pytest
15+
- pytest-timeout
16+
- markdown
17+
- pre-commit
18+
- geostatspy
19+
- gstools
20+
- scikit-learn
21+
- attrs
22+
- multiprocess
23+
- openai
24+
- tenacity
25+
- markdown2
26+
- python-dotenv
27+
- pillow
28+
- opencv-python
29+
- albumentations
30+
- grad-cam

0 commit comments

Comments
 (0)