Merge pull request #1871 from codalab/podmanUpdates

Didayolo · web-flow · commit 30791eb69806 · 2025-10-07T15:00:33.000+02:00
Podman updates &amp; More Competition container security
diff --git a/Containerfile.compute_worker_podman b/Containerfile.compute_worker_podman
@@ -1,44 +1,34 @@
-FROM fedora:37
+FROM fedora:42
 
 # Include deps
 RUN dnf -y update && \
-    # https://bugzilla.redhat.com/show_bug.cgi?id=1995337#c3
-    rpm --setcaps shadow-utils 2>/dev/null && \
     dnf -y install podman fuse-overlayfs python3.9 \
         --exclude container-selinux && \
     dnf clean all && \
     rm -rf /var/cache /var/log/dnf* /var/log/yum.*
 
-# Setup user
-RUN useradd worker; \
-echo -e "worker:1:999\nworker:1001:64535" > /etc/subuid; \
-echo -e "worker:1:999\nworker:1001:64535" > /etc/subgid;
+# Copy the podman-connections file to allow for podman inside the container to connect to podman on the host, running containers alongside podman instead of inside
+COPY podman/podman-connections.json /root/.config/containers/podman-connections.json
 
 # Copy over the podman container configuration
 COPY podman/containers.conf /etc/containers/containers.conf
-COPY podman/worker-containers.conf /home/worker/.config/containers/containers.conf
+COPY podman/worker-containers.conf /root/.config/containers/containers.conf
 
 # Copy over the podman storage configuration
-COPY podman/worker-storage.conf /home/worker/.config/containers/storage.conf
+COPY podman/worker-storage.conf /root/.config/containers/storage.conf
 
-RUN mkdir -p /home/worker/.local/share/containers && \
-    chown worker:worker -R /home/worker && \
-    chmod 644 /etc/containers/containers.conf
+RUN mkdir -p /root/.local/share/containers
 
 # Copy & modify the defaults to provide reference if runtime changes needed.
 # Changes here are required for running with fuse-overlay storage inside container.
 RUN sed -e 's|^#mount_program|mount_program|g' \
-           -e '/additionalimage.*/a "/var/lib/shared",' \
-           -e 's|^mountopt[[:space:]]*=.*$|mountopt = "nodev,fsync=0"|g' \
-           /usr/share/containers/storage.conf \
-           > /etc/containers/storage.conf
+        -e '/additionalimage.*/a "/var/lib/shared",' \
+        -e 's|^mountopt[[:space:]]*=.*$|mountopt = "nodev,fsync=0"|g' \
+        /usr/share/containers/storage.conf \
+        > /etc/containers/storage.conf
 
 # Add volume for containers
-VOLUME /home/worker/.local/share/containers
-
-# Create directory for tmp space
-RUN mkdir /codabench && \
-    chown worker:worker /codabench
+VOLUME /root/.local/share/containers
 
 # Set up podman registry for dockerhub
 RUN echo -e "[registries.search]\nregistries = ['docker.io']\n" > /etc/containers/registries.conf
@@ -47,18 +37,18 @@ RUN echo -e "[registries.search]\nregistries = ['docker.io']\n" > /etc/container
 ENV PYTHONUNBUFFERED 1
 ENV CONTAINER_ENGINE_EXECUTABLE podman
 
-WORKDIR /home/worker/compute_worker
-
-ADD compute_worker/ /home/worker/compute_worker
+WORKDIR /root/compute_worker
 
-RUN chown worker:worker -R /home/worker/compute_worker
+ADD compute_worker/ /root/compute_worker
 
-RUN curl -sSL https://install.python-poetry.org | python3.9 -
+RUN curl -sSL https://install.python-poetry.org | python3.9 - --version 1.8.3
 # Poetry location so future commands (below) work
 ENV PATH $PATH:/root/.local/bin
+
 # Want poetry to use system python of docker container
 RUN poetry config virtualenvs.create false
 RUN poetry config virtualenvs.in-project false
+
 # So we get 3.9
 RUN poetry config virtualenvs.prefer-active-python true
 COPY ./compute_worker/pyproject.toml ./
@@ -69,4 +59,4 @@ CMD celery -A compute_worker worker \
     -l info \
     -Q compute-worker \
     -n compute-worker@%n \
-    --concurrency=1
+    --concurrency=1
diff --git a/Containerfile.compute_worker_podman_gpu b/Containerfile.compute_worker_podman_gpu
@@ -1,75 +1,8 @@
-FROM fedora:37
+FROM codalab/codabench_worker_podman:latest
 
 # Include deps
-RUN curl -s -L https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo | tee /etc/yum.repos.d/cuda.repo && \
-    curl -s -L https://nvidia.github.io/nvidia-docker/rhel9.0/nvidia-docker.repo | tee /etc/yum.repos.d/nvidia-docker.repo && \
-    rpm -Uvh http://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-$(rpm -E %fedora).noarch.rpm && \
-    rpm -Uvh http://download1.rpmfusion.org/nonfree/fedora/rpmfusion-nonfree-release-$(rpm -E %fedora).noarch.rpm && \
+RUN dnf -y config-manager addrepo --from-repofile=https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo && \
     dnf -y update && \
-    dnf module install -y nvidia-driver:latest-dkms && \
-    dnf -y install podman fuse-overlayfs python3.9 nvidia-container-runtime nvidia-container-toolkit \
-     cuda --exclude container-selinux && \
+    dnf -y install nvidia-container-runtime nvidia-container-toolkit --exclude container-selinux && \
     dnf clean all && \
-    rm -rf /var/cache /var/log/dnf* /var/log/yum.*
-
-# Setup user
-RUN useradd worker; \
-echo -e "worker:1:999\nworker:1001:64535" > /etc/subuid; \
-echo -e "worker:1:999\nworker:1001:64535" > /etc/subgid;
-
-# Copy over the podman container configuration
-COPY podman/containers.conf /etc/containers/containers.conf
-COPY podman/worker-containers.conf /home/worker/.config/containers/containers.conf
-
-# Copy over the podman storage configuration
-COPY podman/worker-storage.conf /home/worker/.config/containers/storage.conf
-
-RUN mkdir -p /home/worker/.local/share/containers && \
-    chown worker:worker -R /home/worker && \
-    chmod 644 /etc/containers/containers.conf
-
-# Copy & modify the defaults to provide reference if runtime changes needed.
-# Changes here are required for running with fuse-overlay storage inside container.
-RUN sed -e 's|^#mount_program|mount_program|g' \
-           -e '/additionalimage.*/a "/var/lib/shared",' \
-           -e 's|^mountopt[[:space:]]*=.*$|mountopt = "nodev,fsync=0"|g' \
-           /usr/share/containers/storage.conf \
-           > /etc/containers/storage.conf; sed -i 's/^#no-cgroups = false/no-cgroups = true/;' /etc/nvidia-container-runtime/config.toml
-
-
-# Add volume for containers
-VOLUME /home/worker/.local/share/containers
-
-# This makes output not buffer and return immediately, nice for seeing results in stdout
-ENV PYTHONUNBUFFERED 1
-ENV CONTAINER_ENGINE_EXECUTABLE podman
-
-# Create directory for tmp space
-RUN mkdir /codabench && \
-    chown worker:worker /codabench && \
-# Set up podman registry for dockerhub
-    echo -e "[registries.search]\nregistries = ['docker.io']\n" > /etc/containers/registries.conf && \
-
-WORKDIR /home/worker/compute_worker
-
-ADD compute_worker/ /home/worker/compute_worker
-
-RUN curl -sSL https://install.python-poetry.org | python3.9 -
-# Poetry location so future commands (below) work
-ENV PATH $PATH:/root/.local/bin
-# Want poetry to use system python of docker container
-RUN poetry config virtualenvs.create false
-RUN poetry config virtualenvs.in-project false
-# So we get 3.9
-RUN poetry config virtualenvs.prefer-active-python true
-COPY ./compute_worker/pyproject.toml ./
-COPY ./compute_worker/poetry.lock ./
-RUN poetry install
-
-RUN chown worker:worker -R /home/worker/compute_worker
-
-CMD nvidia-smi && celery -A compute_worker worker \
-    -l info \
-    -Q compute-worker \
-    -n compute-worker@%n \
-    --concurrency=1
+    rm -rf /var/cache /var/log/dnf* /var/log/yum.*
diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py
@@ -622,9 +622,6 @@ async def _run_program_directory(self, program_dir, kind):
                 logger.info(
                     "Program directory missing metadata, assuming it's going to be handled by ingestion"
                 )
-                # Copy submission files into prediction output
-                # This is useful for results submissions but wrongly uses storage
-                shutil.copytree(program_dir, self.output_dir)
                 return
             else:
                 raise SubmissionException("Program directory missing 'metadata.yaml/metadata'")
@@ -660,21 +657,28 @@ async def _run_program_directory(self, program_dir, kind):
             # Don't allow subprocesses to raise privileges
             '--security-opt=no-new-privileges',
 
-            # Set the volumes
-            '-v', f'{self._get_host_path(program_dir)}:/app/program',
-            '-v', f'{self._get_host_path(self.output_dir)}:/app/output',
+            # Set the volumes: ro for Read Only,  z to allow multiple containers to access the volume (useful for podman)
+            '-v', f'{self._get_host_path(program_dir)}:/app/program:z',
+            '-v', f'{self._get_host_path(self.output_dir)}:/app/output:z',
             '-v', f'{self.data_dir}:/app/data:ro',
 
             # Start in the right directory
             '-w', '/app/program',
-
+            
+            # Set the user namespace mode for the container
+            '--userns', 'host',
+            # Drop all capabilities
+            '--cap-drop', 'all', 
             # Don't buffer python output, so we don't lose any
             '-e', 'PYTHONUNBUFFERED=1',
         ]
 
         # GPU or not
-        if os.environ.get("USE_GPU"):
+        if os.environ.get("USE_GPU") and CONTAINER_ENGINE_EXECUTABLE=='docker':
             engine_cmd.extend(['--gpus', 'all'])
+        # For podman specifically
+        if os.environ.get("USE_GPU") and CONTAINER_ENGINE_EXECUTABLE=='podman':
+            engine_cmd.extend(['--device', 'nvidia.com/gpu=all'])
 
         if kind == 'ingestion':
             # program here is either scoring program or submission, depends on if this ran during Prediction or Scoring
diff --git a/documentation/docs/Newsletters_Archive/CodaLab-in-2024.md b/documentation/docs/Newsletters_Archive/CodaLab-in-2024.md
@@ -19,7 +19,7 @@ Contributors community is very active with **143 pull requests** this year. Sinc
 ## Introducing Codabench
 [Codabench](https://codabench.org/), the modernized version of [CodaLab](https://codalab.lisn.fr/), was released in summer 2023, and [presented at JCAD days](https://www.canal-u.tv/chaines/jcad/codalab-competitions-and-codabench-open-source-platforms-to-organize-scientific) in November 2024! Codabench platform software is now concentrating all development effort of the community. In addition to CodaLab features, it offers improved performance, live logs, more transparency, data-centric benchmarks and more!
 
-We warmly encourage you to use [codabench.org](https://codabench.org/) for all your new competitions and benchmarks. Note that CodaLab bundles are compatible with Codabench, easing the transition, as explained in the following Wiki page: [How-to-transition-from-CodaLab-to-Codabench](https://github.com/codalab/codabench/wiki/How-to-transition-from-CodaLab-to-Codabench%3F)
+We warmly encourage you to use [codabench.org](https://codabench.org/) for all your new competitions and benchmarks. Note that CodaLab bundles are compatible with Codabench, easing the transition, as explained in the following Wiki page: [How to transition from CodaLab to Codabench](../Organizers/Benchmark_Creation/How-to-transition-from-CodaLab-to-Codabench.md)
 
 CodaLab and Codabench are hosted on servers located at [Paris-Saclay university](https://www.universite-paris-saclay.fr/), maintained by [LISN lab](http://lisn.upsaclay.fr/).
 
diff --git a/documentation/docs/Organizers/Benchmark_Creation/Advanced-Tutorial.md b/documentation/docs/Organizers/Benchmark_Creation/Advanced-Tutorial.md
@@ -1,5 +1,5 @@
 
-Here is an advanced tutorial. If you are new to CodaBench, please refer to [get started tutorial](https://github.com/codalab/codabench/wiki/Getting-started-with-Codabench) first.
+Here is an advanced tutorial. If you are new to CodaBench, please refer to [get started tutorial](Getting-started-with-Codabench.md) first.
 In this article, you'll learn how to use more advanced features and how to create benchmarks using either the editor or bundles.
 Before proceeding to our tutorial, make sure you have registered for an account on the [Codabench](https://www.codabench.org/) website.
 
diff --git a/documentation/docs/Organizers/Benchmark_Creation/Competition-Bundle-Structure.md b/documentation/docs/Organizers/Benchmark_Creation/Competition-Bundle-Structure.md
@@ -108,9 +108,9 @@ The scoring program outputs a `scores.json` file containing the results for each
 {"accuracy": 0.886, "duration": 42.4}
 ```
 
-The keys should match the leaderboard columns keys defined in [the `competition.yaml` file](https://github.com/codalab/codabench/wiki/Yaml-Structure#leaderboards).
+The keys should match the leaderboard columns keys defined in [the `competition.yaml` file](Yaml-Structure.md#leaderboards).
 
-The scoring program can also output detailed results as an HTML file for each submission. [Click here for more information](https://github.com/codalab/codabench/wiki/Detailed-Results-and-Visualizations).
+The scoring program can also output detailed results as an HTML file for each submission. [Click here for more information](Detailed-Results-and-Visualizations.md).
 
 ### Ingestion Program
 The ingestion program is a file that gets ran to generate the predictions from the submissions if necessary. This is usually a python script or a script in another language, but it can generally be anything.
@@ -122,7 +122,7 @@ Example: Here's what an ingestion `metdata.yaml` might look like this:
 command: python3 /app/program/ingestion.py /app/input_data/ /app/output/ /app/program /app/ingested_program
 ```
 
-Just like the example above, this specifies we're using python to run our ingestion program. Please note that it is not necessary to pass these directories as arguments to the programs, but it can be convenient. More information about the folder layout [here](https://github.com/codalab/codabench/wiki/Submission-Docker-Container-Layout#submission-container).
+Just like the example above, this specifies we're using python to run our ingestion program. Please note that it is not necessary to pass these directories as arguments to the programs, but it can be convenient. More information about the folder layout [here](../../Developers_and_Administrators/Submission-Docker-Container-Layout.md#submission-container).
 
 ### Input Data
 This is usually the test data used to generate predictions from a user's code submission when paired with an ingestion program.
diff --git a/documentation/docs/Organizers/Benchmark_Creation/Competition-Creation:-Form.md b/documentation/docs/Organizers/Benchmark_Creation/Competition-Creation:-Form.md
@@ -43,7 +43,7 @@ By clicking `Add phase`, you should be presented with a modal for phase creation
 - Name: The name of your phase
 - Start: The start day of your phase
 - End: The end day of your phase
-- Tasks: Here you can assign one or multiple task objects to your phase. Tasks are problems that the submission should be solving. For more information, see the explanation on competition structure [here](https://github.com/codalab/codabench/wiki/Competition-Bundle-Structure#what-is-a-competition): If you don't have any tasks created yet, click the green button at the bottom of the new phase modal titled `Manage Tasks/Datasets`
+- Tasks: Here you can assign one or multiple task objects to your phase. Tasks are problems that the submission should be solving. For more information, see the explanation on competition structure [here](Competition-Bundle-Structure.md#what-is-a-competition): If you don't have any tasks created yet, click the green button at the bottom of the new phase modal titled `Manage Tasks/Datasets`
 - Description: The description of your phase
 
 *Advanced*
diff --git a/documentation/docs/Organizers/Benchmark_Creation/How-to-transition-from-CodaLab-to-Codabench.md b/documentation/docs/Organizers/Benchmark_Creation/How-to-transition-from-CodaLab-to-Codabench.md
@@ -8,7 +8,7 @@
 
 - Live logs during submission processes
 - Storage quotas 
-- [Computation servers management](https://github.com/codalab/codabench/wiki/Server-status-page) for all users. 
+- [Computation servers management](../Running_a_benchmark/Server-status-page.md) for all users. 
 
 It also emphasizes on benchmarking, allowing dataset submissions and multiple leaderboard rows per user. Finally, future project development and maintenance will be focused on Codabench.
 
@@ -56,5 +56,5 @@ If you don’t have any previous competition, and want to learn how to create on
 
 ## Concluding remarks
 
-Codabench, the new version of the competition and benchmark platform CodaLab, was launched on August 2023 and is already receiving great attention. For users accustomed to CodaLab, the transition to Codabench is quick and easy. Indeed, competition bundles are back-compatible, and all that is required is to create an account on Codabench. To go further, you can refer to [Codabench’s Wiki](https://github.com/codalab/codabench/wiki/).
+Codabench, the new version of the competition and benchmark platform CodaLab, was launched on August 2023 and is already receiving great attention. For users accustomed to CodaLab, the transition to Codabench is quick and easy. Indeed, competition bundles are back-compatible, and all that is required is to create an account on Codabench. To go further, you can refer to [Codabench’s Wiki](https://wiki.codabench.org).
 
diff --git a/documentation/docs/Organizers/Running_a_benchmark/Compute-worker-installation-with-Podman.md b/documentation/docs/Organizers/Running_a_benchmark/Compute-worker-installation-with-Podman.md
@@ -1,10 +1,12 @@
 Here is the specification for compute worker installation by using Podman. 
-## Requirements for host machine
+## Requirements for the host machine
 
 We need to install Podman on the VM. We use Debian based OS, like Ubuntu. Ubuntu is recommended, because it has better Nvidia driver support. 
 
 `sudo apt install podman `
 
+After installing Podman, you will need to launch the service associated to it with `systemctl --user enable --now podman`
+
 Then, configure where Podman will download the images: Podman will use Dockerhub by adding this line into `/etc/containers/registries.conf `:
 
 `unqualified-search-registries = ["docker.io"] `
@@ -19,12 +21,13 @@ BROKER_USE_SSL=True
 CONTAINER_ENGINE_EXECUTABLE=podman
 ```
 
-**Create user for running Podman container**
-```bash
-useradd worker
-```
+You will also need to create the `codabench` folder defined in the `.env` file, as well as change its permissions to the user that is running the compute worker.
 
-!!! note "In order to use podman later to launch the computer worker, you need to have logged out completely first from whatever user you were using and log in as "worker". If you fail to do so, your environment will likely store environment variables tied to the original user and launching podman will not work."
+```bash title="In your terminal"
+sudo mkdir /codabench
+sudo mkdir /codabench/data
+sudo chown -R $(id -u):$(id -g) /codabench
+```
 
 ## For GPU compute worker VM
 
@@ -103,6 +106,10 @@ podman run --rm -it \
 ```
 The result should show as same as the command `nvidia-smi` above.
 
+You will also need to add this line in your `.env` file:
+```bash
+USE_GPU=True
+```
 
 ## Compute worker installation 
 
@@ -112,14 +119,17 @@ Run the compute worker container :
 
 ```bash
 podman run -d \
-    --env-file .env \
-    --name compute_worker \
-    --security-opt="label=disable" \
-    --device /dev/fuse --user worker \
-    --restart unless-stopped \
-    --log-opt max-size=50m \
-    --log-opt max-file=3 \
-    codalab/codabench_worker_podman:0.1
+ --volume /run/user/$(id -u)/podman/podman.sock:/run/user/1000/podman/podman.sock:U \
+ --env-file .env \
+ --name compute_worker \
+ --security-opt="label=disable" \
+ --userns host \
+ --restart unless-stopped \
+ --log-opt max-size=50m \
+ --log-opt max-file=3 \
+ --cap-drop all \
+ --volume /codabench:/codabench:U,z \
+ codalab/codabench_worker_podman:latest 
 ```
 
 ### For GPU container
@@ -129,12 +139,17 @@ Run the GPU compute worker container
 ```bash
 podman run -d \
     --env-file .env \
-    --privileged \
+    --device nvidia.com/gpu=all \
     --name gpu_compute_worker \
-    --device /dev/fuse --user worker \
+    --device /dev/fuse \
     --security-opt="label=disable" \
     --restart unless-stopped \
     --log-opt max-size=50m \
     --log-opt max-file=3 \
-    codalab/codabench_worker_podman_gpu:0.2
-```
+    --hostname ${HOSTNAME} \
+    --userns host \
+    --volume /home/codalab/worker/codabench:/codabench:z,U \
+    --cap-drop=all \
+    --volume /run/user/$(id -u)/podman/podman.sock:/run/user/1000/podman/podman.sock:U \
+    codalab/codabench_worker_podman_gpu:latest
+```
diff --git a/documentation/docs/Participants/User_Participating-in-a-Competition.md b/documentation/docs/Participants/User_Participating-in-a-Competition.md
@@ -22,7 +22,7 @@ Making a submission to a benchmark involves uploading a bundle (.zip archive) co
 
 On this page, you can make new submissions, and see previous submissions for each phase in the competition.
 
-You can also view all your submissions in the [Resources Interface](https://github.com/codalab/codabench/wiki/Task-&-Dataset-Management).
+You can also view all your submissions in the [Resources Interface](../Organizers/Running_a_benchmark/Resource-Management.md).
 
 ### Viewing Benchmark Results
 You can keep up with the progress of benchmarks you are participating in by clicking on the **Results** tab. This will display the leaderboard.
diff --git a/documentation/docs/index.md b/documentation/docs/index.md
diff --git a/documentation/mkdocs.yml b/documentation/mkdocs.yml
diff --git a/podman/containers.conf b/podman/containers.conf
diff --git a/podman/podman-connections.json b/podman/podman-connections.json
