Merge pull request #1991 from codalab/develop

Merge develop into master (1.21.0)

Author: Didayolo

.env_sample

@@ -104,3 +104,10 @@ ENABLE_SIGN_IN=True
 # GS_PUBLIC_BUCKET_NAME=public
 # GS_PRIVATE_BUCKET_NAME=private
 # GOOGLE_APPLICATION_CREDENTIALS=/app/certs/google-storage-api.json
+
+
+# -----------------------------------------------------------------------------
+# Logging (Serialized outputs the logs in JSON format)
+# -----------------------------------------------------------------------------
+LOG_LEVEL=info
+SERIALIZED=false

.github/workflows/ghPages-dev.yml

@@ -0,0 +1,25 @@
+name: github-pages-dev 
+on:
+  push:
+    branches:
+      - develop
+permissions:
+  contents: write
+jobs:
+  deploy-dev:
+    runs-on: ubuntu-latest
+    defaults:
+        run:
+            working-directory: documentation/
+    steps:
+      - uses: actions/checkout@v4
+      - name: Configure Git Credentials
+        run: |
+          git config user.name github-actions[bot]
+          git config user.email 41898282+github-actions[bot]@users.noreply.github.com
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+      - run: curl -LsSf https://astral.sh/uv/install.sh | sh
+      - run: /home/runner/.local/bin/uv sync 
+      - run: git fetch origin gh-pages --depth=1 && PDF=1 /home/runner/.local/bin/uv run mike deploy -u dev --push

.github/workflows/ghPages-prod.yml

@@ -0,0 +1,25 @@
+name: github-pages-prod
+on:
+  push:
+    tags:
+      - '*'
+permissions:
+  contents: write
+jobs:
+  deploy-prod:
+    runs-on: ubuntu-latest
+    defaults:
+        run:
+            working-directory: documentation/
+    steps:
+      - uses: actions/checkout@v4
+      - name: Configure Git Credentials
+        run: |
+          git config user.name github-actions[bot]
+          git config user.email 41898282+github-actions[bot]@users.noreply.github.com
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+      - run: curl -LsSf https://astral.sh/uv/install.sh | sh
+      - run: /home/runner/.local/bin/uv sync 
+      - run: git fetch origin gh-pages --depth=1 && PDF=1 /home/runner/.local/bin/uv run mike deploy -u  ${{ github.ref_name }} latest --push

Containerfile.compute_worker_podman

@@ -1,44 +1,34 @@
-FROM fedora:37
+FROM fedora:42
 
 # Include deps
 RUN dnf -y update && \
-    # https://bugzilla.redhat.com/show_bug.cgi?id=1995337#c3
-    rpm --setcaps shadow-utils 2>/dev/null && \
     dnf -y install podman fuse-overlayfs python3.9 \
         --exclude container-selinux && \
     dnf clean all && \
     rm -rf /var/cache /var/log/dnf* /var/log/yum.*
 
-# Setup user
-RUN useradd worker; \
-echo -e "worker:1:999\nworker:1001:64535" > /etc/subuid; \
-echo -e "worker:1:999\nworker:1001:64535" > /etc/subgid;
+# Copy the podman-connections file to allow for podman inside the container to connect to podman on the host, running containers alongside podman instead of inside
+COPY podman/podman-connections.json /root/.config/containers/podman-connections.json
 
 # Copy over the podman container configuration
 COPY podman/containers.conf /etc/containers/containers.conf
-COPY podman/worker-containers.conf /home/worker/.config/containers/containers.conf
+COPY podman/worker-containers.conf /root/.config/containers/containers.conf
 
 # Copy over the podman storage configuration
-COPY podman/worker-storage.conf /home/worker/.config/containers/storage.conf
+COPY podman/worker-storage.conf /root/.config/containers/storage.conf
 
-RUN mkdir -p /home/worker/.local/share/containers && \
-    chown worker:worker -R /home/worker && \
-    chmod 644 /etc/containers/containers.conf
+RUN mkdir -p /root/.local/share/containers
 
 # Copy & modify the defaults to provide reference if runtime changes needed.
 # Changes here are required for running with fuse-overlay storage inside container.
 RUN sed -e 's|^#mount_program|mount_program|g' \
-           -e '/additionalimage.*/a "/var/lib/shared",' \
-           -e 's|^mountopt[[:space:]]*=.*$|mountopt = "nodev,fsync=0"|g' \
-           /usr/share/containers/storage.conf \
-           > /etc/containers/storage.conf
+        -e '/additionalimage.*/a "/var/lib/shared",' \
+        -e 's|^mountopt[[:space:]]*=.*$|mountopt = "nodev,fsync=0"|g' \
+        /usr/share/containers/storage.conf \
+        > /etc/containers/storage.conf
 
 # Add volume for containers
-VOLUME /home/worker/.local/share/containers
-
-# Create directory for tmp space
-RUN mkdir /codabench && \
-    chown worker:worker /codabench
+VOLUME /root/.local/share/containers
 
 # Set up podman registry for dockerhub
 RUN echo -e "[registries.search]\nregistries = ['docker.io']\n" > /etc/containers/registries.conf
@@ -47,26 +37,26 @@ RUN echo -e "[registries.search]\nregistries = ['docker.io']\n" > /etc/container
 ENV PYTHONUNBUFFERED 1
 ENV CONTAINER_ENGINE_EXECUTABLE podman
 
-WORKDIR /home/worker/compute_worker
-
-ADD compute_worker/ /home/worker/compute_worker
+WORKDIR /root/compute_worker
 
-RUN chown worker:worker -R /home/worker/compute_worker
+ADD compute_worker/ /root/compute_worker
 
-RUN curl -sSL https://install.python-poetry.org | python3.9 -
+RUN curl -sSL https://install.python-poetry.org | python3.9 - --version 1.8.3
 # Poetry location so future commands (below) work
 ENV PATH $PATH:/root/.local/bin
+
 # Want poetry to use system python of docker container
 RUN poetry config virtualenvs.create false
 RUN poetry config virtualenvs.in-project false
+
 # So we get 3.9
 RUN poetry config virtualenvs.prefer-active-python true
 COPY ./compute_worker/pyproject.toml ./
 COPY ./compute_worker/poetry.lock ./
 RUN poetry install
-
+COPY ./src/settings/logs_loguru.py /usr/bin
 CMD celery -A compute_worker worker \
     -l info \
     -Q compute-worker \
     -n compute-worker@%n \
-    --concurrency=1
+    --concurrency=1

Containerfile.compute_worker_podman_gpu

@@ -1,75 +1,8 @@
-FROM fedora:37
+FROM codalab/codabench_worker_podman:latest
 
 # Include deps
-RUN curl -s -L https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo | tee /etc/yum.repos.d/cuda.repo && \
-    curl -s -L https://nvidia.github.io/nvidia-docker/rhel9.0/nvidia-docker.repo | tee /etc/yum.repos.d/nvidia-docker.repo && \
-    rpm -Uvh http://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-$(rpm -E %fedora).noarch.rpm && \
-    rpm -Uvh http://download1.rpmfusion.org/nonfree/fedora/rpmfusion-nonfree-release-$(rpm -E %fedora).noarch.rpm && \
+RUN dnf -y config-manager addrepo --from-repofile=https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo && \
     dnf -y update && \
-    dnf module install -y nvidia-driver:latest-dkms && \
-    dnf -y install podman fuse-overlayfs python3.9 nvidia-container-runtime nvidia-container-toolkit \
-     cuda --exclude container-selinux && \
+    dnf -y install nvidia-container-runtime nvidia-container-toolkit --exclude container-selinux && \
     dnf clean all && \
-    rm -rf /var/cache /var/log/dnf* /var/log/yum.*
-
-# Setup user
-RUN useradd worker; \
-echo -e "worker:1:999\nworker:1001:64535" > /etc/subuid; \
-echo -e "worker:1:999\nworker:1001:64535" > /etc/subgid;
-
-# Copy over the podman container configuration
-COPY podman/containers.conf /etc/containers/containers.conf
-COPY podman/worker-containers.conf /home/worker/.config/containers/containers.conf
-
-# Copy over the podman storage configuration
-COPY podman/worker-storage.conf /home/worker/.config/containers/storage.conf
-
-RUN mkdir -p /home/worker/.local/share/containers && \
-    chown worker:worker -R /home/worker && \
-    chmod 644 /etc/containers/containers.conf
-
-# Copy & modify the defaults to provide reference if runtime changes needed.
-# Changes here are required for running with fuse-overlay storage inside container.
-RUN sed -e 's|^#mount_program|mount_program|g' \
-           -e '/additionalimage.*/a "/var/lib/shared",' \
-           -e 's|^mountopt[[:space:]]*=.*$|mountopt = "nodev,fsync=0"|g' \
-           /usr/share/containers/storage.conf \
-           > /etc/containers/storage.conf; sed -i 's/^#no-cgroups = false/no-cgroups = true/;' /etc/nvidia-container-runtime/config.toml
-
-
-# Add volume for containers
-VOLUME /home/worker/.local/share/containers
-
-# This makes output not buffer and return immediately, nice for seeing results in stdout
-ENV PYTHONUNBUFFERED 1
-ENV CONTAINER_ENGINE_EXECUTABLE podman
-
-# Create directory for tmp space
-RUN mkdir /codabench && \
-    chown worker:worker /codabench && \
-# Set up podman registry for dockerhub
-    echo -e "[registries.search]\nregistries = ['docker.io']\n" > /etc/containers/registries.conf && \
-
-WORKDIR /home/worker/compute_worker
-
-ADD compute_worker/ /home/worker/compute_worker
-
-RUN curl -sSL https://install.python-poetry.org | python3.9 -
-# Poetry location so future commands (below) work
-ENV PATH $PATH:/root/.local/bin
-# Want poetry to use system python of docker container
-RUN poetry config virtualenvs.create false
-RUN poetry config virtualenvs.in-project false
-# So we get 3.9
-RUN poetry config virtualenvs.prefer-active-python true
-COPY ./compute_worker/pyproject.toml ./
-COPY ./compute_worker/poetry.lock ./
-RUN poetry install
-
-RUN chown worker:worker -R /home/worker/compute_worker
-
-CMD nvidia-smi && celery -A compute_worker worker \
-    -l info \
-    -Q compute-worker \
-    -n compute-worker@%n \
-    --concurrency=1
+    rm -rf /var/cache /var/log/dnf* /var/log/yum.*

Dockerfile.compute_worker

@@ -25,6 +25,7 @@ COPY ./compute_worker/poetry.lock ./
 RUN poetry config virtualenvs.prefer-active-python true && poetry install
 
 ADD compute_worker .
+COPY ./src/settings/logs_loguru.py /usr/bin
 
 CMD celery -A compute_worker worker \
     -l info \