Skip to content

Commit c35eaf3

Browse files
ObadaSObadaS
authored
Merge branch 'develop' into workerPackagesUpdate
2 parents 0b2dc04 + 54550ad commit c35eaf3

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

69 files changed

+2704
-906
lines changed

‎.circleci/config.yml‎

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -15,39 +15,49 @@ jobs:
1515
#
1616
# In another terminal:
1717
# $ open vnc://0.0.0.0:5900
18-
name: Download and start X for VNC viewing
18+
name: "Setup: Enable VNC access"
1919
command: |
2020
sudo apt-get update
2121
sudo apt-get install -y x11vnc
2222
# x11vnc -forever -nopw
2323
background: true
2424

25-
- run: cp .env_circleci .env
25+
- run:
26+
name: "Setup: Copy environment variables"
27+
command: cp .env_circleci .env
2628

2729

2830
- run:
29-
name: Making gross directories for minio, cannot be made by docker for some reason
31+
name: "Setup: Create directories for MinIO (cannot be made by docker for some reason)"
3032
command: |
3133
sudo mkdir -p var/minio/public
3234
sudo mkdir -p var/minio/private
3335
3436
- run:
35-
name: Build containers and collect static
37+
name: "Docker: Build containers and collect static files"
3638
command: |
3739
docker compose -f docker-compose.yml -f docker-compose.selenium.yml up -d
3840
docker compose -f docker-compose.yml -f docker-compose.selenium.yml exec django python manage.py collectstatic --noinput
3941
40-
- run: docker-compose exec django flake8 src/
41-
- run: docker pull codalab/codalab-legacy:py37 # not available without "not e2e" tests as they pull ahead of time
42-
- run: docker pull codalab/codalab-legacy:py3 # not available without "not e2e" tests as they pull ahead of time
43-
- run: docker pull vergilgxw/autotable:v2 # not available without "not e2e" tests as they pull ahead of time
42+
- run:
43+
name: "Lint: Check code style with flake8"
44+
command: docker-compose exec django flake8 src/
45+
46+
47+
- run:
48+
name: "Docker: Pull required images"
49+
# not available without "not e2e" tests as they pull ahead of time
50+
command: |
51+
docker pull codalab/codalab-legacy:py37
52+
docker pull codalab/codalab-legacy:py3
53+
docker pull vergilgxw/autotable:v2
4454
4555
- run:
46-
name: pytest
56+
name: "Tests: Run unit/integration tests (excluding e2e)"
4757
command: docker compose -f docker-compose.yml -f docker-compose.selenium.yml exec django py.test src/ -m "not e2e"
4858

4959
- run:
50-
name: e2e tests
60+
name: "Tests: Run end-to-end (E2E) tests"
5161
command: docker compose -f docker-compose.yml -f docker-compose.selenium.yml exec django py.test src/tests/functional/ -m e2e
5262
no_output_timeout: 60m
5363

‎.env_sample‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,3 +104,10 @@ ENABLE_SIGN_IN=True
104104
# GS_PUBLIC_BUCKET_NAME=public
105105
# GS_PRIVATE_BUCKET_NAME=private
106106
# GOOGLE_APPLICATION_CREDENTIALS=/app/certs/google-storage-api.json
107+
108+
109+
# -----------------------------------------------------------------------------
110+
# Logging (Serialized outputs the logs in JSON format)
111+
# -----------------------------------------------------------------------------
112+
LOG_LEVEL=info
113+
SERIALIZED=false

‎Containerfile.compute_worker_podman‎

Lines changed: 18 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,44 +1,34 @@
1-
FROM fedora:37
1+
FROM fedora:42
22

33
# Include deps
44
RUN dnf -y update && \
5-
# https://bugzilla.redhat.com/show_bug.cgi?id=1995337#c3
6-
rpm --setcaps shadow-utils 2>/dev/null && \
75
dnf -y install podman fuse-overlayfs python3.9 \
86
--exclude container-selinux && \
97
dnf clean all && \
108
rm -rf /var/cache /var/log/dnf* /var/log/yum.*
119

12-
# Setup user
13-
RUN useradd worker; \
14-
echo -e "worker:1:999\nworker:1001:64535" > /etc/subuid; \
15-
echo -e "worker:1:999\nworker:1001:64535" > /etc/subgid;
10+
# Copy the podman-connections file to allow for podman inside the container to connect to podman on the host, running containers alongside podman instead of inside
11+
COPY podman/podman-connections.json /root/.config/containers/podman-connections.json
1612

1713
# Copy over the podman container configuration
1814
COPY podman/containers.conf /etc/containers/containers.conf
19-
COPY podman/worker-containers.conf /home/worker/.config/containers/containers.conf
15+
COPY podman/worker-containers.conf /root/.config/containers/containers.conf
2016

2117
# Copy over the podman storage configuration
22-
COPY podman/worker-storage.conf /home/worker/.config/containers/storage.conf
18+
COPY podman/worker-storage.conf /root/.config/containers/storage.conf
2319

24-
RUN mkdir -p /home/worker/.local/share/containers && \
25-
chown worker:worker -R /home/worker && \
26-
chmod 644 /etc/containers/containers.conf
20+
RUN mkdir -p /root/.local/share/containers
2721

2822
# Copy & modify the defaults to provide reference if runtime changes needed.
2923
# Changes here are required for running with fuse-overlay storage inside container.
3024
RUN sed -e 's|^#mount_program|mount_program|g' \
31-
-e '/additionalimage.*/a "/var/lib/shared",' \
32-
-e 's|^mountopt[[:space:]]*=.*$|mountopt = "nodev,fsync=0"|g' \
33-
/usr/share/containers/storage.conf \
34-
> /etc/containers/storage.conf
25+
-e '/additionalimage.*/a "/var/lib/shared",' \
26+
-e 's|^mountopt[[:space:]]*=.*$|mountopt = "nodev,fsync=0"|g' \
27+
/usr/share/containers/storage.conf \
28+
> /etc/containers/storage.conf
3529

3630
# Add volume for containers
37-
VOLUME /home/worker/.local/share/containers
38-
39-
# Create directory for tmp space
40-
RUN mkdir /codabench && \
41-
chown worker:worker /codabench
31+
VOLUME /root/.local/share/containers
4232

4333
# Set up podman registry for dockerhub
4434
RUN echo -e "[registries.search]\nregistries = ['docker.io']\n" > /etc/containers/registries.conf
@@ -47,26 +37,26 @@ RUN echo -e "[registries.search]\nregistries = ['docker.io']\n" > /etc/container
4737
ENV PYTHONUNBUFFERED 1
4838
ENV CONTAINER_ENGINE_EXECUTABLE podman
4939

50-
WORKDIR /home/worker/compute_worker
51-
52-
ADD compute_worker/ /home/worker/compute_worker
40+
WORKDIR /root/compute_worker
5341

54-
RUN chown worker:worker -R /home/worker/compute_worker
42+
ADD compute_worker/ /root/compute_worker
5543

56-
RUN curl -sSL https://install.python-poetry.org | python3.9 -
44+
RUN curl -sSL https://install.python-poetry.org | python3.9 - --version 1.8.3
5745
# Poetry location so future commands (below) work
5846
ENV PATH $PATH:/root/.local/bin
47+
5948
# Want poetry to use system python of docker container
6049
RUN poetry config virtualenvs.create false
6150
RUN poetry config virtualenvs.in-project false
51+
6252
# So we get 3.9
6353
RUN poetry config virtualenvs.prefer-active-python true
6454
COPY ./compute_worker/pyproject.toml ./
6555
COPY ./compute_worker/poetry.lock ./
6656
RUN poetry install
67-
57+
COPY ./src/settings/logs_loguru.py /usr/bin
6858
CMD celery -A compute_worker worker \
6959
-l info \
7060
-Q compute-worker \
7161
-n compute-worker@%n \
72-
--concurrency=1
62+
--concurrency=1

‎Containerfile.compute_worker_podman_gpu‎

Lines changed: 4 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -1,75 +1,8 @@
1-
FROM fedora:37
1+
FROM codalab/codabench_worker_podman:latest
22

33
# Include deps
4-
RUN curl -s -L https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo | tee /etc/yum.repos.d/cuda.repo && \
5-
curl -s -L https://nvidia.github.io/nvidia-docker/rhel9.0/nvidia-docker.repo | tee /etc/yum.repos.d/nvidia-docker.repo && \
6-
rpm -Uvh http://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-$(rpm -E %fedora).noarch.rpm && \
7-
rpm -Uvh http://download1.rpmfusion.org/nonfree/fedora/rpmfusion-nonfree-release-$(rpm -E %fedora).noarch.rpm && \
4+
RUN dnf -y config-manager addrepo --from-repofile=https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo && \
85
dnf -y update && \
9-
dnf module install -y nvidia-driver:latest-dkms && \
10-
dnf -y install podman fuse-overlayfs python3.9 nvidia-container-runtime nvidia-container-toolkit \
11-
cuda --exclude container-selinux && \
6+
dnf -y install nvidia-container-runtime nvidia-container-toolkit --exclude container-selinux && \
127
dnf clean all && \
13-
rm -rf /var/cache /var/log/dnf* /var/log/yum.*
14-
15-
# Setup user
16-
RUN useradd worker; \
17-
echo -e "worker:1:999\nworker:1001:64535" > /etc/subuid; \
18-
echo -e "worker:1:999\nworker:1001:64535" > /etc/subgid;
19-
20-
# Copy over the podman container configuration
21-
COPY podman/containers.conf /etc/containers/containers.conf
22-
COPY podman/worker-containers.conf /home/worker/.config/containers/containers.conf
23-
24-
# Copy over the podman storage configuration
25-
COPY podman/worker-storage.conf /home/worker/.config/containers/storage.conf
26-
27-
RUN mkdir -p /home/worker/.local/share/containers && \
28-
chown worker:worker -R /home/worker && \
29-
chmod 644 /etc/containers/containers.conf
30-
31-
# Copy & modify the defaults to provide reference if runtime changes needed.
32-
# Changes here are required for running with fuse-overlay storage inside container.
33-
RUN sed -e 's|^#mount_program|mount_program|g' \
34-
-e '/additionalimage.*/a "/var/lib/shared",' \
35-
-e 's|^mountopt[[:space:]]*=.*$|mountopt = "nodev,fsync=0"|g' \
36-
/usr/share/containers/storage.conf \
37-
> /etc/containers/storage.conf; sed -i 's/^#no-cgroups = false/no-cgroups = true/;' /etc/nvidia-container-runtime/config.toml
38-
39-
40-
# Add volume for containers
41-
VOLUME /home/worker/.local/share/containers
42-
43-
# This makes output not buffer and return immediately, nice for seeing results in stdout
44-
ENV PYTHONUNBUFFERED 1
45-
ENV CONTAINER_ENGINE_EXECUTABLE podman
46-
47-
# Create directory for tmp space
48-
RUN mkdir /codabench && \
49-
chown worker:worker /codabench && \
50-
# Set up podman registry for dockerhub
51-
echo -e "[registries.search]\nregistries = ['docker.io']\n" > /etc/containers/registries.conf && \
52-
53-
WORKDIR /home/worker/compute_worker
54-
55-
ADD compute_worker/ /home/worker/compute_worker
56-
57-
RUN curl -sSL https://install.python-poetry.org | python3.9 -
58-
# Poetry location so future commands (below) work
59-
ENV PATH $PATH:/root/.local/bin
60-
# Want poetry to use system python of docker container
61-
RUN poetry config virtualenvs.create false
62-
RUN poetry config virtualenvs.in-project false
63-
# So we get 3.9
64-
RUN poetry config virtualenvs.prefer-active-python true
65-
COPY ./compute_worker/pyproject.toml ./
66-
COPY ./compute_worker/poetry.lock ./
67-
RUN poetry install
68-
69-
RUN chown worker:worker -R /home/worker/compute_worker
70-
71-
CMD nvidia-smi && celery -A compute_worker worker \
72-
-l info \
73-
-Q compute-worker \
74-
-n compute-worker@%n \
75-
--concurrency=1
8+
rm -rf /var/cache /var/log/dnf* /var/log/yum.*

‎Dockerfile.compute_worker‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ COPY ./compute_worker/poetry.lock ./
2525
RUN poetry config virtualenvs.prefer-active-python true && poetry install
2626

2727
ADD compute_worker .
28+
COPY ./src/settings/logs_loguru.py /usr/bin
2829

2930
CMD celery -A compute_worker worker \
3031
-l info \

0 commit comments

Comments
 (0)