Merge pull request #2012 from codalab/develop

Rebase "datasets" branch

Author: Didayolo

.circleci/config.yml

@@ -38,6 +38,9 @@ jobs:
             docker compose -f docker-compose.yml -f docker-compose.selenium.yml exec django python manage.py collectstatic --noinput
 
       - run: docker-compose exec django flake8 src/
+      - run: docker pull codalab/codalab-legacy:py37 # not available without "not e2e" tests as they pull ahead of time
+      - run: docker pull codalab/codalab-legacy:py3 # not available without "not e2e" tests as they pull ahead of time
+      - run: docker pull vergilgxw/autotable:v2 # not available without "not e2e" tests as they pull ahead of time
 
       - run:
           name: pytest
@@ -48,6 +51,13 @@ jobs:
           command: docker compose -f docker-compose.yml -f docker-compose.selenium.yml exec django py.test src/tests/functional/ -m e2e
           no_output_timeout: 60m
 
+      # Example to run specific set of tests (for debugging individual tests from a batch of tests)
+      # - run:
+      #     name: e2e tests - competitions
+      #     command: docker compose -f docker-compose.yml -f docker-compose.selenium.yml exec django py.test src/tests/functional/test_competitions.py -m e2e
+      #     no_output_timeout: 60m
+
+
       - store_artifacts:
           path: artifacts/
 

.github/workflows/ghPages-dev.yml

@@ -0,0 +1,25 @@
+name: github-pages-dev 
+on:
+  push:
+    branches:
+      - develop
+permissions:
+  contents: write
+jobs:
+  deploy-dev:
+    runs-on: ubuntu-latest
+    defaults:
+        run:
+            working-directory: documentation/
+    steps:
+      - uses: actions/checkout@v4
+      - name: Configure Git Credentials
+        run: |
+          git config user.name github-actions[bot]
+          git config user.email 41898282+github-actions[bot]@users.noreply.github.com
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+      - run: curl -LsSf https://astral.sh/uv/install.sh | sh
+      - run: /home/runner/.local/bin/uv sync 
+      - run: git fetch origin gh-pages --depth=1 && PDF=1 /home/runner/.local/bin/uv run mike deploy -u dev --push

.github/workflows/ghPages-prod.yml

@@ -0,0 +1,25 @@
+name: github-pages-prod
+on:
+  push:
+    tags:
+      - '*'
+permissions:
+  contents: write
+jobs:
+  deploy-prod:
+    runs-on: ubuntu-latest
+    defaults:
+        run:
+            working-directory: documentation/
+    steps:
+      - uses: actions/checkout@v4
+      - name: Configure Git Credentials
+        run: |
+          git config user.name github-actions[bot]
+          git config user.email 41898282+github-actions[bot]@users.noreply.github.com
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+      - run: curl -LsSf https://astral.sh/uv/install.sh | sh
+      - run: /home/runner/.local/bin/uv sync 
+      - run: git fetch origin gh-pages --depth=1 && PDF=1 /home/runner/.local/bin/uv run mike deploy -u  ${{ github.ref_name }} latest --push

Dockerfile.compute_worker

@@ -1,21 +1,28 @@
-FROM --platform=linux/amd64 python:3.9
+FROM --platform=linux/amd64 fedora:42
 
 # This makes output not buffer and return immediately, nice for seeing results in stdout
 ENV PYTHONUNBUFFERED 1
 
 # Install Docker
-RUN apt-get update && curl -fsSL https://get.docker.com | sh
+RUN dnf -y install dnf-plugins-core && \
+    dnf-3 config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo && \
+    dnf -y update && \
+    dnf install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin && \
+    dnf install -y python3.9 && \
+    dnf clean all && \
+    rm -rf /var/cache /var/log/dnf* /var/log/yum.*
 
 
-RUN curl -sSL https://install.python-poetry.org | python3 - --version 1.8.3
+RUN curl -sSL https://install.python-poetry.org | python3.9 - --version 1.8.3
 # Poetry location so future commands (below) work
 ENV PATH $PATH:/root/.local/bin
 # Want poetry to use system python of docker container
 RUN poetry config virtualenvs.create false
 RUN poetry config virtualenvs.in-project false
 COPY ./compute_worker/pyproject.toml ./
 COPY ./compute_worker/poetry.lock ./
-RUN poetry install
+# To use python3.9 instead of system python
+RUN poetry config virtualenvs.prefer-active-python true && poetry install
 
 ADD compute_worker .
 

Dockerfile.compute_worker_gpu

@@ -1,38 +1,12 @@
-FROM --platform=linux/amd64 python:3.9
-
-# This makes output not buffer and return immediately, nice for seeing results in stdout
-ENV PYTHONUNBUFFERED 1
-
-# Install Docker
-RUN apt-get update && curl -fsSL https://get.docker.com | sh
-
-
+FROM --platform=linux/amd64 codalab/competitions-v2-compute-worker:latest
 # Nvidia Container Toolkit for cuda use with docker
 # [source](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html)
-RUN curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \
-    && curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
-    sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
-    tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
-RUN apt-get update -y;
-RUN apt-get install -y nvidia-container-toolkit
+# Include deps
+RUN dnf -y config-manager addrepo --from-repofile=https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo && \
+    dnf -y update && \
+    dnf -y install nvidia-container-runtime nvidia-container-toolkit --exclude container-selinux && \
+    dnf clean all && \
+    rm -rf /var/cache /var/log/dnf* /var/log/yum.*
 # Make it explicit that we're using GPUs
 # BB - not convinced we need this
 ENV USE_GPU 1
-
-RUN curl -sSL https://install.python-poetry.org | python3 - --version 1.8.3
-# Poetry location so future commands (below) work
-ENV PATH $PATH:/root/.local/bin
-# Want poetry to use system python of docker container
-RUN poetry config virtualenvs.create false
-RUN poetry config virtualenvs.in-project false
-COPY ./compute_worker/pyproject.toml ./
-COPY ./compute_worker/poetry.lock ./
-RUN poetry install
-
-ADD compute_worker .
-
-CMD celery -A compute_worker worker \
-    -l info \
-    -Q compute-worker \
-    -n compute-worker@%n \
-    --concurrency=1

compute_worker/compute_worker.py

@@ -302,7 +302,9 @@ def get_detailed_results_file_path(self):
     async def send_detailed_results(self, file_path):
         logger.info(f"Updating detailed results {file_path} - {self.detailed_results_url}")
         self._put_file(self.detailed_results_url, file=file_path, content_type='text/html')
-        async with websockets.connect(self.websocket_url) as websocket:
+        websocket_url = f"{self.websocket_url}?kind=detailed_results"
+        logger.info(f"Connecting to {websocket_url} for detailed results")
+        async with websockets.connect(websocket_url) as websocket:
             await websocket.send(json.dumps({
                 "kind": 'detailed_result_update',
             }))
@@ -390,10 +392,14 @@ async def _send_data_through_socket(self, error_message):
         - Docker image pull failure logs
         - Execution time limit exceeded logs
         """
-        logger.info(f"Connecting to {self.websocket_url} to send docker image pull error")
+        # Create a unique websocket URL for error messages
+        websocket_url = f"{self.websocket_url}?kind=error_logs"
+        logger.info(f"Connecting to {websocket_url} to send error message")
+        
+        logger.info(f"Connecting to {websocket_url} to send docker image pull error")
 
         # connect to web socket
-        websocket = await websockets.connect(self.websocket_url)
+        websocket = await websockets.connect(websocket_url)
 
         # define websocket errors
         websocket_errors = (socket.gaierror, websockets.WebSocketException, websockets.ConnectionClosedError, ConnectionRefusedError)
@@ -416,7 +422,7 @@ async def _send_data_through_socket(self, error_message):
             # no error in websocket message sending
             logger.info(f"Error sent successfully through websocket")
 
-        logger.info(f"Disconnecting from websocket {self.websocket_url}")
+        logger.info(f"Disconnecting from websocket {websocket_url}")
 
         # close websocket
         await websocket.close()
@@ -500,8 +506,11 @@ async def _run_container_engine_cmd(self, engine_cmd, kind):
         }
 
         # Start websocket, it will reconnect in the stdout/stderr listener loop below
-        logger.info(f"Connecting to {self.websocket_url}")
-        websocket = await websockets.connect(self.websocket_url)
+        # This ensures each task has its own independent WebSocket connection
+        websocket_url = f"{self.websocket_url}?kind={kind}"
+        logger.debug(f"WORKER_MARKER: Connecting to {websocket_url}")
+        websocket = await websockets.connect(websocket_url)
+        # websocket = await websockets.connect(self.websocket_url) # old BB
         websocket_errors = (socket.gaierror, websockets.WebSocketException, websockets.ConnectionClosedError, ConnectionRefusedError)
 
         # Function to read a line, if the line is larger than the buffer size we will
@@ -522,7 +531,7 @@ async def _readline_or_chunk(stream):
                 logs = [self.logs[kind][key] for key in ('stdout', 'stderr')]
                 for value in logs:
                     try:
-                        out = await asyncio.wait_for(_readline_or_chunk(value["stream"]), timeout=.1)
+                        out = await asyncio.wait_for(_readline_or_chunk(value["stream"]), timeout=0.1)
                         if out:
                             value["data"] += out
                             print("WS: " + str(out))
@@ -535,32 +544,36 @@ async def _readline_or_chunk(stream):
                     except asyncio.TimeoutError:
                         continue
             except websocket_errors:
+                logger.debug("\n\nWebsocket error (line 538)\n\n")
                 try:
                     # do we need to await websocket.close() on the old socket? before making a new one probably not?
                     await websocket.close()
                 except Exception as e:
                     logger.error(e)
-                    logger.info(e)
                     # TODO: catch proper exceptions here..! What can go wrong failing to close?
                     pass
 
                 # try to reconnect a few times
                 tries = 0
                 while tries < 3 and not websocket.open:
                     try:
-                        websocket = await websockets.connect(self.websocket_url)
+                        logger.debug(f"\n\nAttempting to reconnect in 2 seconds (attempt {tries+1}/3)")
+                        websocket = await websockets.connect(websocket_url)
+                        logger.debug(f"\n\nSuccessfully reconnected to {websocket_url}")
                     except websocket_errors:
+                        logger.error(f"\n\nReconnection attempt {tries+1} failed: {websocket_errors}")
                         await asyncio.sleep(2)
                         tries += 1
 
         self.logs[kind]["end"] = time.time()
 
-        logger.info(f"Process exited with {proc.returncode}")
-        logger.info(f"Disconnecting from websocket {self.websocket_url}")
+        logger.debug(f"Process exited with {proc.returncode}")
+        logger.debug(f"Disconnecting from websocket {websocket_url}")
 
         # Communicate that the program is closing
         self.completed_program_counter += 1
 
+        logger.debug(f"WORKER_MARKER: Disconnecting from {websocket_url}, program counter = {self.completed_program_counter}")
         await websocket.close()
 
     def _get_host_path(self, *paths):
@@ -609,6 +622,9 @@ async def _run_program_directory(self, program_dir, kind):
                 logger.info(
                     "Program directory missing metadata, assuming it's going to be handled by ingestion"
                 )
+                # Copy submission files into prediction output
+                # This is useful for results submissions but wrongly uses storage
+                shutil.copytree(program_dir, self.output_dir)
                 return
             else:
                 raise SubmissionException("Program directory missing 'metadata.yaml/metadata'")

docker-compose.selenium.yml

@@ -4,11 +4,12 @@ services:
     environment:
       - SELENIUM_HOSTNAME=selenium
       - SUBMISSIONS_API_URL=http://django:36475/api
+      - WEBSOCKET_ALLOWED_ORIGINS=*
     ports:
       - 36475:36475
 
   selenium:
-    image: selenium/standalone-firefox:124.0
+    image: selenium/standalone-firefox:120.0
     volumes:
       - ./src/tests/functional/test_files:/test_files/
       - ./artifacts:/artifacts/:z

docker-compose.yml

@@ -53,7 +53,7 @@ services:
   # Minio local storage helper
   #-----------------------------------------------
   minio:
-    image: minio/minio:RELEASE.2020-10-03T02-19-42Z
+    image: minio/minio:RELEASE.2025-04-22T22-12-26Z
     command: server /export
     volumes:
       - ./var/minio:/export
@@ -62,11 +62,11 @@ services:
       - $MINIO_PORT:9000
     env_file: .env
     healthcheck:
-      test: ["CMD", "nc", "-z", "minio", "9000"]
+      test: ["CMD", "curl", "-I", "http://minio:9000/minio/health/live"]
       interval: 5s
       retries: 5
   createbuckets:
-    image: minio/mc
+    image: minio/mc:RELEASE.2025-07-21T05-28-08Z
     depends_on:
       minio:
         condition: service_healthy
@@ -78,7 +78,7 @@ services:
       /bin/sh -c "
       set -x;
       if [ -n \"$MINIO_ACCESS_KEY\" ] && [ -n \"$MINIO_SECRET_KEY\" ] && [ -n \"$MINIO_PORT\" ]; then
-        until /usr/bin/mc config host add minio_docker http://minio:$MINIO_PORT $MINIO_ACCESS_KEY $MINIO_SECRET_KEY && break; do 
+        until /usr/bin/mc alias set minio_docker http://minio:$MINIO_PORT $MINIO_ACCESS_KEY $MINIO_SECRET_KEY && break; do 
           echo '...waiting...' && sleep 5; 
         done;
         /usr/bin/mc mb minio_docker/$AWS_STORAGE_BUCKET_NAME || echo 'Bucket $AWS_STORAGE_BUCKET_NAME already exists.';

documentation/.gitignore

@@ -0,0 +1,14 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+.cache/
+# Virtual environments
+.venv
+
+uv.lock
+
+site/*

documentation/.python-version

@@ -0,0 +1 @@
+3.12