Merge pull request #2062 from codalab/develop

Merge develop into master (1.21.1)

Author: ObadaS

.circleci/config.yml

@@ -15,39 +15,49 @@ jobs:
           #
           #     In another terminal:
           #         $ open vnc://0.0.0.0:5900
-          name: Download and start X for VNC viewing
+          name: "Setup: Enable VNC access"
           command: |
             sudo apt-get update
             sudo apt-get install -y x11vnc
           #  x11vnc -forever -nopw
           background: true
 
-      - run: cp .env_circleci .env
+      - run: 
+          name: "Setup: Copy environment variables"
+          command: cp .env_circleci .env
 
 
       - run:
-          name: Making gross directories for minio, cannot be made by docker for some reason
+          name: "Setup: Create directories for MinIO (cannot be made by docker for some reason)"
           command: |
             sudo mkdir -p var/minio/public
             sudo mkdir -p var/minio/private
 
       - run:
-          name: Build containers and collect static
+          name: "Docker: Build containers and collect static files"
           command: |
             docker compose -f docker-compose.yml -f docker-compose.selenium.yml up -d
             docker compose -f docker-compose.yml -f docker-compose.selenium.yml exec django python manage.py collectstatic --noinput
 
-      - run: docker-compose exec django flake8 src/
-      - run: docker pull codalab/codalab-legacy:py37 # not available without "not e2e" tests as they pull ahead of time
-      - run: docker pull codalab/codalab-legacy:py3 # not available without "not e2e" tests as they pull ahead of time
-      - run: docker pull vergilgxw/autotable:v2 # not available without "not e2e" tests as they pull ahead of time
+      - run: 
+          name: "Lint: Check code style with flake8"
+          command: docker-compose exec django flake8 src/
+
+
+      - run: 
+          name: "Docker: Pull required images"
+          # not available without "not e2e" tests as they pull ahead of time
+          command: |
+            docker pull codalab/codalab-legacy:py37 
+            docker pull codalab/codalab-legacy:py3 
+            docker pull vergilgxw/autotable:v2 
 
       - run:
-          name: pytest
+          name: "Tests: Run unit/integration tests (excluding e2e)"
           command: docker compose -f docker-compose.yml -f docker-compose.selenium.yml exec django py.test src/ -m "not e2e"
 
       - run:
-          name: e2e tests
+          name: "Tests: Run end-to-end (E2E) tests"
           command: docker compose -f docker-compose.yml -f docker-compose.selenium.yml exec django py.test src/tests/functional/ -m e2e
           no_output_timeout: 60m
 

.github/CODE_OF_CONDUCT.md

@@ -31,7 +31,7 @@ This Code of Conduct applies to all interactions related to this project, includ
 
 ## 5. Reporting Issues
 
-If you witness or experience any behavior that violates this Code of Conduct, please report it to **info@codalab.org**. All reports will be reviewed and handled confidentially.
+If you witness or experience any behavior that violates this Code of Conduct, please report it to **info@codabench.org**. All reports will be reviewed and handled confidentially.
 
 ## 6. Enforcement
 
@@ -49,7 +49,7 @@ This Code of Conduct is adapted from the [Contributor Covenant](https://www.cont
 ---
 
 ### Questions?
-If you have any questions or concerns, please reach out to **info@codalab.org**.
+If you have any questions or concerns, please reach out to **info@codabench.org**.
 
 
 

compute_worker/celery_config.py

@@ -1,8 +1,11 @@
 import os
-
+import ssl
 
 broker_url = os.environ.get('BROKER_URL')
-broker_use_ssl = os.environ.get('BROKER_USE_SSL', False)
+if os.environ.get('BROKER_USE_SSL', False):
+    broker_use_ssl = {
+        "cert_reqs": ssl.CERT_NONE,
+    }
 worker_concurrency = 1
 worker_prefetch_multiplier = 1
 task_acks_late = True

compute_worker/compute_worker.py

@@ -21,7 +21,7 @@
 import websockets
 import yaml
 from billiard.exceptions import SoftTimeLimitExceeded
-from celery import Celery, task, utils
+from celery import Celery, shared_task, utils
 from kombu import Queue, Exchange
 from urllib3 import Retry
 
@@ -113,7 +113,7 @@ class ExecutionTimeLimitExceeded(Exception):
 # The main compute worker entrypoint, this is how a job is ran at the highest
 # level.
 # -----------------------------------------------------------------------------
-@task(name="compute_worker_run")
+@shared_task(name="compute_worker_run")
 def run_wrapper(run_args):
     logger.info(f"Received run arguments: {run_args}")
     run = Run(run_args)
@@ -632,6 +632,9 @@ async def _run_program_directory(self, program_dir, kind):
                 logger.warning(
                     "Program directory missing metadata, assuming it's going to be handled by ingestion"
                 )
+                # Copy submission files into prediction output
+                # This is useful for results submissions but wrongly uses storage
+                shutil.copytree(program_dir, self.output_dir)
                 return
             else:
                 raise SubmissionException("Program directory missing 'metadata.yaml/metadata'")