Include logs from all Python code, not just from uvicorn

Author: apdavison

api/deployment/Dockerfile.prod

@@ -37,6 +37,7 @@ ENV PYTHONPATH  /home/docker:${SITEDIR}:${VENV}/lib/python3.11/site-packages:/us
 
 RUN echo "daemon off;" >> /etc/nginx/nginx.conf
 RUN rm /etc/nginx/sites-enabled/default
+COPY log_conf.yaml ${SITEDIR}/log_conf.yaml
 COPY deployment/nginx-app-prod.conf /etc/nginx/sites-enabled/nginx-app.conf
 COPY deployment/supervisor-app.conf /etc/supervisor/conf.d/
 

api/deployment/supervisor-app.conf

@@ -12,7 +12,7 @@ stderr_logfile_maxbytes = 0
 socket = unix:///tmp/uvicorn.sock
 socket_owner = www-data
 user = www-data
-command = /home/docker/venv/bin/uvicorn simqueue.main:app --proxy-headers --uds /tmp/uvicorn.sock --timeout-keep-alive 300
+command = /home/docker/venv/bin/uvicorn simqueue.main:app --proxy-headers --uds /tmp/uvicorn.sock --timeout-keep-alive 300 --log-config /home/docker/site/log_conf.yaml
 numprocs = 2
 process_name = uvicorn-%(process_num)d
 stdout_logfile = /dev/stdout

api/log_conf.yaml

@@ -0,0 +1,34 @@
+version: 1
+disable_existing_loggers: False
+formatters:
+  default:
+    # "()": uvicorn.logging.DefaultFormatter
+    format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+  access:
+    # "()": uvicorn.logging.AccessFormatter
+    format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+handlers:
+  default:
+    formatter: default
+    class: logging.StreamHandler
+    stream: ext://sys.stderr
+  access:
+    formatter: access
+    class: logging.StreamHandler
+    stream: ext://sys.stdout
+loggers:
+  uvicorn.error:
+    level: INFO
+    handlers:
+      - default
+    propagate: no
+  uvicorn.access:
+    level: INFO
+    handlers:
+      - access
+    propagate: no
+root:
+  level: INFO
+  handlers:
+    - default
+  propagate: no

api/requirements.txt

@@ -11,3 +11,4 @@ requests
 numpy
 python-slugify
 ebrains-drive
+PyYAML

api/requirements.txt.lock

@@ -21,6 +21,7 @@ pydantic==2.10.6
 pydantic_core==2.27.2
 python-slugify==8.0.4
 pytz==2025.1
+PyYAML==6.0.3
 requests==2.32.3
 sniffio==1.3.1
 SQLAlchemy==2.0.39

api/simqueue/utils.py

@@ -38,6 +38,9 @@ async def check_quotas(collab: str, hardware_platform: str, user: str = None):
     for quota in available_quotas:
         if quota["usage"] < quota["limit"]:
             return True
+    logger.info(
+        f"No quota available for user {user} on {hardware_platform}. collab={collab}, available quotas: {available_quotas}"
+    )
     return False