Merge branch 'HEA-752/Dagster-GraphQL-API-is-intermittently-failing-with-a-ProtocolError-when-accessed-via-the-revproxy-Django-view' of github.com-work2:FEWS-NET/HEA-Database-Development into HEA-752/Dagster-GraphQL-API-is-intermittently-failing-with-a-ProtocolError-when-accessed-via-the-revproxy-Django-view

Author: girumb

.github/workflows/01-build-then-test.yml

@@ -271,9 +271,6 @@ jobs:
           docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/log ./ || true
           docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/coverage.txt ./ || true
 
-          # Save the database schema as an artifact
-          docker compose run --no-deps --rm --entrypoint dbtoyaml app --no-owner --no-privileges test_${PGDATABASE} > schema.yml
-          diff pyrseas/schema.yaml schema.yml > schema.diff || true
       - name: "Upload test artifacts"
         if: success() || failure()
         uses: actions/upload-artifact@v4
@@ -400,9 +397,6 @@ jobs:
           # Copy the artifacts out of the Docker container to project directory
           docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/log ./ || true
           docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/coverage.txt ./ || true
-          # Save the database schema as an artifact
-          docker compose run --no-deps --rm --entrypoint dbtoyaml app --no-owner --no-privileges test_${PGDATABASE} > schema.yml
-          diff pyrseas/schema.yaml schema.yml > schema.diff || true
       - name: "Upload test artifacts"
         if: success() || failure()
         uses: actions/upload-artifact@v4
@@ -530,9 +524,6 @@ jobs:
           # Copy the artifacts out of the Docker container to project directory
           docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/log ./ || true
           docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/coverage.txt ./ || true
-          # Save the database schema as an artifact
-          docker compose run --no-deps --rm --entrypoint dbtoyaml app --no-owner --no-privileges test_${PGDATABASE} > schema.yml
-          diff pyrseas/schema.yaml schema.yml > schema.diff || true
       - name: "Upload test artifacts"
         if: success() || failure()
         uses: actions/upload-artifact@v4
@@ -661,11 +652,6 @@ jobs:
           # Copy the artifacts out of the Docker container to project directory
           docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/log ./ || true
           docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/coverage.txt ./ || true
-          # The prod image does not include pyrseas/dbtoyaml. Building a test image to include that
-          docker compose build app
-          # Save the database schema as an artifact
-          docker compose run --no-deps --rm --entrypoint dbtoyaml app --no-owner --no-privileges test_${PGDATABASE} > schema.yml
-          diff pyrseas/schema.yaml schema.yml > schema.diff || true
       - name: "Upload test artifacts"
         if: success() || failure()
         uses: actions/upload-artifact@v4

README.md

@@ -23,3 +23,53 @@ baseline
 
 This produces a .puml file that can be rendered using a PlantUML
 server, either within your IDE or using a service like http://www.plantuml.com/.
+
+## Debugging inside Docker Containers
+
+The `LAUNCHER` environment sets a wrapper program around the Python process
+(`gunicorn`, `dagster-daemon`, `dagster-webserver`). This can be used to
+enable a debugger inside Docker Containers:
+
+1. Set `LAUNCHER="python3 -m debugpy --listen 0.0.0.0:5678"` in `.env`
+2. Create Launch Configurations in Visual Studio Code like:
+
+```json
+			{
+				"name": "Python: Attach to app (Docker Container)",
+				"type": "debugpy",
+				"request": "attach",
+				"connect": {
+					"host": "localhost",
+					"port": 5678
+				},
+				"pathMappings": [
+					{
+						"localRoot": "${workspaceFolder:hea}",
+						"remoteRoot": "/usr/src/app"
+					}
+				],
+				"django": true,
+				"justMyCode": false
+			},
+			{
+				"name": "Python: Attach to dagster-daemon (Docker Container)",
+				"type": "debugpy",
+				"request": "attach",
+				"connect": {
+					"host": "localhost",
+					"port": 5680
+				},
+				"pathMappings": [
+					{
+						"localRoot": "${workspaceFolder:hea}",
+						"remoteRoot": "/usr/src/app"
+					}
+				],
+				"django": true,
+				"justMyCode": false
+			}
+```
+
+3. Start the Docker containers as normal, and then use the Run and Debug
+pane in Visual Studio code to launch the configuration that attaches to
+the desired server.

docker-compose.override.yml

@@ -15,6 +15,7 @@ services:
     build:
       target: dev
     ports:
+      - "5678:5678"
       - "8000:8000"
       - "8888:8888"
     volumes:
@@ -30,6 +31,9 @@ services:
       - ./env.example:/usr/src/app/.env
     environment:
       DJANGO_SETTINGS_MODULE: hea.settings.local
+      LAUNCHER: ${LAUNCHER}  # e.g. "debugpy" or "ddtrace"
+      # Disable frozen modules warning
+      PYDEVD_DISABLE_FILE_VALIDATION: 1
       # Put .coverage in a writable directory
       COVERAGE_FILE: log/.coverage
     command:
@@ -41,6 +45,7 @@ services:
     restart: no
     ports:
       - "3000:3000"
+      - "5679:5678"
     volumes:
       - ./:/usr/src/app
       # Separate volumes for writable directories inside the container
@@ -54,9 +59,14 @@ services:
       - ./env.example:/usr/src/app/.env
     environment:
       DJANGO_SETTINGS_MODULE: hea.settings.local
+      LAUNCHER: ${LAUNCHER}  # e.g. "debugpy" or "ddtrace"
+      # Disable frozen modules warning
+      PYDEVD_DISABLE_FILE_VALIDATION: 1
 
   dagster_daemon:
     restart: no
+    ports:
+      - "5680:5678"
     volumes:
       - ./:/usr/src/app
       # Separate volumes for writable directories inside the container
@@ -70,4 +80,7 @@ services:
       - ./env.example:/usr/src/app/.env
     environment:
       DJANGO_SETTINGS_MODULE: hea.settings.local
+      LAUNCHER: ${LAUNCHER}  # e.g. "debugpy" or "ddtrace"
+      # Disable frozen modules warning
+      PYDEVD_DISABLE_FILE_VALIDATION: 1
 

docker-compose.yml

@@ -110,14 +110,7 @@ services:
       MINIO_ENDPOINT_URL: http://minio:9000
       SUPPORT_EMAIL_ADDRESS: ${SUPPORT_EMAIL_ADDRESS}
       DJANGO_MIGRATE: 1
-      KILUIGI_INTERMEDIATETARGET_BACKEND_CLASS: ${KILUIGI_INTERMEDIATETARGET_BACKEND_CLASS}
-      KILUIGI_INTERMEDIATETARGET_ROOT_PATH: ${KILUIGI_INTERMEDIATETARGET_ROOT_PATH}
-      KILUIGI_FINALTARGET_BACKEND_CLASS: ${KILUIGI_FINALTARGET_BACKEND_CLASS}
-      KILUIGI_FINALTARGET_ROOT_PATH: ${KILUIGI_FINALTARGET_ROOT_PATH}
-      KILUIGI_REPORTTARGET_BACKEND_CLASS: ${KILUIGI_REPORTTARGET_BACKEND_CLASS}
-      KILUIGI_REPORTTARGET_ROOT_PATH: ${KILUIGI_REPORTTARGET_ROOT_PATH}      
       GOOGLE_APPLICATION_CREDENTIALS: ${GOOGLE_APPLICATION_CREDENTIALS}
-      GOOGLE_ADMIN_EMAIL: ${GOOGLE_ADMIN_EMAIL}
     command:
       - --timeout=3600
       - --workers=12

docker/app/Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 python:3.12-bookworm as base
+FROM python:3.12-bookworm as base
 
 # set up apt repositories for postgres installation
 RUN curl -s https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/pgdg.gpg >/dev/null && \

docker/app/run_dagster_daemon.sh

@@ -13,5 +13,8 @@ echo Setting up logs
 touch log/django.log
 chown -R django:django log/*
 
-echo Starting Dagster
-gosu django dagster-daemon run $*
+echo Starting Dagster Daemon
+if [ x"$LAUNCHER" != x"" ]; then
+    echo using ${LAUNCHER}
+fi
+gosu django ${LAUNCHER} /usr/local/bin/dagster-daemon run $*

docker/app/run_dagster_webserver.sh

@@ -13,5 +13,8 @@ echo Setting up logs
 touch log/django.log
 chown -R django:django log/*
 
-echo Starting Dagster
-gosu django dagster-webserver -h 0.0.0.0 -p 3000 -m pipelines --path-prefix /${DAGSTER_WEBSERVER_PREFIX} $*
+echo Starting Dagster Webserver
+if [ x"$LAUNCHER" != x"" ]; then
+    echo using ${LAUNCHER}
+fi
+gosu django ${LAUNCHER} /usr/local/bin/dagster-webserver -h 0.0.0.0 -p 3000 -m pipelines --path-prefix /${DAGSTER_WEBSERVER_PREFIX} $*

docker/app/run_django.sh

@@ -40,7 +40,10 @@ touch log/django_sql.log
 chown -R django:django log/*
 
 echo Starting Gunicorn with DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE}
-gosu django gunicorn ${APP}.asgi:application \
+if [ x"$LAUNCHER" != x"" ]; then
+    echo using ${LAUNCHER}
+fi
+gosu django ${LAUNCHER} /usr/local/bin/gunicorn ${APP}.asgi:application \
     --name ${APP}${ENV} \
     --worker-class uvicorn.workers.UvicornWorker \
     --config $(dirname $(readlink -f "$0"))/gunicorn_config.py \

docker/db/Dockerfile

@@ -1,3 +1,5 @@
-FROM --platform=linux/amd64 postgis/postgis:17-3.5
+# Use a third party multicarch base image for compatibility with both ARM and AMD architectures
+# until PostGIS fix https://github.com/postgis/docker-postgis/issues/216
+FROM ghcr.io/baosystems/postgis:17-3.5
 
 COPY create_db.sh /docker-entrypoint-initdb.d/create_db.sh

docker/db/create_db.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-psql --set=CLIENT=$CLIENT --set=APP=$APP --set=ENV=$ENV --set=POSTGRES_PASSWORD=$POSTGRES_PASSWORD --set=DAGSTER_PASSWORD=$DAGSTER_PASSWORD --set=CREATE_TEMPLATE=${CREATE_TEMPLATE:-false} -d postgres --echo-all << EOF
+psql --set=CLIENT=$CLIENT --set=APP=$APP --set=ENV=$ENV --set=POSTGRES_PASSWORD=$POSTGRES_PASSWORD --set=CREATE_TEMPLATE=${CREATE_TEMPLATE:-false} -d postgres --echo-all << EOF
 
 \set DATABASE :CLIENT :APP :ENV
 \set OWNER :CLIENT :APP :ENV
@@ -74,7 +74,7 @@ ALTER DEFAULT PRIVILEGES IN SCHEMA :SCHEMA
   GRANT SELECT ON TABLES TO :OWNER;
 
 \set DAGSTER :CLIENT :APP :ENV
-CREATE ROLE :DAGSTER PASSWORD :'DAGSTER_PASSWORD' NOLOGIN CREATEDB NOCREATEROLE NOSUPERUSER;
+CREATE ROLE :DAGSTER PASSWORD :'POSTGRES_PASSWORD' NOLOGIN CREATEDB NOCREATEROLE NOSUPERUSER;
 COMMENT ON ROLE :DAGSTER IS 'Main Dagster pipeline user for :CLIENT :PRJ :ENV';
 GRANT :DAGSTER TO :OWNER;
 GRANT CONNECT, TEMPORARY, CREATE ON DATABASE :DATABASE TO :DAGSTER;