Merge pull request #285 from American-Institutes-for-Research/HEA-1066/better_docker_compose

Prevent local .env file from breaking Dagster Docker containers - see HEA-1066

Author: rhunwicks

.github/workflows/01-build-then-test.yml

@@ -21,6 +21,14 @@ env:
   GOOGLE_ADMIN_EMAIL: ""
   GOOGLE_APPLICATION_CREDENTIALS: ""
 
+  # Ingestion parameters - use secrets if actual credentials are required
+  BSS_METADATA_WORKBOOK: "gdrive://Database Design/BSS Metadata"
+  BSS_METADATA_STORAGE_OPTIONS: '{"token": "service_account", "access": "read_only", "creds": "valid_creds_object", "root_file_id": "0AOJ0gJ8sjnO7Uk9PVA"}'
+  BSS_LABEL_RECOGNITION_WORKBOOK: "gdrive://Database Design/BSS Labels (CI).xlsx"
+  BSS_LABEL_RECOGNITION_STORAGE_OPTIONS: '{"token": "service_account", "access": "read_only", "creds": "valid_creds_object", "root_file_id": "0AOJ0gJ8sjnO7Uk9PVA"}'
+  BSS_FILES_FOLDER: "gdrive://Discovery Folder/Baseline Storage Sheets (BSS)"
+  BSS_FILES_STORAGE_OPTIONS: '{"token": "service_account", "access": "read_only", "creds": "valid_creds_object", "root_file_id": "0AOJ0gJ8sjnO7Uk9PVA"}'
+
   # AWS Variables
   ECR_REGISTRY: "888016039450.dkr.ecr.us-east-1.amazonaws.com"
   AWS_REGION: "us-east-1"

docker-compose.override.yml

@@ -51,6 +51,8 @@ services:
       - /usr/src/app/jupyter
       - /usr/src/app/log
       - /usr/src/app/media
+      # Suppress use of the local outer .env file
+      - /dev/null:/usr/src/app/.env
     environment:
       DJANGO_SETTINGS_MODULE: hea.settings.local
       LAUNCHER: ${LAUNCHER}  # e.g. "debugpy" or "ddtrace"
@@ -69,6 +71,8 @@ services:
       - /usr/src/app/jupyter
       - /usr/src/app/log
       - /usr/src/app/media
+      # Suppress use of the local outer .env file
+      - /dev/null:/usr/src/app/.env
     environment:
       DJANGO_SETTINGS_MODULE: hea.settings.local
       LAUNCHER: ${LAUNCHER}  # e.g. "debugpy" or "ddtrace"

docker-compose.yml

@@ -89,8 +89,8 @@ services:
       DJANGO_SETTINGS_MODULE: ${APP}.settings.insecure
       DOMAIN: ${APP}lcl.localdomain,${APP}lcl,localhost,127.0.0.1
       SECRET_KEY: ${SECRET_KEY}
-      PGHOST: ${PGHOST}
-      PGPORT: ${PGPORT}
+      PGHOST: db
+      PGPORT: 5432
       PGUSER: ${CLIENT}${APP}${ENV}
       PGDATABASE: ${CLIENT}${APP}${ENV}
       PGPASSWORD: ${PGPASSWORD}
@@ -106,7 +106,7 @@ services:
       DAGSTER_S3_LOG_PATH: ${DAGSTER_S3_LOG_PATH}
       AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER}
       AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD}
-      AWS_ENDPOINT_URL_S3: ${MINIO_ENDPOINT_URL}
+      AWS_ENDPOINT_URL_S3: http://minio:9000
       MINIO_ROOT_USER: ${MINIO_ROOT_USER}
       MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
       MINIO_ENDPOINT_URL: http://minio:9000
@@ -115,6 +115,13 @@ services:
       DJANGO_MIGRATE: 1
       GOOGLE_APPLICATION_CREDENTIALS: ${GOOGLE_APPLICATION_CREDENTIALS}
       BASELINE_EXPLORER_API_ROOT_URL: ${BASELINE_EXPLORER_API_ROOT_URL}
+      # Ingestion Parameters
+      BSS_METADATA_WORKBOOK: ${BSS_METADATA_WORKBOOK}
+      BSS_METADATA_STORAGE_OPTIONS: ${BSS_METADATA_STORAGE_OPTIONS}
+      BSS_FILES_FOLDER: ${BSS_FILES_FOLDER}
+      BSS_FILES_STORAGE_OPTIONS: ${BSS_FILES_STORAGE_OPTIONS}
+      BSS_LABEL_RECOGNITION_WORKBOOK: ${BSS_LABEL_RECOGNITION_WORKBOOK}
+      BSS_LABEL_RECOGNITION_STORAGE_OPTIONS: ${BSS_LABEL_RECOGNITION_STORAGE_OPTIONS}
     command:
       - --timeout=3600
       - --workers=12

docker/app/run_dagster_daemon.sh

@@ -13,7 +13,7 @@ echo Setting up logs
 touch log/django.log
 chown -R django:django log/*
 
-echo Starting Dagster Daemon
+echo Starting Dagster Daemon using DAGSTER_HOME ${DAGSTER_HOME}
 if [ x"$LAUNCHER" != x"" ]; then
     echo using ${LAUNCHER}
 fi

docker/app/run_dagster_webserver.sh

@@ -13,7 +13,7 @@ echo Setting up logs
 touch log/django.log
 chown -R django:django log/*
 
-echo Starting Dagster Webserver
+echo Starting Dagster Webserver using DAGSTER_HOME ${DAGSTER_HOME}
 if [ x"$LAUNCHER" != x"" ]; then
     echo using ${LAUNCHER}
 fi