48 feature add monitoring stack (#257)

Author: phwissmann

.gitignore

@@ -30,5 +30,5 @@ data/postgres/**
 data/prefect/**
 data/uppy-server-data/**
 .site/**
-.secrets/**
-.certs/**
+**/.secrets/**
+**/.certs/**

.env.development deployment/.env.development.env.development renamed to deployment/.env.development

@@ -6,7 +6,7 @@ PREFECT_LOGGING_LEVEL=DEBUG
 PREFECT_WORKER_LTS_VOLUME_NAME=lts-mock-volume
 PREFECT_JOB_TEMPLATE=prefect-jobtemplate-dev.json
 LTS_FREE_SPACE_PERCENTAGE=2
-PREFECT_VARS_FILE=./backend/prefect/vars_dev.toml
+PREFECT_VARS_FILE=../backend/prefect/vars_dev.toml
 
 ### Authentik
 # Use `AUTH_MIDDLEWARE=authentik` to protect access to dashboards
@@ -16,14 +16,14 @@ AUTHENTIK_HOST=https://authentik.ethz.ch
 AUTHENTIK_INSECURE=true
 
 ### Scicat
-SCICAT_USER_FILE=./.secrets/scicatuser.txt
-SCICAT_PASSWORD_FILE=./.secrets/scicatpass.txt
+SCICAT_USER_FILE=.secrets/scicatuser.txt
+SCICAT_PASSWORD_FILE=.secrets/scicatpass.txt
 SCICAT_ENDPOINT=https://scopem-openem2.ethz.ch/scicat/backend
 SCICAT_INGESTOR_GROUP=ingestor
 
 ### IDP
-IDP_CLIENT_SECRET_FILE=./.secrets/idpclientsecret.txt
-IDP_PASSWORD_FILE=./.secrets/idppassword.txt
+IDP_CLIENT_SECRET_FILE=.secrets/idpclientsecret.txt
+IDP_PASSWORD_FILE=.secrets/idppassword.txt
 IDP_URL=https://scopem-openem2.ethz.ch/keycloak
 IDP_USERNAME=archiver-service
 IDP_REALM=facility

.env.prod deployment/.env.prod.env.prod renamed to deployment/.env.prod

@@ -54,7 +54,7 @@ PREFECT_ARCHIVAL_WORKPOOL_NAME=archival-docker-workpool
 # Workpool name for retrieval jobs
 PREFECT_RETRIEVAL_WORKPOOL_NAME=retrieval-docker-workpool
 
-PREFECT_VARS_FILE=./backend/prefect/vars.toml
+PREFECT_VARS_FILE=../backend/prefect/vars.toml
 ### Authentik
 # Use `AUTH_MIDDLEWARE=authentik` to protect access to dashboards
 AUTH_MIDDLEWARE=authentik
@@ -66,7 +66,7 @@ AUTHENTIK_INSECURE=true
 ### Scicat
 SCICAT_ENDPOINT=https://scicat.development.psi.ch
 SCICAT_API_PREFIX=/api/v3
-SCICAT_USER_FILE=./.secrets/scicatuser_prod.txt
-SCICAT_PASSWORD_FILE=./.secrets/scicatpass_prod.txt
+SCICAT_USER_FILE=.secrets/scicatuser_prod.txt
+SCICAT_PASSWORD_FILE=.secrets/scicatpass_prod.txt
 SCICAT_INGESTOR_GROUP=unx-openem
 

.prefectignore deployment/.prefectignore.prefectignore renamed to deployment/.prefectignore

@@ -3,7 +3,7 @@ services:
     image: ${OPENEM_BACKEND_IMAGE_NAME}:${OPENEM_IMAGE_TAG}
     container_name: archiver-service-api
     build:
-      context: ./backend/api
+      context: ../backend/api
       dockerfile: ./Dockerfile
     environment:
       - PREFECT_LOGGING_LEVEL=${PREFECT_LOGGING_LEVEL}
@@ -120,7 +120,7 @@ services:
     image: ${PREFECT_CONFIG_IMAGE_NAME}:${OPENEM_IMAGE_TAG:-latest}
     container_name: prefect-config
     build:
-      context: ./backend/
+      context: ../backend/
       dockerfile: ./prefect/config.Dockerfile
       args:
         PREFECT_VERSION: ${PREFECT_VERSION}
@@ -146,7 +146,7 @@ services:
         source: ${PREFECT_VARS_FILE}
         target: /var/local/vars.toml
       - type: bind
-        source: ./backend/prefect/concurrency-limits.toml
+        source: ../backend/prefect/concurrency-limits.toml
         target: /var/local/concurrency-limits.toml
     command: "-v /var/local/vars.toml -s MINIO_USER_FILE MINIO_PASSWORD_FILE GITHUB_USER_FILE GITHUB_PASSWORD_FILE SCICAT_USER_FILE SCICAT_PASSWORD_FILE -l /var/local/concurrency-limits.toml"
     depends_on:
@@ -163,7 +163,7 @@ services:
     # Uses the runtime image: prefect will import the code and there it needs all packages installed
     image: ${PREFECT_RUNTIME_IMAGE_NAME}:${OPENEM_IMAGE_TAG:-latest}
     build:
-      context: ./backend/archiver
+      context: ../backend/archiver
       dockerfile: ../prefect/runtime.Dockerfile
       args:
         LTS_ROOT_FOLDER: ${LTS_ROOT_FOLDER}
@@ -201,16 +201,22 @@ services:
       - "traefik.enable=false"
 volumes:
   lts-mock-volume:
-    driver: local
+    name: scopemarchiver_lts-mock-volume
+    # driver: local
+    external: true
   nfs-lts-share:
     driver_opts:
       type: "nfs"
       o: "nfsvers=3,addr=lts22,nolock"
       device: ":/scopem_lts_nfs_test_lts22"
   postgres-volume-new:
-    driver: local
+    name: scopemarchiver_postgres-volume-new
+    # driver: local
+    external: true
   prefect-volume-new:
-    driver: local
+    name: scopemarchiver_prefect-volume-new
+    # driver: local
+    external: true
 
 secrets:
   postgres_user:

archiver-service.docker-compose.yml …ment/archiver-service.docker-compose.ymlarchiver-service.docker-compose.yml renamed to deployment/archiver-service.docker-compose.yml archiver-service.docker-compose.yml renamed to deployment/archiver-service.docker-compose.yml

@@ -1,3 +1,4 @@
+name: scopemarchiver
 include:
   # Traefik
   - traefik.docker-compose.yml
@@ -8,10 +9,11 @@ include:
   # S3 storage
   # - minio.docker-compose.yml
   # Monitoring stack
-  # - monitoring.docker-compose.yml
+  - monitoring.docker-compose.yml
 
 
 networks:
   scopemarchiver_network:
     name: scopemarchiver_network
-    driver: bridge
+    # driver: bridge
+    external: true

docker-compose.yml deployment/docker-compose.ymldocker-compose.yml renamed to deployment/docker-compose.yml

@@ -0,0 +1 @@
+HOST=scopem-openem2.ethz.ch