diff --git a/deployment/tz-single-server-deployment/.env b/deployment/tz-single-server-deployment/.env new file mode 100644 index 0000000..bebc2df --- /dev/null +++ b/deployment/tz-single-server-deployment/.env @@ -0,0 +1,27 @@ +OL_REFERENCE_UI_VERSION=5.2.9 +OL_REQUISITION_VERSION=8.3.9 +#OL_REQUISITION_VERSION=8.3.7-RC1 +OL_REFERENCEDATA_VERSION=15.2.7 +OL_AUTH_VERSION=4.3.4 +OL_NOTIFICATION_VERSION=4.3.4 +OL_FULFILLMENT_VERSION=9.0.6-SNAPSHOT +OL_CCE_VERSION=1.3.4 +OL_STOCKMANAGEMENT_VERSION=5.1.11 +OL_REPORT_VERSION=1.2.3 +OL_BUQ_VERSION=1.0.1-SNAPSHOT +OL_DHIS2_INTEGRATION=1.0.0 +OL_HAPIFHIR_VERSION=2.0.3 +OL_ONE_NETWORK_INTEGRATION_VERSION=0.0.1 +OL_ONE_NETWORK_INTEGRATION_REFERENCEDATA_EXTENSION_VERSION=0.0.1 +OL_ONE_NETWORK_INTEGRATION_STOCKMANAGEMENT_EXTENSION_VERSION=0.0.1 +OL_DIAGNOSTICS_VERSION=1.1.3 + +OL_NGINX_VERSION=5 +OL_RSYSLOG_VERSION=1 + +OL_POSTGRES_VERSION=12-debezium +TZ_UI_VERSION=1.0.0-SNAPSHOT + +OL_EQUIPMENT_VERSION=1.0.0-SNAPSHOT +TZ_COMMON_VERSION=1.0.0-SNAPSHOT +TZ_POST_EXTENSION_VERSION=1.0.2-SNAPSHOT diff --git a/deployment/tz-single-server-deployment/cleanDb.sh b/deployment/tz-single-server-deployment/cleanDb.sh new file mode 100755 index 0000000..e7d68f7 --- /dev/null +++ b/deployment/tz-single-server-deployment/cleanDb.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -e + +# ensure some environment variables are set +: "${DATABASE_URL:?DATABASE_URL not set in environment}" +: "${POSTGRES_USER:?POSTGRES_USER not set in environment}" +: "${POSTGRES_PASSWORD:?POSTGRES_PASSWORD not set in environment}" + +# pull apart some of those pieces stuck together in DATABASE_URL + +URL=`echo ${DATABASE_URL} | sed -E 's/^jdbc\:(.+)/\1/'` # jdbc: +: "${URL:?URL not parsed}" + +HOST=`echo ${DATABASE_URL} | sed -E 's/^.*\/{2}(.+):.*$/\1/'` # //: +: "${HOST:?HOST not parsed}" + +PORT=`echo ${DATABASE_URL} | sed -E 's/^.*\:([0-9]+)\/.*$/\1/'` # :/ +: "${PORT:?Port not parsed}" + +DB=`echo ${DATABASE_URL} | sed -E 's/^.*\/(.+)\?*$/\1/'` # /? +: "${DB:?DB not set}" + +# pgpassfile makes it easy and safe to login +echo "${HOST}:${PORT}:${DB}:${POSTGRES_USER}:${POSTGRES_PASSWORD}" > pgpassfile +chmod 600 pgpassfile + +# hasta la vista, schema +export PGPASSFILE='pgpassfile' +psql "${URL}" -U ${POSTGRES_USER} -t -c "select 'drop schema if exists \"' || schemaname || '\" cascade;' from (select n.nspname as schemaname from pg_catalog.pg_namespace n where n.nspname !~ '^pg_' AND n.nspname <> 'information_schema') as schemaname;" | psql "${URL}" -U ${POSTGRES_USER} 2>&1 diff --git a/deployment/tz-single-server-deployment/config/Dockerfile b/deployment/tz-single-server-deployment/config/Dockerfile new file mode 100644 index 0000000..8f90e99 --- /dev/null +++ b/deployment/tz-single-server-deployment/config/Dockerfile @@ -0,0 +1,7 @@ +FROM gliderlabs/alpine:3.4 + +COPY log/logback.xml /config/log/logback.xml +COPY scalyr/agent.json /config/scalyr/agent.json + +CMD /bin/true + diff --git a/deployment/tz-single-server-deployment/config/log/logback.xml b/deployment/tz-single-server-deployment/config/log/logback.xml new file mode 100644 index 0000000..fed4342 --- /dev/null +++ b/deployment/tz-single-server-deployment/config/log/logback.xml @@ -0,0 +1,54 @@ + + + + + log + LOCAL0 + [%15.15thread] %level %logger %msg + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/deployment/tz-single-server-deployment/config/postgres/init.sql b/deployment/tz-single-server-deployment/config/postgres/init.sql new file mode 100644 index 0000000..ab84f5e --- /dev/null +++ b/deployment/tz-single-server-deployment/config/postgres/init.sql @@ -0,0 +1 @@ +ALTER ROLE postgres SUPERUSER; \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/config/scalyr/agent.json b/deployment/tz-single-server-deployment/config/scalyr/agent.json new file mode 100644 index 0000000..4fdcff9 --- /dev/null +++ b/deployment/tz-single-server-deployment/config/scalyr/agent.json @@ -0,0 +1,37 @@ +// Configuration for the Scalyr Agent. For help: +// +// https://www.scalyr.com/help/scalyr-agent-2 + +{ + import_vars: [ "SCALYR_API_KEY", "VIRTUAL_HOST"], + + // Enter a "Write Logs" api key for your account. These are available at https://www.scalyr.com/keys + api_key: "$SCALYR_API_KEY", + + // Fields describing this server. These fields are attached to each log message, and + // can be used to filter data from a particular server or group of servers. + server_attributes: { + // Fill in this field if you'd like to override the server's hostname. + serverHost: "$VIRTUAL_HOST", + org: "OpenLMIS", + + // You can add whatever additional fields you'd like. + // tier: "production" + } + + // Log files to upload to Scalyr. You can use '*' wildcards here. + logs: [ + { path: "/var/log/nginx/log/access.log", attributes: {parser: "accessLog"} }, + { path: "/var/log/nginx/log/error.log" } , + { path: "/var/log/auth-tomcat/access_log.log", attributes: {parser: "accessLog"} }, + { path: "/var/log/messages", attributes: {parser: "systemLog"} }, + ], + + global_monitor_sample_interval: "5", + + monitors: [ + { + module: "scalyr_agent.builtin_monitors.docker_monitor" + } + ] +} diff --git a/deployment/tz-single-server-deployment/deploy_to_env.sh b/deployment/tz-single-server-deployment/deploy_to_env.sh new file mode 100755 index 0000000..39926d0 --- /dev/null +++ b/deployment/tz-single-server-deployment/deploy_to_env.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env sh + +export KEEP_OR_WIPE="keep" + +../shared/init_env_gh.sh +cat $settings_training > settings.env +../shared/pull_images.sh $1 + +../shared/restart.sh $1 diff --git a/deployment/tz-single-server-deployment/docker-compose.yml b/deployment/tz-single-server-deployment/docker-compose.yml new file mode 100644 index 0000000..301ca3f --- /dev/null +++ b/deployment/tz-single-server-deployment/docker-compose.yml @@ -0,0 +1,382 @@ +version: '3.5' +services: + + consul: + command: -server -bootstrap + image: gliderlabs/consul-server + ports: + - "8300" + - "8400" + - "8500:8500" + - "53" + networks: + - public_network + + nginx: + image: openlmis/nginx:${OL_NGINX_VERSION} + ports: + - "8090:80" + env_file: settings.env + environment: + NGINX_LOG_DIR: '/var/log/nginx/log' + volumes: + - 'nginx-log:/var/log/nginx/log' + - 'consul-template-log:/var/log/consul-template' + depends_on: [consul] + networks: + - public_network + + tanzania-ui: + restart: always + image: openlmistz/tanzania-ui:${TZ_UI_VERSION} + env_file: settings.env + depends_on: [consul] + networks: + - public_network + + requisition: + restart: always + image: openlmis/requisition:${OL_REQUISITION_VERSION} + env_file: settings.env + environment: + JAVA_OPTS: '-server -Xmx1024m -Dlogging.config=/config/log/logback.xml' + spring_profiles_active: ${spring_profiles_active} + volumes: + - 'service-config:/config' + depends_on: [log] + command: ["/wait-for-postgres.sh", "/run.sh"] + mem_limit: '1.3G' + cpus: 2 + networks: + - public_network + + referencedata: + restart: always + image: openlmis/referencedata:${OL_REFERENCEDATA_VERSION} + env_file: settings.env + environment: + JAVA_OPTS: '-server -Xmx1024m -Dlogging.config=/config/log/logback.xml' + spring_profiles_active: ${spring_profiles_active} + volumes: + - 'service-config:/config' + depends_on: [log] + command: ["/wait-for-postgres.sh", "/run.sh"] + mem_limit: '1.3G' + cpus: 2 + networks: + - public_network + + auth: + restart: always + image: openlmis/auth:${OL_AUTH_VERSION} + env_file: settings.env + environment: + JAVA_OPTS: '-server -Xmx512m -Dlogging.config=/config/log/logback.xml' + spring_profiles_active: ${spring_profiles_active} + volumes: + - 'service-config:/config' + - 'auth-tomcat-log:/var/log/auth-tomcat' + depends_on: [log] + command: ["/wait-for-postgres.sh", "/run.sh"] + mem_limit: '0.7G' + cpus: 2 + networks: + - public_network + + notification: + restart: always + image: openlmis/notification:${OL_NOTIFICATION_VERSION} + env_file: settings.env + environment: + JAVA_OPTS: '-server -Xmx512m -Dlogging.config=/config/log/logback.xml' + spring_profiles_active: ${spring_profiles_active} + volumes: + - 'service-config:/config' + depends_on: [log] + command: ["/wait-for-postgres.sh", "/run.sh"] + mem_limit: '0.7G' + cpus: 2 + networks: + - public_network + + fulfillment: + restart: always + image: openlmistz/tz-openlmis-fulfillment:${OL_FULFILLMENT_VERSION} + env_file: settings.env + environment: + JAVA_OPTS: '-server -Xmx512m -Dlogging.config=/config/log/logback.xml' + spring_profiles_active: ${spring_profiles_active} + volumes: + - 'service-config:/config' + depends_on: [log, db] + command: ["/wait-for-postgres.sh", "/run.sh"] + mem_limit: '0.7G' + cpus: 2 + networks: + - public_network + + budget: + restart: always + image: openlmistz/budget:latest + env_file: settings.env + environment: + JAVA_OPTS: '-server -Xmx512m -Dlogging.config=/config/log/logback.xml' + spring_profiles_active: ${spring_profiles_active} + volumes: + - 'service-config:/config' + depends_on: [log] + command: ["/wait-for-postgres.sh", "/run.sh"] + mem_limit: '0.7G' + cpus: 2 + networks: + - public_network + + equipment: + restart: always + image: openlmistz/equipment:${OL_EQUIPMENT_VERSION} + env_file: settings.env + environment: + JAVA_OPTS: '-server -Xmx512m -Dlogging.config=/config/log/logback.xml' + spring_profiles_active: ${spring_profiles_active} + volumes: + - 'service-config:/config' + depends_on: [log, cce, auth, nginx] + command: ["/wait-for-postgres.sh", "/run.sh"] + mem_limit: '0.7G' + cpus: 2 + networks: + - public_network + + common: + restart: always + image: openlmistz/common:${TZ_COMMON_VERSION} + env_file: settings.env + environment: + JAVA_OPTS: '-server -Xmx1024m -Dlogging.config=/config/log/logback.xml' + spring_profiles_active: ${spring_profiles_active} + volumes: + - 'service-config:/config' + depends_on: [log] + command: [ "/wait-for-postgres.sh", "/run.sh" ] + mem_limit: '1.3G' + cpus: 2 + networks: + - public_network + + cce: + restart: always + image: openlmis/cce:${OL_CCE_VERSION} + env_file: settings.env + environment: + JAVA_OPTS: '-server -Xmx512m -Dlogging.config=/config/log/logback.xml' + spring_profiles_active: ${spring_profiles_active} + volumes: + - 'service-config:/config' + depends_on: [log] + command: ["/wait-for-postgres.sh", "/run.sh"] + mem_limit: '0.7G' + cpus: 2 + networks: + - public_network + + pcmt-integration: + image: openlmistz/pcmt-integration:1.0.1-SNAPSHOT + environment: + JAVA_OPTS: '-server -Xmx512m -Dlogging.config=/config/log/logback.xml' + links: + - db + - log + volumes: + - 'service-config:/config' + depends_on: [ log ] + ports: + - "8080" + env_file: settings.env + mem_limit: '0.7G' + cpus: 2 + networks: + - public_network + + stockmanagement: + restart: always + image: openlmis/stockmanagement:${OL_STOCKMANAGEMENT_VERSION} + env_file: settings.env + environment: + JAVA_OPTS: '-server -Xmx512m -Dlogging.config=/config/log/logback.xml' + spring_profiles_active: ${spring_profiles_active} + volumes: + - 'service-config:/config' + depends_on: [log] + command: ["/wait-for-postgres.sh", "/run.sh"] + mem_limit: '0.7G' + cpus: 2 + networks: + - public_network + + buq: + restart: always + image: openlmistz/buq:${OL_BUQ_VERSION} + env_file: settings.env + environment: + JAVA_OPTS: '-server -Xmx512m -Dlogging.config=/config/log/logback.xml' + spring_profiles_active: ${spring_profiles_active} + volumes: + - 'service-config:/config' + depends_on: [ log ] + command: [ "/wait-for-postgres.sh", "/run.sh" ] + mem_limit: '0.7G' + cpus: 2 + networks: + - public_network + + report: + restart: always + image: openlmis/report:${OL_REPORT_VERSION} + env_file: settings.env + environment: + JAVA_OPTS: '-server -Xmx512m -Dlogging.config=/config/log/logback.xml -Dflyway.locations=classpath:db/migration,filesystem:/demo-data' + spring_profiles_active: ${spring_profiles_active} + volumes: + - 'service-config:/config' + depends_on: [log] + command: ["/wait-for-postgres.sh", "/run.sh"] + mem_limit: '0.7G' + cpus: 2 + networks: + - public_network + + hapifhir: + restart: always + image: openlmis/hapifhir:${OL_HAPIFHIR_VERSION} + env_file: settings.env + environment: + JAVA_OPTS: '-server -Xmx512m -Dlogging.config=/config/log/logback.xml' + spring_profiles_active: ${spring_profiles_active} + volumes: + - 'service-config:/config' + depends_on: [ log ] + command: [ "/wait-for-postgres.sh", "/run.sh" ] + mem_limit: '0.7G' + cpus: 2 + networks: + - public_network + + diagnostics: + restart: always + image: openlmis/diagnostics:${OL_DIAGNOSTICS_VERSION} + env_file: settings.env + environment: + JAVA_OPTS: '-server -Xmx512m -Dlogging.config=/config/log/logback.xml' + spring_profiles_active: ${spring_profiles_active} + volumes: + - 'service-config:/config' + depends_on: [log] + mem_limit: '0.7G' + cpus: 2 + networks: + - public_network + + dhis2-integration: + image: openlmis/dhis2-integration:${OL_DHIS2_INTEGRATION} + env_file: settings.env + environment: + JAVA_OPTS: '-server -Xmx512m -Dlogging.config=/config/log/logback.xml' + spring_profiles_active: ${spring_profiles_active} + volumes: + - 'service-config:/config' + depends_on: [log] + mem_limit: '0.7G' + cpus: 2 + networks: + - public_network + + db: + restart: always + image: openlmis/postgres:${OL_POSTGRES_VERSION} + command: postgres -c 'max_connections=1000' + env_file: settings.env + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: p@ssw0rd + ports: + - 5432:5432 + networks: + public_network: + aliases: + - olmis-db + volumes: + - 'pg-data-prod:/var/lib/postgresql/data' + depends_on: [consul] + mem_limit: '1.3G' + cpus: 4 + + log: + restart: always + image: openlmis/rsyslog:${OL_RSYSLOG_VERSION} + volumes: + - 'syslog:/var/log' + depends_on: + - service-configuration + - consul + networks: + - public_network + + scalyr: + image: openlmis/scalyr + container_name: scalyr-agent + env_file: settings.env + volumes: + - 'service-config:/config' + - '/run/docker.sock:/var/scalyr/docker.sock' + - 'syslog:/var/log' + - 'nginx-log:/var/log/nginx/log' + - 'auth-tomcat-log:/var/log/auth-tomcat' + networks: + - public_network + + service-configuration: + build: + context: ./config + volumes: + - service-config:/config + networks: + - public_network + + ftp: + image: driesva/proftpd + ports: + - "${OL_FTP_PORT_21:-21}:21" + - "${OL_FTP_PORT_20:-20}:20" + - "21100-21110:21100-21110" + env_file: settings.env + depends_on: [consul] + networks: + - public_network + + redis: + image: redis:3.2.12 + depends_on: [consul] + networks: + - public_network + +volumes: + syslog: + external: false + nginx-log: + external: false + auth-tomcat-log: + external: false + consul-template-log: + external: false + service-config: + external: false + pg-data-tz: + external: true + pg-data-prod: + external: true + fulfillment-extensions: + external: false + +networks: + public_network: + external: true diff --git a/deployment/tz-single-server-deployment/nginx-config/report.training.tz.elmis-dev.org b/deployment/tz-single-server-deployment/nginx-config/report.training.tz.elmis-dev.org new file mode 100644 index 0000000..29a440b --- /dev/null +++ b/deployment/tz-single-server-deployment/nginx-config/report.training.tz.elmis-dev.org @@ -0,0 +1,42 @@ +# Redirect HTTP to HTTPS +server { + listen 80; + server_name report.training.tz.elmis-dev.org; + return 301 https://$host$request_uri; +} + +# HTTPS Configuration +server { + listen 443 ssl; + server_name report.training.tz.elmis-dev.org; + + ssl_certificate /etc/letsencrypt/live/report.training.tz.elmis-dev.org/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/report.training.tz.elmis-dev.org/privkey.pem; + + location / { + proxy_pass http://localhost:8088; # Ensure Superset is running on this port + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + + add_header Content-Security-Policy "frame-ancestors https://training.tz.elmis-dev.org"; + #add_header X-Frame-Options "ALLOW-FROM https://training.tz.elmis-dev.org"; + #add_header Content-Security-Policy "default-src 'self'; img-src 'self' https://www.google-analytics.com data:; script-src 'self' https://www.google-analytics.com; style-src 'self' 'unsafe-inline';"; + + # Optional: Serve static files directly for better performance + #location /static/ { + # alias /path/to/superset/static/; + #} + + # Optional: Handle WebSocket connections + location /websocket { + proxy_pass http://localhost:8088; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } +} diff --git a/deployment/tz-single-server-deployment/nginx-config/training.tz.elmis-dev.org b/deployment/tz-single-server-deployment/nginx-config/training.tz.elmis-dev.org new file mode 100644 index 0000000..0e0daf9 --- /dev/null +++ b/deployment/tz-single-server-deployment/nginx-config/training.tz.elmis-dev.org @@ -0,0 +1,49 @@ +# HTTP Server Block - Redirect to HTTPS +server { + listen 80; + server_name training.tz.elmis-dev.org; + + # Redirect all HTTP traffic to HTTPS + return 301 https://$host$request_uri; +} + +# HTTPS Server Block +server { + listen 443 ssl; + server_name training.tz.elmis-dev.org; + + # SSL Configuration + ssl_certificate /etc/letsencrypt/live/training.tz.elmis-dev.org/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/training.tz.elmis-dev.org/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; # Recommended SSL settings from Let's Encrypt + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # DH parameters for stronger encryption + add_header Content-Security-Policy "frame-ancestors https://training.tz.elmis-dev.org"; + # Security Headers + #add_header Content-Security-Policy "default-src 'self'; img-src 'self' https://www.google-analytics.com data:; script-src 'self' https://www.google-analytics.com; style-src 'self' 'unsafe-inline';"; + #add_header X-Frame-Options "SAMEORIGIN"; + #add_header X-Content-Type-Options "nosniff"; + #add_header X-XSS-Protection "1; mode=block"; + #add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"; # Enable HSTS + + # Proxy Settings + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # Root Location - Serve your primary application + location / { + proxy_pass http://localhost:8090; # Update with your actual backend service + } + + # Error Pages + error_page 404 /404.html; + location = /404.html { + internal; + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + internal; + } +} diff --git a/deployment/tz-single-server-deployment/nginx.conf b/deployment/tz-single-server-deployment/nginx.conf new file mode 100644 index 0000000..e030a79 --- /dev/null +++ b/deployment/tz-single-server-deployment/nginx.conf @@ -0,0 +1,32 @@ +worker_processes 1; + +events { + worker_connections 1024; +} + +http { +server { + listen 443 ssl; + server_name training.tz.elmis-dev.org; + + ssl_certificate /etc/letsencrypt/live/elmis.moh.go.tz/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/elmis.moh.go.tz/privkey.pem; + ssl on; + + ssl_prefer_server_ciphers on; + + location / { + proxy_pass http://training.tz.elmis-dev.org:80; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} + +server { + listen 80; + server_name training.tz.elmis-dev.org; + return 301 https://$host$request_uri; +} +} diff --git a/deployment/tz-single-server-deployment/reporting/.env b/deployment/tz-single-server-deployment/reporting/.env new file mode 100644 index 0000000..1e577ce --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/.env @@ -0,0 +1,27 @@ +OL_CONFLUENT_VERSION=6.1.2 +OL_NIFI_VERSION=1.9.2 +OL_POSTGRES_VERSION=12-debezium +OL_SUPERSET_VERSION=3.0.3 +OL_SUPERSET_PATCHUP_VERSION=v0.5.1 +OL_RSYSLOG_VERSION=1 +OL_NGINX_VERSION=4 +OL_KAFKA_TOPOLOGY_BUILDER_VERSION=4.4 + +REPORTING_POSTGRES_PORT=5432 + +### NiFi Service ### +# The JVM Memory initial heap size +NIFI_JVM_HEAP_INIT="2g" +# The JVM Memory maximum heap size +NIFI_JVM_HEAP_MAX="2g" + +### Scalyr Service ### +# Path to the Docker socket file on the host machine +# Instructions on how to get the path can be found +# here https://www.scalyr.com/help/install-agent-docker +SCALYR_DOCKER_SOCK=/run/docker.sock + +### Rsyslog ### +# The UDP port to send logs to the syslog container +SYSLOG_UDP_PORT=51400 + diff --git a/deployment/tz-single-server-deployment/reporting/README.md b/deployment/tz-single-server-deployment/reporting/README.md new file mode 100644 index 0000000..e7c9e82 --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/README.md @@ -0,0 +1,168 @@ +# OpenLMIS Reporting Stack + +## Deploying to a Server +1. *Prerequisite:* You should first [deploy OpenLMIS](../README.md). OpenLMIS and the reporting stack must run on separate (virtual) machines. Your OpenLMIS instance should have HTTPS enabled to connect to the reporting stack. + +2. *Prerequisite:* You should have a server with two domain names pointing to it, one for Superset and one for NiFi (e.g. `nifi.foo.openlmis.org` and `superset.foo.openlmis.org`). Ports 80 and 443 should be open on the server. + +3. Configure OpenLMIS to authenticate with the reporting stack's components. If your OpenLMIS instance is using the `demo-data` spring profile, this is already setup. + You need two users in OpenLMIS: one for Superset and one for NiFi. The instructions below describe how to set up these users ("OAuth User for Superset" and "OpenLMIS user with all permissions for Superset"). + +4. From this directory (`openlmis-ref-distro/reporting/`) on the server, copy and configure your settings. + ``` + $ cp settings-sample.env settings.env + ``` + * Edit `settings.env` to match your setup. Despite generating the passwords, you will likely need to change + `VIRTUAL_HOST`, `TRUSTED_HOSTNAME`, `OL_BASE_URL` (to point to OpenLMIS) and `NIFI_DOMAIN_NAME` and `SUPERSET_DOMAIN_NAME` (which should point to the reporting stack). + Details on all the environment variables are below. + +5. Bring up the reporting stack by running [docker-compose](https://docs.docker.com/compose/) on the server: + ```sh + docker-compose up --build -d + ``` + * You might, at times, want to take down the stack including all the created volumes. We recommend you do this if you ever change configurations in either the [./db](./db) or [./config](./config) directories: + + ```sh + docker-compose down -v + ``` + +6. The Superset webapp should now be running on your server. From your local browser, navigate to your Superset domain. You should be presented with a sign-in screen: + Superset sign-in screen, with the first provider button outlined in red + +7. Click the arrow (outlined in red above) to select OpenLMIS OAuth provider. Then, click the "Sign In" button. Enter the credentials for your OpenLMIS user for Superset. On the next page, approve all the requested permissions. Now, when you navigate to the Superset domain, a dashboard should populate with data from your OpenLMIS instance. + +8. If you have an OpenLMIS user with the proper permissions, you can also log in to OpenLMIS and navigate to `https:///#!/reports/list` and then select a report that uses the reporting stack (e.g. stockouts) to access Superset from OpenLMIS. This will require authorizing Superset by following the on-screen instructions. + +9. You should be able to access NiFi by navigating to the `NIFI_DOMAIN_NAME`, if you set it up. + +## Upgrading Superset to a new version + +Upgrading Superset to new version consist of two steps: +1. Updating configuration volume with latest Superset configuration and customization files. +1. Building and starting a new Docker image with the upgraded version. + +##### Updating configuration volume + +The `./upgrade-superset-config.sh` script can be used to update the configuration volume in one go. +It must be executed on the Superset's Docker host system. +The scripts starts a 'dummy' container linked ot the config volume, and copies (with override) the new content +of `./config/services/superset`. + +##### Running new Docker image + +There are no special considerations. +Start and build the Docker image included in this project as described in **Deploying to a Server** section. + +## OAuth User for Superset + +In order to use user authentication in Superset by an OpenLMIS instance we need to create additional user in OpenLMIS. +It is the specific user with `authorizedGrantTypes` set to `authorization_code` + +Example of a SQL statement creating that user (superset:changeme): +``` +INSERT INTO auth.oauth_client_details (clientId,authorities,authorizedGrantTypes,clientSecret,"scope") +VALUES ('superset','TRUSTED_CLIENT','authorization_code','changeme','read,write'); +``` + +Don't forget to set newly created user's credentials in settings.env. Example: +``` +OL_SUPERSET_USER=superset +OL_SUPERSET_PASSWORD=changeme +``` + +## OpenLMIS user with all permissions for Superset + +The ETL process conducted via NiFi requires a user which has all permissions (all program + supervisory node pairs) to read data from all requisitions in the system. It should not be a simple admin, because sometime it doesn't has all permissions (eg. for requisitions) + +The simplest way to create that user is using the https://github.com/OpenLMIS/openlmis-refdata-seed tool. + +Note: Created user must have an email address. + + +Don't forget to set newly created user's credentials in settings.env. Example: +``` +OL_ADMIN_USERNAME=administrator +OL_ADMIN_PASSWORD=password +``` + +## Environment variables + +The environment variables in `settings.env` have to be set to successfully run the reporting stack. +The sample settings file can be found [here](settings-sample.env). The sample file contains description of each field. + +## Running Locally + +* While it is possible to run the reporting stack locally, it is not entirely supported, and you cannot run both the reporting stack and OpenLMIS locally at the same time. +* It's easiest to access superset and nifi by leaving `.env` alone, and adding + entries to our host's `/etc/hosts` file: + ``` + 127.0.0.1 nifi.local superset.local + ``` + +### Running Setup Without Scalyr + +There are some cases (when running on a dev machine, for instance) where you would prefer to spin-up the stack without the Scalyr container running. To do that, run docker-compose this way: + +```sh +docker-compose up --build -d --scale scalyr=0 +``` + +## Syncronization / Replication + +This stack uses Debezium and Kafka-Connect with Postgres connectors to replicate +the data (using Change Data Capture) from the OpenLMIS instance, to this +reporting stack. + +Operationally this means: +- The first time the stack is started, it must snapshot all of the data in + OpenLMIS, and transfer it here. This takes awhile - possibly days for large + data sets. +- If the stack is destroyed (as in the database deleted, or the volumes of + kafka or debezium deleted), then the replication between OpenLMIS and this + stack must be removed in the OpenLMIS side. +- If this stack becomes unavailable to OpenLMIS, the OpenLMIS db server will + accumulate WAL segments, and quickly use disk storage. This normally rectifies + itself once this stack is made available again to OpenLMIS. However it's + possible that this doesn't happen in time, and the OpenLMIS server will run + out of disk space. In this case the replication slots need to be manually + removed. + +### Removing replication slots in OpenLMIS + +The replication slots in OpenLMIS are automatically created by this stack upon +first connection. These replication slots will need to be removed in certain +cases. If this is done, then the replication progress between OpenLMIS and +this stack will be broken, and a new connection + snapshotting will likely need +to occur before operation can resume. + +To remove: + +1. obtain a PSQL connection to the OpenLMIS database. + ```shell + docker exec -it openlmis-db-1 psql -U postgres open_lmis + ``` +1. Show replications slots: + ```sql + select slot_name from pg_replication_slots; + + slot_name + ------------------- + dbz_referencedata + dbz_requisition + ``` + 1. Remove the slots: + ```sql + select pg_drop_replication_slot('dbz_referencedata'); + select pg_drop_replication_slot('dbz_requisition'); + ``` +## CI/CD + +### KEEP_OR_WIPE + +Is an environment variable that when set to `wipe` will be destructive. + +- `keep`: (default if not specified), does not delete database or volumes, + except for the config volume, which needs to be removed to allow for configuration + changes. +- `wipe`: **destructive**, will delete db and all volumes. If set the + [syncronization](#syncronization--replication) process will need to be restarted. \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/Dockerfile b/deployment/tz-single-server-deployment/reporting/config/Dockerfile new file mode 100644 index 0000000..da230b8 --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/Dockerfile @@ -0,0 +1,8 @@ +FROM alpine:3.11 + +COPY ./services /config +COPY ./init.sh /init.sh + +RUN apk add --update curl && rm -rf /var/cache/apk/* + +CMD /bin/true \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/init.sh b/deployment/tz-single-server-deployment/reporting/config/init.sh new file mode 100755 index 0000000..d8040ff --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/init.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +echo "Waiting for consul to be available" +while ! curl -f "http://consul:8500/v1/agent/self"; do + sleep 10; +done + +set -e + +curl \ + -X PUT \ + -H "Content-Type: application/json" \ + --data "{\"name\": \"superset-service\", \"address\": \"superset\", \"id\": \"superset1\", \"port\": 8088, \"tags\": [\"openlmis-service\"], \"check\": { \"http\": \"http://superset:8088/health\", \"method\": \"GET\", \"interval\": \"30s\", \"timeout\": \"10s\"}}" \ + http://consul:8500/v1/agent/service/register + +curl \ + -X PUT \ + --data "{ \"upstream\": \"superset-service\", \"enable_basic_auth\": false, \"behind_load_balancer\": ${SUPERSET_BEHIND_LOAD_BALANCER}, \"redirect_http_traffic\": ${SUPERSET_LOAD_BALANCER_REDIRECT_HTTP}, \"enable_ssl\": ${SUPERSET_ENABLE_SSL}, \"ssl_cert\": \"/config/nginx/tls/${SUPERSET_SSL_CERT}\", \"ssl_key\": \"/config/nginx/tls/${SUPERSET_SSL_KEY}\", \"ssl_cert_chain\": \"/config/nginx/tls/${SUPERSET_SSL_CERT_CHAIN}\"}" \ + http://consul:8500/v1/kv/resources/${SUPERSET_DOMAIN_NAME} + +curl \ + -X PUT \ + -H "Content-Type: application/json" \ + --data "{\"name\": \"nifi-service\", \"address\": \"nifi\", \"id\": \"nifi1\", \"port\": 8080, \"tags\": [\"openlmis-service\"], \"check\": { \"http\": \"http://nifi:8080\", \"method\": \"GET\", \"interval\": \"30s\", \"timeout\": \"10s\"}}" \ + http://consul:8500/v1/agent/service/register + +curl \ + -X PUT \ + --data "{ \"upstream\": \"nifi-service\", \"enable_basic_auth\": true, \"behind_load_balancer\": ${NIFI_BEHIND_LOAD_BALANCER}, \"redirect_http_traffic\": ${NIFI_LOAD_BALANCER_REDIRECT_HTTP}, \"enable_ssl\": ${NIFI_ENABLE_SSL}, \"ssl_cert\": \"/config/nginx/tls/${NIFI_SSL_CERT}\", \"ssl_key\": \"/config/nginx/tls/${NIFI_SSL_KEY}\", \"ssl_cert_chain\": \"/config/nginx/tls/${NIFI_SSL_CERT_CHAIN}\"}" \ + http://consul:8500/v1/kv/resources/${NIFI_DOMAIN_NAME} diff --git a/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/register.sh b/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/register.sh new file mode 100755 index 0000000..b651166 --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/register.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +set -e + +until curl -s -f "http://connect:8083/"; do + >&2 echo -e "\nDebezium Connect is unavailable - sleeping" + sleep 10 +done + +>&2 echo -e "\nDebezium Connect is up - registering connectors" + +# ensure some environment variables are set +: "${DATABASE_URL:?DATABASE_URL not set in environment}" +: "${POSTGRES_USER:?POSTGRES_USER not set in environment}" +: "${POSTGRES_PASSWORD:?POSTGRES_PASSWORD not set in environment}" +: "${SRC_POSTGRES_HOST:?SRC_POSTGRES_HOST not set in environment}" +: "${SRC_POSTGRES_PORT:?SRC_POSTGRES_POST not set in environment}" +: "${SRC_POSTGRES_DB:?SRC_POSTGRES_DB not set in environment}" +: "${SRC_POSTGRES_USER:?SRC_POSTGRES_USER not set in environment}" +: "${SRC_POSTGRES_PASSWORD:?SRC_POSTGRES_PASSWORD not set in environment}" + +# pull apart some of those pieces stuck together in DATABASE_URL +POSTGRES_HOST=`echo ${DATABASE_URL} | sed -E 's/^.*\/{2}(.+):.*$/\1/'` # //: +: "${POSTGRES_HOST:?Host not parsed}" + +POSTGRES_PORT=`echo ${DATABASE_URL} | sed -E 's/^.*\:([0-9]+)\/.*$/\1/'` # :/ +: "${POSTGRES_PORT:?Port not parsed}" + +POSTGRES_DB=`echo ${DATABASE_URL} | sed -E 's/^.*\/(.+)\?*$/\1/'` # /? +: "${POSTGRES_DB:?DB not set}" + +echo -e "\n\nReplacing database info of source JSON files, then registering" +for f in /config/connect/source-*.json +do + echo -e "\n\nProcessing $f file..." + mv $f temp.json + jq -r --arg dbhost "$SRC_POSTGRES_HOST" --arg dbport "$SRC_POSTGRES_PORT" --arg dbname "$SRC_POSTGRES_DB" --arg dbuser "$SRC_POSTGRES_USER" --arg dbpassword "$SRC_POSTGRES_PASSWORD" '.config["database.hostname"] |= $dbhost | .config["database.port"] |= $dbport | .config["database.dbname"] |= $dbname | .config["database.user"] |= $dbuser | .config["database.password"] |= $dbpassword' temp.json > $f + rm temp.json + curl -s -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://connect:8083/connectors/ -d @$f +done + +echo -e "\n\nReplacing database info of sink JSON files, then registering" +for f in /config/connect/sink-*.json +do + echo -e "\n\nProcessing $f file..." + mv $f temp.json + jq -r --arg dburl "$DATABASE_URL" --arg dbuser "$POSTGRES_USER" --arg dbpassword "$SUPERSET_POSTGRES_PASSWORD" '.config["connection.url"] |= $dburl | .config["connection.user"] |= $dbuser | .config["connection.password"] |= $dbpassword' temp.json > $f + rm temp.json + curl -s -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://connect:8083/connectors/ -d @$f +done + +echo -e "\n\nRegistered all connectors!" diff --git a/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-contract-equipments.json b/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-contract-equipments.json new file mode 100644 index 0000000..82ae40d --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-contract-equipments.json @@ -0,0 +1,17 @@ +{ + "name": "sink-contract-equipments", + "config": { + "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector", + "tasks.max": "1", + "topics": "public.kafka_contract_equipments", + "connection.url": "jdbc:postgresql://reporting-db:5432/open_lmis_reporting?stringtype=unspecified", + "connection.user": "postgres", + "connection.password": "p@ssw0rd", + "auto.create": "true", + "insert.mode": "upsert", + "pk.fields": "contractid,equipmentid", + "pk.mode": "record_key", + "delete.enabled": "true" + } +} + \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-contract-facilities.json b/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-contract-facilities.json new file mode 100644 index 0000000..fd8e98c --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-contract-facilities.json @@ -0,0 +1,17 @@ +{ + "name": "sink-contract-facilities", + "config": { + "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector", + "tasks.max": "1", + "topics": "public.kafka_contract_facilities", + "connection.url": "jdbc:postgresql://reporting-db:5432/open_lmis_reporting?stringtype=unspecified", + "connection.user": "postgres", + "connection.password": "p@ssw0rd", + "auto.create": "true", + "insert.mode": "upsert", + "pk.fields": "contractid,facilityid", + "pk.mode": "record_key", + "delete.enabled": "true" + } +} + \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-contract-service-types.json b/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-contract-service-types.json new file mode 100644 index 0000000..634f528 --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-contract-service-types.json @@ -0,0 +1,17 @@ +{ + "name": "sink-contract-service-types", + "config": { + "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector", + "tasks.max": "1", + "topics": "public.kafka_contract_service_types", + "connection.url": "jdbc:postgresql://reporting-db:5432/open_lmis_reporting?stringtype=unspecified", + "connection.user": "postgres", + "connection.password": "p@ssw0rd", + "auto.create": "true", + "insert.mode": "upsert", + "pk.fields": "contractid,servicetypeid", + "pk.mode": "record_key", + "delete.enabled": "true" + } +} + \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-equipment.json b/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-equipment.json new file mode 100644 index 0000000..e011ba0 --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-equipment.json @@ -0,0 +1,17 @@ +{ + "name": "sink-equipment", + "config": { + "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector", + "tasks.max": "1", + "topics": "public.kafka_contracts,public.kafka_equipment_categories,public.kafka_equipment_disciplines,public.kafka_equipment_energy_types,public.kafka_equipment_fund_sources,public.kafka_equipment_inventory,public.kafka_equipment_maintenance_request_reasons,public.kafka_equipment_maintenance_requests,public.kafka_equipment_models,public.kafka_equipment_service_logs,public.kafka_equipment_type_programs,public.kafka_equipment_types,public.kafka_equipments,public.kafka_service_types,public.kafka_vendor", + "connection.url": "jdbc:postgresql://reporting-db:5432/open_lmis_reporting?stringtype=unspecified", + "connection.user": "postgres", + "connection.password": "p@ssw0rd", + "auto.create": "true", + "insert.mode": "upsert", + "pk.fields": "id", + "pk.mode": "record_key", + "delete.enabled": "true" + } +} + diff --git a/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-orderable-identifiers.json b/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-orderable-identifiers.json new file mode 100644 index 0000000..668093d --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-orderable-identifiers.json @@ -0,0 +1,17 @@ +{ + "name": "sink-orderable-identifiers", + "config": { + "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector", + "tasks.max": "1", + "topics": "public.kafka_orderable_identifiers", + "connection.url": "jdbc:postgresql://reporting-db:5432/open_lmis_reporting?stringtype=unspecified", + "connection.user": "postgres", + "connection.password": "p@ssw0rd", + "auto.create": "true", + "insert.mode": "upsert", + "pk.fields": "key,orderableid,orderableversionnumber", + "pk.mode": "record_key", + "delete.enabled": "true" + } +} + \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-orderables.json b/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-orderables.json new file mode 100644 index 0000000..83fb861 --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-orderables.json @@ -0,0 +1,17 @@ +{ + "name": "sink-orderables", + "config": { + "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector", + "tasks.max": "1", + "topics": "public.kafka_orderables", + "connection.url": "jdbc:postgresql://reporting-db:5432/open_lmis_reporting?stringtype=unspecified", + "connection.user": "postgres", + "connection.password": "p@ssw0rd", + "auto.create": "true", + "insert.mode": "upsert", + "pk.fields": "id,versionnumber", + "pk.mode": "record_key", + "delete.enabled": "true" + } +} + \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-referencedata.json b/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-referencedata.json new file mode 100644 index 0000000..8c817cb --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-referencedata.json @@ -0,0 +1,17 @@ +{ + "name": "sink-referencedata", + "config": { + "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector", + "tasks.max": "1", + "topics": "public.kafka_commodity_types,public.kafka_geographic_levels,public.kafka_geographic_zones,public.kafka_facility_operators,public.kafka_facility_types,public.kafka_facilities,public.kafka_ideal_stock_amounts,public.kafka_lots,public.kafka_processing_periods,public.kafka_processing_schedules,public.kafka_program_orderables,public.kafka_programs,public.kafka_requisition_group_program_schedules,public.kafka_trade_items,public.kafka_rights,public.kafka_users,public.kafka_roles,public.kafka_supervisory_nodes,public.kafka_requisition_groups,public.kafka_supply_lines,public.kafka_right_assignments", + "connection.url": "jdbc:postgresql://reporting-db:5432/open_lmis_reporting?stringtype=unspecified", + "connection.user": "postgres", + "connection.password": "p@ssw0rd", + "auto.create": "true", + "insert.mode": "upsert", + "pk.fields": "id", + "pk.mode": "record_key", + "delete.enabled": "true" + } +} + \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-requisition-group-members.json b/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-requisition-group-members.json new file mode 100644 index 0000000..ed36717 --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-requisition-group-members.json @@ -0,0 +1,17 @@ +{ + "name": "sink-requisition-group-members", + "config": { + "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector", + "tasks.max": "1", + "topics": "public.kafka_requisition_group_members", + "connection.url": "jdbc:postgresql://reporting-db:5432/open_lmis_reporting?stringtype=unspecified", + "connection.user": "postgres", + "connection.password": "p@ssw0rd", + "auto.create": "true", + "insert.mode": "upsert", + "pk.fields": "requisitiongroupid,facilityid", + "pk.mode": "record_key", + "delete.enabled": "true" + } +} + \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-requisition.json b/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-requisition.json new file mode 100644 index 0000000..41a182c --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-requisition.json @@ -0,0 +1,17 @@ +{ + "name": "sink-requisition", + "config": { + "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector", + "tasks.max": "1", + "topics": "public.kafka_requisitions,public.kafka_requisition_line_items,public.kafka_status_changes,public.kafka_stock_adjustments,public.kafka_stock_adjustment_reasons", + "connection.url": "jdbc:postgresql://reporting-db:5432/open_lmis_reporting?stringtype=unspecified", + "connection.user": "postgres", + "connection.password": "p@ssw0rd", + "auto.create": "true", + "insert.mode": "upsert", + "pk.fields": "id", + "pk.mode": "record_key", + "delete.enabled": "true" + } +} + \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-supported-programs.json b/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-supported-programs.json new file mode 100644 index 0000000..5bacd40 --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/sink-supported-programs.json @@ -0,0 +1,17 @@ +{ + "name": "sink-supported-programs", + "config": { + "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector", + "tasks.max": "1", + "topics": "public.kafka_supported_programs", + "connection.url": "jdbc:postgresql://reporting-db:5432/open_lmis_reporting?stringtype=unspecified", + "connection.user": "postgres", + "connection.password": "p@ssw0rd", + "auto.create": "true", + "insert.mode": "upsert", + "pk.fields": "facilityid,programid", + "pk.mode": "record_key", + "delete.enabled": "true" + } +} + \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/source-equipment.json b/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/source-equipment.json new file mode 100644 index 0000000..0eb3437 --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/source-equipment.json @@ -0,0 +1,33 @@ +{ + "name": "source-equipment", + "config": { + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "tasks.max": "1", + "plugin.name": "wal2json", + "database.hostname": "olmis-db", + "database.port": "5432", + "database.user": "postgres", + "database.password": "p@ssw0rd", + "database.dbname": "open_lmis", + "database.server.name": "openlmis", + "table.whitelist": "equipment\\.contract_equipments,equipment\\.contract_facilities,equipment\\.contract_service_types,equipment\\.contracts,equipment\\.equipment_catalog_items,equipment\\.equipment_categories,equipment\\.equipment_disciplines,equipment\\.equipment_energy_types,equipment\\.equipment_fund_sources,equipment\\.equipment_inventory,equipment\\.equipment_inventory_items,equipment\\.equipment_maintenance_request_reasons,equipment\\.equipment_maintenance_requests,equipment\\.equipment_models,equipment\\.equipment_operation_modes,equipment\\.equipment_orderables,equipment\\.equipment_service_logs,equipment\\.equipment_test_types,equipment\\.equipment_type_programs,equipment\\.equipment_types,equipment\\.equipments,equipment\\.reasons_not_functional,equipment\\.service_types,equipment\\.vendor", + "database.history.kafka.bootstrap.servers": "kafka:29092", + "database.history.kafka.topic": "openlmis", + "slot.name": "dbz_equipment", + "heartbeat.interval.ms": 30000, + "heartbeat.action.query": "CREATE TABLE IF NOT EXISTS debezium_heartbeat (id SERIAL PRIMARY KEY, ts TIMESTAMPTZ); INSERT INTO debezium_heartbeat (id, ts) VALUES (1, NOW()) ON CONFLICT(id) DO UPDATE SET ts=NOW();", + "transforms": "unwrap,route", + "transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState", + "transforms.unwrap.drop.tombstones": "false", + "transforms.unwrap.delete.handling.mode": "none", + "transforms.route.type": "org.apache.kafka.connect.transforms.RegexRouter", + "transforms.route.regex": "([^.]+)\\.([^.]+)\\.([^.]+)", + "transforms.route.replacement": "public.kafka_$3", + "max.batch.size": "1024", + "max.queue.size": "4096", + "topic.creation.default.compression.type": "snappy", + "topic.creation.default.replication.factor": 1, + "topic.creation.default.partitions": 1 + } + } + \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/source-referencedata.json b/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/source-referencedata.json new file mode 100644 index 0000000..c518a3a --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/source-referencedata.json @@ -0,0 +1,33 @@ +{ + "name": "source-referencedata", + "config": { + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "tasks.max": "1", + "plugin.name": "wal2json", + "database.hostname": "olmis-db", + "database.port": "5432", + "database.user": "postgres", + "database.password": "p@ssw0rd", + "database.dbname": "open_lmis", + "database.server.name": "openlmis", + "table.whitelist": "referencedata\\.commodity_types,referencedata\\.geographic_levels,referencedata\\.geographic_zones,referencedata\\.facility_operators,referencedata\\.facility_types,referencedata\\.facilities,referencedata\\.ideal_stock_amounts,referencedata\\.lots,referencedata\\.orderable_identifiers,referencedata\\.orderables,referencedata\\.processing_periods,referencedata\\.processing_schedules,referencedata\\.program_orderables,referencedata\\.programs,referencedata\\.requisition_group_members,referencedata\\.requisition_group_program_schedules,referencedata\\.supported_programs,referencedata\\.trade_items,referencedata\\.rights,referencedata\\.users,referencedata\\.roles,referencedata\\.supervisory_nodes,referencedata\\.requisition_groups,referencedata\\.supply_lines,referencedata\\.right_assignments", + "database.history.kafka.bootstrap.servers": "kafka:29092", + "database.history.kafka.topic": "openlmis", + "column.blacklist": "referencedata.facilities.location,referencedata.geographic_zones.boundary", + "slot.name": "dbz_referencedata", + "time.precision.mode": "connect", + "heartbeat.interval.ms": 3000, + "heartbeat.action.query": "CREATE TABLE IF NOT EXISTS debezium_heartbeat (id SERIAL PRIMARY KEY, ts TIMESTAMPTZ); INSERT INTO debezium_heartbeat (id, ts) VALUES (1, NOW()) ON CONFLICT(id) DO UPDATE SET ts=NOW();", + "transforms": "unwrap,route", + "transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState", + "transforms.unwrap.drop.tombstones": "false", + "transforms.unwrap.delete.handling.mode": "none", + "transforms.route.type": "org.apache.kafka.connect.transforms.RegexRouter", + "transforms.route.regex": "([^.]+)\\.([^.]+)\\.([^.]+)", + "transforms.route.replacement": "public.kafka_$3", + "topic.creation.default.compression.type": "lz4", + "topic.creation.default.replication.factor": 1, + "topic.creation.default.partitions": 1 + } + } + \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/source-requisition.json b/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/source-requisition.json new file mode 100644 index 0000000..e4bf6ea --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/backupconnect/source-requisition.json @@ -0,0 +1,33 @@ +{ + "name": "source-requisition", + "config": { + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "tasks.max": "1", + "plugin.name": "wal2json", + "database.hostname": "olmis-db", + "database.port": "5432", + "database.user": "postgres", + "database.password": "p@ssw0rd", + "database.dbname": "open_lmis", + "database.server.name": "openlmis", + "table.whitelist": "requisition\\.requisitions,requisition\\.requisition_line_items,requisition\\.status_changes,requisition\\.stock_adjustments,requisition\\.stock_adjustment_reasons", + "database.history.kafka.bootstrap.servers": "kafka:29092", + "database.history.kafka.topic": "openlmis", + "slot.name": "dbz_requisition", + "heartbeat.interval.ms": 30000, + "heartbeat.action.query": "CREATE TABLE IF NOT EXISTS debezium_heartbeat (id SERIAL PRIMARY KEY, ts TIMESTAMPTZ); INSERT INTO debezium_heartbeat (id, ts) VALUES (1, NOW()) ON CONFLICT(id) DO UPDATE SET ts=NOW();", + "transforms": "unwrap,route", + "transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState", + "transforms.unwrap.drop.tombstones": "false", + "transforms.unwrap.delete.handling.mode": "none", + "transforms.route.type": "org.apache.kafka.connect.transforms.RegexRouter", + "transforms.route.regex": "([^.]+)\\.([^.]+)\\.([^.]+)", + "transforms.route.replacement": "public.kafka_$3", + "max.batch.size": "1024", + "max.queue.size": "4096", + "topic.creation.default.compression.type": "snappy", + "topic.creation.default.replication.factor": 1, + "topic.creation.default.partitions": 1 + } + } + \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/services/connect/register.sh b/deployment/tz-single-server-deployment/reporting/config/services/connect/register.sh new file mode 100755 index 0000000..b651166 --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/connect/register.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +set -e + +until curl -s -f "http://connect:8083/"; do + >&2 echo -e "\nDebezium Connect is unavailable - sleeping" + sleep 10 +done + +>&2 echo -e "\nDebezium Connect is up - registering connectors" + +# ensure some environment variables are set +: "${DATABASE_URL:?DATABASE_URL not set in environment}" +: "${POSTGRES_USER:?POSTGRES_USER not set in environment}" +: "${POSTGRES_PASSWORD:?POSTGRES_PASSWORD not set in environment}" +: "${SRC_POSTGRES_HOST:?SRC_POSTGRES_HOST not set in environment}" +: "${SRC_POSTGRES_PORT:?SRC_POSTGRES_POST not set in environment}" +: "${SRC_POSTGRES_DB:?SRC_POSTGRES_DB not set in environment}" +: "${SRC_POSTGRES_USER:?SRC_POSTGRES_USER not set in environment}" +: "${SRC_POSTGRES_PASSWORD:?SRC_POSTGRES_PASSWORD not set in environment}" + +# pull apart some of those pieces stuck together in DATABASE_URL +POSTGRES_HOST=`echo ${DATABASE_URL} | sed -E 's/^.*\/{2}(.+):.*$/\1/'` # //: +: "${POSTGRES_HOST:?Host not parsed}" + +POSTGRES_PORT=`echo ${DATABASE_URL} | sed -E 's/^.*\:([0-9]+)\/.*$/\1/'` # :/ +: "${POSTGRES_PORT:?Port not parsed}" + +POSTGRES_DB=`echo ${DATABASE_URL} | sed -E 's/^.*\/(.+)\?*$/\1/'` # /? +: "${POSTGRES_DB:?DB not set}" + +echo -e "\n\nReplacing database info of source JSON files, then registering" +for f in /config/connect/source-*.json +do + echo -e "\n\nProcessing $f file..." + mv $f temp.json + jq -r --arg dbhost "$SRC_POSTGRES_HOST" --arg dbport "$SRC_POSTGRES_PORT" --arg dbname "$SRC_POSTGRES_DB" --arg dbuser "$SRC_POSTGRES_USER" --arg dbpassword "$SRC_POSTGRES_PASSWORD" '.config["database.hostname"] |= $dbhost | .config["database.port"] |= $dbport | .config["database.dbname"] |= $dbname | .config["database.user"] |= $dbuser | .config["database.password"] |= $dbpassword' temp.json > $f + rm temp.json + curl -s -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://connect:8083/connectors/ -d @$f +done + +echo -e "\n\nReplacing database info of sink JSON files, then registering" +for f in /config/connect/sink-*.json +do + echo -e "\n\nProcessing $f file..." + mv $f temp.json + jq -r --arg dburl "$DATABASE_URL" --arg dbuser "$POSTGRES_USER" --arg dbpassword "$SUPERSET_POSTGRES_PASSWORD" '.config["connection.url"] |= $dburl | .config["connection.user"] |= $dbuser | .config["connection.password"] |= $dbpassword' temp.json > $f + rm temp.json + curl -s -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://connect:8083/connectors/ -d @$f +done + +echo -e "\n\nRegistered all connectors!" diff --git a/deployment/tz-single-server-deployment/reporting/config/services/connect/sink-equipment.json b/deployment/tz-single-server-deployment/reporting/config/services/connect/sink-equipment.json new file mode 100644 index 0000000..e011ba0 --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/connect/sink-equipment.json @@ -0,0 +1,17 @@ +{ + "name": "sink-equipment", + "config": { + "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector", + "tasks.max": "1", + "topics": "public.kafka_contracts,public.kafka_equipment_categories,public.kafka_equipment_disciplines,public.kafka_equipment_energy_types,public.kafka_equipment_fund_sources,public.kafka_equipment_inventory,public.kafka_equipment_maintenance_request_reasons,public.kafka_equipment_maintenance_requests,public.kafka_equipment_models,public.kafka_equipment_service_logs,public.kafka_equipment_type_programs,public.kafka_equipment_types,public.kafka_equipments,public.kafka_service_types,public.kafka_vendor", + "connection.url": "jdbc:postgresql://reporting-db:5432/open_lmis_reporting?stringtype=unspecified", + "connection.user": "postgres", + "connection.password": "p@ssw0rd", + "auto.create": "true", + "insert.mode": "upsert", + "pk.fields": "id", + "pk.mode": "record_key", + "delete.enabled": "true" + } +} + diff --git a/deployment/tz-single-server-deployment/reporting/config/services/connect/sink-fulfillment.json b/deployment/tz-single-server-deployment/reporting/config/services/connect/sink-fulfillment.json new file mode 100644 index 0000000..9f2706b --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/connect/sink-fulfillment.json @@ -0,0 +1,17 @@ +{ + "name": "sink-fulfillment", + "config": { + "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector", + "tasks.max": "1", + "topics": "public.kafka_orders,public.kafka_order_line_items,public.kafka_shipments,public.kafka_shipment_line_items", + "connection.url": "jdbc:postgresql://reporting-db:5432/open_lmis_reporting?stringtype=unspecified", + "connection.user": "postgres", + "connection.password": "p@ssw0rd", + "auto.create": "true", + "insert.mode": "upsert", + "pk.fields": "id", + "pk.mode": "record_key", + "delete.enabled": "true" + } +} + \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/services/connect/sink-orderable-identifiers.json b/deployment/tz-single-server-deployment/reporting/config/services/connect/sink-orderable-identifiers.json new file mode 100644 index 0000000..668093d --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/connect/sink-orderable-identifiers.json @@ -0,0 +1,17 @@ +{ + "name": "sink-orderable-identifiers", + "config": { + "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector", + "tasks.max": "1", + "topics": "public.kafka_orderable_identifiers", + "connection.url": "jdbc:postgresql://reporting-db:5432/open_lmis_reporting?stringtype=unspecified", + "connection.user": "postgres", + "connection.password": "p@ssw0rd", + "auto.create": "true", + "insert.mode": "upsert", + "pk.fields": "key,orderableid,orderableversionnumber", + "pk.mode": "record_key", + "delete.enabled": "true" + } +} + \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/services/connect/sink-orderables.json b/deployment/tz-single-server-deployment/reporting/config/services/connect/sink-orderables.json new file mode 100644 index 0000000..83fb861 --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/connect/sink-orderables.json @@ -0,0 +1,17 @@ +{ + "name": "sink-orderables", + "config": { + "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector", + "tasks.max": "1", + "topics": "public.kafka_orderables", + "connection.url": "jdbc:postgresql://reporting-db:5432/open_lmis_reporting?stringtype=unspecified", + "connection.user": "postgres", + "connection.password": "p@ssw0rd", + "auto.create": "true", + "insert.mode": "upsert", + "pk.fields": "id,versionnumber", + "pk.mode": "record_key", + "delete.enabled": "true" + } +} + \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/services/connect/sink-referencedata.json b/deployment/tz-single-server-deployment/reporting/config/services/connect/sink-referencedata.json new file mode 100644 index 0000000..495e24e --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/connect/sink-referencedata.json @@ -0,0 +1,17 @@ +{ + "name": "sink-referencedata", + "config": { + "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector", + "tasks.max": "1", + "topics": "public.kafka_commodity_types,public.kafka_geographic_levels,public.kafka_geographic_zones,public.kafka_facility_operators,public.kafka_facility_types,public.kafka_facilities,public.kafka_ideal_stock_amounts,public.kafka_lots,public.kafka_processing_periods,public.kafka_processing_schedules,public.kafka_program_orderables,public.kafka_programs,public.kafka_requisition_group_program_schedules,public.kafka_trade_items,public.kafka_rights,public.kafka_users,public.kafka_roles,public.kafka_supervisory_nodes,public.kafka_requisition_groups,public.kafka_supply_lines,public.kafka_right_assignments,public.kafka_facility_type_approved_products", + "connection.url": "jdbc:postgresql://reporting-db:5432/open_lmis_reporting?stringtype=unspecified", + "connection.user": "postgres", + "connection.password": "p@ssw0rd", + "auto.create": "true", + "insert.mode": "upsert", + "pk.fields": "id", + "pk.mode": "record_key", + "delete.enabled": "true" + } +} + \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/services/connect/sink-requisition-group-members.json b/deployment/tz-single-server-deployment/reporting/config/services/connect/sink-requisition-group-members.json new file mode 100644 index 0000000..ed36717 --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/connect/sink-requisition-group-members.json @@ -0,0 +1,17 @@ +{ + "name": "sink-requisition-group-members", + "config": { + "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector", + "tasks.max": "1", + "topics": "public.kafka_requisition_group_members", + "connection.url": "jdbc:postgresql://reporting-db:5432/open_lmis_reporting?stringtype=unspecified", + "connection.user": "postgres", + "connection.password": "p@ssw0rd", + "auto.create": "true", + "insert.mode": "upsert", + "pk.fields": "requisitiongroupid,facilityid", + "pk.mode": "record_key", + "delete.enabled": "true" + } +} + \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/services/connect/sink-requisition.json b/deployment/tz-single-server-deployment/reporting/config/services/connect/sink-requisition.json new file mode 100644 index 0000000..41a182c --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/connect/sink-requisition.json @@ -0,0 +1,17 @@ +{ + "name": "sink-requisition", + "config": { + "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector", + "tasks.max": "1", + "topics": "public.kafka_requisitions,public.kafka_requisition_line_items,public.kafka_status_changes,public.kafka_stock_adjustments,public.kafka_stock_adjustment_reasons", + "connection.url": "jdbc:postgresql://reporting-db:5432/open_lmis_reporting?stringtype=unspecified", + "connection.user": "postgres", + "connection.password": "p@ssw0rd", + "auto.create": "true", + "insert.mode": "upsert", + "pk.fields": "id", + "pk.mode": "record_key", + "delete.enabled": "true" + } +} + \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/services/connect/sink-supported-programs.json b/deployment/tz-single-server-deployment/reporting/config/services/connect/sink-supported-programs.json new file mode 100644 index 0000000..5bacd40 --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/connect/sink-supported-programs.json @@ -0,0 +1,17 @@ +{ + "name": "sink-supported-programs", + "config": { + "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector", + "tasks.max": "1", + "topics": "public.kafka_supported_programs", + "connection.url": "jdbc:postgresql://reporting-db:5432/open_lmis_reporting?stringtype=unspecified", + "connection.user": "postgres", + "connection.password": "p@ssw0rd", + "auto.create": "true", + "insert.mode": "upsert", + "pk.fields": "facilityid,programid", + "pk.mode": "record_key", + "delete.enabled": "true" + } +} + \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/services/connect/source-equipment.json b/deployment/tz-single-server-deployment/reporting/config/services/connect/source-equipment.json new file mode 100644 index 0000000..7f612d7 --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/connect/source-equipment.json @@ -0,0 +1,33 @@ +{ + "name": "source-equipment", + "config": { + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "tasks.max": "1", + "plugin.name": "wal2json", + "database.hostname": "olmis-db", + "database.port": "5432", + "database.user": "postgres", + "database.password": "p@ssw0rd", + "database.dbname": "open_lmis", + "database.server.name": "openlmis", + "table.whitelist": "equipment\\.contracts,equipment\\.equipment_catalog_items,equipment\\.equipment_categories,equipment\\.equipment_disciplines,equipment\\.equipment_energy_types,equipment\\.equipment_fund_sources,equipment\\.equipment_inventory,equipment\\.equipment_inventory_items,equipment\\.equipment_maintenance_request_reasons,equipment\\.equipment_maintenance_requests,equipment\\.equipment_models,equipment\\.equipment_operation_modes,equipment\\.equipment_orderables,equipment\\.equipment_service_logs,equipment\\.equipment_test_types,equipment\\.equipment_type_programs,equipment\\.equipment_types,equipment\\.equipments,equipment\\.reasons_not_functional,equipment\\.service_types,equipment\\.vendor", + "database.history.kafka.bootstrap.servers": "kafka:29092", + "database.history.kafka.topic": "openlmis", + "slot.name": "dbz_equipment", + "heartbeat.interval.ms": 30000, + "heartbeat.action.query": "CREATE TABLE IF NOT EXISTS debezium_heartbeat (id SERIAL PRIMARY KEY, ts TIMESTAMPTZ); INSERT INTO debezium_heartbeat (id, ts) VALUES (1, NOW()) ON CONFLICT(id) DO UPDATE SET ts=NOW();", + "transforms": "unwrap,route", + "transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState", + "transforms.unwrap.drop.tombstones": "false", + "transforms.unwrap.delete.handling.mode": "none", + "transforms.route.type": "org.apache.kafka.connect.transforms.RegexRouter", + "transforms.route.regex": "([^.]+)\\.([^.]+)\\.([^.]+)", + "transforms.route.replacement": "public.kafka_$3", + "max.batch.size": "1024", + "max.queue.size": "4096", + "topic.creation.default.compression.type": "snappy", + "topic.creation.default.replication.factor": 1, + "topic.creation.default.partitions": 1 + } + } + \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/services/connect/source-fulfillment.json b/deployment/tz-single-server-deployment/reporting/config/services/connect/source-fulfillment.json new file mode 100644 index 0000000..1629532 --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/connect/source-fulfillment.json @@ -0,0 +1,33 @@ +{ + "name": "source-fulfillment", + "config": { + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "tasks.max": "1", + "plugin.name": "wal2json", + "database.hostname": "olmis-db", + "database.port": "5432", + "database.user": "postgres", + "database.password": "p@ssw0rd", + "database.dbname": "open_lmis", + "database.server.name": "openlmis", + "table.whitelist": "fulfillment\\.orders,fulfillment\\.order_line_items,fulfillment\\.shipments,fulfillment\\.shipment_line_items", + "database.history.kafka.bootstrap.servers": "kafka:29092", + "database.history.kafka.topic": "openlmis", + "slot.name": "dbz_fulfillment", + "heartbeat.interval.ms": 30000, + "heartbeat.action.query": "CREATE TABLE IF NOT EXISTS debezium_heartbeat (id SERIAL PRIMARY KEY, ts TIMESTAMPTZ); INSERT INTO debezium_heartbeat (id, ts) VALUES (1, NOW()) ON CONFLICT(id) DO UPDATE SET ts=NOW();", + "transforms": "unwrap,route", + "transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState", + "transforms.unwrap.drop.tombstones": "false", + "transforms.unwrap.delete.handling.mode": "none", + "transforms.route.type": "org.apache.kafka.connect.transforms.RegexRouter", + "transforms.route.regex": "([^.]+)\\.([^.]+)\\.([^.]+)", + "transforms.route.replacement": "public.kafka_$3", + "max.batch.size": "1024", + "max.queue.size": "4096", + "topic.creation.default.compression.type": "snappy", + "topic.creation.default.replication.factor": 1, + "topic.creation.default.partitions": 1 + } + } + \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/services/connect/source-referencedata.json b/deployment/tz-single-server-deployment/reporting/config/services/connect/source-referencedata.json new file mode 100644 index 0000000..3816a52 --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/connect/source-referencedata.json @@ -0,0 +1,33 @@ +{ + "name": "source-referencedata", + "config": { + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "tasks.max": "1", + "plugin.name": "wal2json", + "database.hostname": "olmis-db", + "database.port": "5432", + "database.user": "postgres", + "database.password": "p@ssw0rd", + "database.dbname": "open_lmis", + "database.server.name": "openlmis", + "table.whitelist": "referencedata\\.commodity_types,referencedata\\.geographic_levels,referencedata\\.geographic_zones,referencedata\\.facility_operators,referencedata\\.facility_types,referencedata\\.facilities,referencedata\\.ideal_stock_amounts,referencedata\\.lots,referencedata\\.orderable_identifiers,referencedata\\.orderables,referencedata\\.processing_periods,referencedata\\.processing_schedules,referencedata\\.program_orderables,referencedata\\.programs,referencedata\\.requisition_group_members,referencedata\\.requisition_group_program_schedules,referencedata\\.supported_programs,referencedata\\.trade_items,referencedata\\.rights,referencedata\\.users,referencedata\\.roles,referencedata\\.supervisory_nodes,referencedata\\.requisition_groups,referencedata\\.supply_lines,referencedata\\.right_assignments,referencedata\\.facility_type_approved_products", + "database.history.kafka.bootstrap.servers": "kafka:29092", + "database.history.kafka.topic": "openlmis", + "column.blacklist": "referencedata.facilities.location,referencedata.geographic_zones.boundary", + "slot.name": "dbz_referencedata", + "time.precision.mode": "connect", + "heartbeat.interval.ms": 3000, + "heartbeat.action.query": "CREATE TABLE IF NOT EXISTS debezium_heartbeat (id SERIAL PRIMARY KEY, ts TIMESTAMPTZ); INSERT INTO debezium_heartbeat (id, ts) VALUES (1, NOW()) ON CONFLICT(id) DO UPDATE SET ts=NOW();", + "transforms": "unwrap,route", + "transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState", + "transforms.unwrap.drop.tombstones": "false", + "transforms.unwrap.delete.handling.mode": "none", + "transforms.route.type": "org.apache.kafka.connect.transforms.RegexRouter", + "transforms.route.regex": "([^.]+)\\.([^.]+)\\.([^.]+)", + "transforms.route.replacement": "public.kafka_$3", + "topic.creation.default.compression.type": "lz4", + "topic.creation.default.replication.factor": 1, + "topic.creation.default.partitions": 1 + } + } + \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/services/connect/source-requisition.json b/deployment/tz-single-server-deployment/reporting/config/services/connect/source-requisition.json new file mode 100644 index 0000000..e4bf6ea --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/connect/source-requisition.json @@ -0,0 +1,33 @@ +{ + "name": "source-requisition", + "config": { + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "tasks.max": "1", + "plugin.name": "wal2json", + "database.hostname": "olmis-db", + "database.port": "5432", + "database.user": "postgres", + "database.password": "p@ssw0rd", + "database.dbname": "open_lmis", + "database.server.name": "openlmis", + "table.whitelist": "requisition\\.requisitions,requisition\\.requisition_line_items,requisition\\.status_changes,requisition\\.stock_adjustments,requisition\\.stock_adjustment_reasons", + "database.history.kafka.bootstrap.servers": "kafka:29092", + "database.history.kafka.topic": "openlmis", + "slot.name": "dbz_requisition", + "heartbeat.interval.ms": 30000, + "heartbeat.action.query": "CREATE TABLE IF NOT EXISTS debezium_heartbeat (id SERIAL PRIMARY KEY, ts TIMESTAMPTZ); INSERT INTO debezium_heartbeat (id, ts) VALUES (1, NOW()) ON CONFLICT(id) DO UPDATE SET ts=NOW();", + "transforms": "unwrap,route", + "transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState", + "transforms.unwrap.drop.tombstones": "false", + "transforms.unwrap.delete.handling.mode": "none", + "transforms.route.type": "org.apache.kafka.connect.transforms.RegexRouter", + "transforms.route.regex": "([^.]+)\\.([^.]+)\\.([^.]+)", + "transforms.route.replacement": "public.kafka_$3", + "max.batch.size": "1024", + "max.queue.size": "4096", + "topic.creation.default.compression.type": "snappy", + "topic.creation.default.replication.factor": 1, + "topic.creation.default.partitions": 1 + } + } + \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/services/kafka-ops/run.sh b/deployment/tz-single-server-deployment/reporting/config/services/kafka-ops/run.sh new file mode 100755 index 0000000..0dc0099 --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/kafka-ops/run.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e + +: "${KAFKA_BROKER:?ERROR: Env var KAFKA_BROKER not set}" + +julie-ops-cli.sh \ + --brokers "$KAFKA_BROKER" \ + --clientConfig ./topology-builder.properties \ + --topology topology \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/services/kafka-ops/topology-builder.properties b/deployment/tz-single-server-deployment/reporting/config/services/kafka-ops/topology-builder.properties new file mode 100644 index 0000000..5132857 --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/kafka-ops/topology-builder.properties @@ -0,0 +1,2 @@ +topology.topic.prefix.format=public.kafka_{{topic}} +topology.project.prefix.format=public \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/services/kafka-ops/topology/topology-equipment.yml b/deployment/tz-single-server-deployment/reporting/config/services/kafka-ops/topology/topology-equipment.yml new file mode 100644 index 0000000..81b9ad4 --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/kafka-ops/topology/topology-equipment.yml @@ -0,0 +1,80 @@ +context: "openlmis" +source: "kafka-ops" +projects: + - name: "equipment" + topics: + - name: "equipments" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "snappy" + - name: "contracts" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "snappy" + - name: "equipment_categories" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "snappy" + - name: "equipment_disciplines" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "snappy" + - name: "equipment_energy_types" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "snappy" + - name: "equipment_fund_sources" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "snappy" + - name: "equipment_inventory" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "snappy" + - name: "equipment_maintenance_request_reasons" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "snappy" + - name: "equipment_maintenance_requests" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "snappy" + - name: "equipment_models" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "snappy" + - name: "equipment_service_logs" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "snappy" + - name: "equipment_type_programs" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "snappy" + - name: "equipment_types" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "snappy" + - name: "service_types" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "snappy" + - name: "vendor" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "snappy" \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/services/kafka-ops/topology/topology-referencedata.yml b/deployment/tz-single-server-deployment/reporting/config/services/kafka-ops/topology/topology-referencedata.yml new file mode 100644 index 0000000..19d3ce3 --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/kafka-ops/topology/topology-referencedata.yml @@ -0,0 +1,130 @@ +context: "openlmis" +source: "kafka-ops" +projects: + - name: "referencedata" + topics: + - name: "commodity_types" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "lz4" + - name: "ideal_stock_amounts" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "lz4" + - name: "facilities" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "lz4" + - name: "facility_operators" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "lz4" + - name: "facility_types" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "lz4" + - name: "geographic_levels" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "lz4" + - name: "geographic_zones" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "lz4" + - name: "lots" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "lz4" + - name: "orderable_identifiers" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "snappy" + - name: "orderables" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "lz4" + - name: "processing_periods" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "lz4" + - name: "processing_schedules" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "lz4" + - name: "program_orderables" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "lz4" + - name: "programs" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "lz4" + - name: "requisition_group_members" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "lz4" + - name: "requisition_group_program_schedules" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "lz4" + - name: "requisition_groups" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "lz4" + - name: "right_assignments" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "lz4" + - name: "rights" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "lz4" + - name: "roles" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "lz4" + - name: "supervisory_nodes" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "lz4" + - name: "supply_lines" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "lz4" + - name: "supported_programs" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "lz4" + - name: "trade_items" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "lz4" + - name: "users" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "lz4" \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/services/kafka-ops/topology/topology-requisition.yml b/deployment/tz-single-server-deployment/reporting/config/services/kafka-ops/topology/topology-requisition.yml new file mode 100644 index 0000000..3b15a98 --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/kafka-ops/topology/topology-requisition.yml @@ -0,0 +1,30 @@ +context: "openlmis" +source: "kafka-ops" +projects: + - name: "requisition" + topics: + - name: "requisitions" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "snappy" + - name: "requisition_line_items" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "snappy" + - name: "status_changes" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "snappy" + - name: "stock_adjustments" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "snappy" + - name: "stock_adjustment_reasons" + config: + replication.factor: 1 + num.partitions: 1 + compression.type: "snappy" \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/services/nginx/consul-template/openlmis.conf b/deployment/tz-single-server-deployment/reporting/config/services/nginx/consul-template/openlmis.conf new file mode 100644 index 0000000..638677f --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/nginx/consul-template/openlmis.conf @@ -0,0 +1,95 @@ +{{ $loaded_services := services }} +{{ $resources := tree (env "RESOURCES_PATH") }} + +server_names_hash_bucket_size 128; + +{{ range $loaded_services }} +{{- if in .Tags (env "SERVICE_TAG") -}} +{{- $current_service := service .Name "any" -}} +{{- if not (eq (len $current_service) 0) }} +upstream {{ .Name }} { + least_conn; + keepalive 128; + {{ range $current_service }}server {{ .Address }}:{{ .Port }}; +{{ end }} +} +{{ end -}} +{{- end -}} +{{ end }} + +log_format upstream_time '$remote_addr - $remote_user [$time_local] ' + '"$request" $status $body_bytes_sent ' + '"$http_referer" "$http_user_agent" ' + '$request_time $upstream_connect_time ' + '$upstream_header_time $upstream_response_time ' + '$pipe'; + +{{ range $resources }} {{ $location := .Key }} {{ $locationData := .Value | parseJSON }} +server { + server_name {{ $location }}; + listen 80; + +{{ if $locationData.enable_ssl }} + return 301 https://$server_name$request_uri; +{{ else }} +{{ if (and ($locationData.behind_load_balancer) ($locationData.redirect_http_traffic)) }} + if ($http_X_Forwarded_Proto = "http") { + return 301 https://$server_name$request_uri; + } +{{ end }} + location / { + proxy_pass http://{{ $locationData.upstream }}; + {{ if not $locationData.behind_load_balancer }} + proxy_set_header X-ProxyScheme http; + proxy_set_header X-Forwarded-Proto http; + proxy_set_header X-ProxyPort 80; + {{ end }} + proxy_set_header X-ProxyHost {{ $location }}; + proxy_set_header Host {{ $location }}; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-ProxyContextPath /; +{{ if $locationData.enable_basic_auth }} + auth_basic "Authentication Required"; + auth_basic_user_file /etc/nginx/.htpasswd; +{{ end }} + } +{{ end }} +} + +{{ if $locationData.enable_ssl }} +server { + server_name {{ $location }}; + + location / { + proxy_pass http://{{ $locationData.upstream }}; + proxy_set_header X-ProxyScheme https; + proxy_set_header X-ProxyHost {{ $location }}; + proxy_set_header Host {{ $location }}; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-ProxyPort 443; + proxy_set_header X-ProxyContextPath /; +{{ if $locationData.enable_basic_auth }} + auth_basic "Authentication Required"; + auth_basic_user_file /etc/nginx/.htpasswd; +{{ end }} + } + + listen 443; + ssl on; + ssl_certificate {{ $locationData.ssl_cert }}; + ssl_certificate_key {{ $locationData.ssl_key }}; + ssl_trusted_certificate {{ $locationData.ssl_cert_chain }}; + ssl_protocols TLSv1.2; + ssl_ciphers "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256"; + ssl_dhparam /etc/ssl/certs/dhparam.pem; + ssl_prefer_server_ciphers on; + keepalive_timeout 70; + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 10m; + add_header Strict-Transport-Security 'max-age=15768000'; + ssl_stapling on; + ssl_session_cache builtin:1000 shared:SSL:10m; +} +{{ end }} +{{ end }} diff --git a/deployment/tz-single-server-deployment/reporting/config/services/nginx/init.sh b/deployment/tz-single-server-deployment/reporting/config/services/nginx/init.sh new file mode 100755 index 0000000..04ae882 --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/nginx/init.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +set -e + +openssl dhparam -dsaparam -out /etc/ssl/certs/dhparam.pem 4096 + +cp -r /config/nginx/consul-template/* /etc/consul-template/ + +echo -n "${NGINX_BASIC_AUTH_USER}:" >> /etc/nginx/.htpasswd +openssl passwd -apr1 ${NGINX_BASIC_AUTH_PW} >> /etc/nginx/.htpasswd + +echo "Waiting for consul to be available" +sleep 240; + +/home/run.sh \ No newline at end of file diff --git a/deployment/tz-single-server-deployment/reporting/config/services/nifi/conf/logback.xml b/deployment/tz-single-server-deployment/reporting/config/services/nifi/conf/logback.xml new file mode 100644 index 0000000..b79cc80 --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/nifi/conf/logback.xml @@ -0,0 +1,170 @@ + + + + + + true + + + + ${org.apache.nifi.bootstrap.config.log.dir}/nifi-app.log + + + ${org.apache.nifi.bootstrap.config.log.dir}/nifi-app_%d{yyyy-MM-dd_HH}.%i.log + 100MB + + 30 + + true + + %date %level [%thread] %logger{40} %msg%n + + + + + ${org.apache.nifi.bootstrap.config.log.dir}/nifi-user.log + + + ${org.apache.nifi.bootstrap.config.log.dir}/nifi-user_%d.log + + 30 + + + %date %level [%thread] %logger{40} %msg%n + + + + + ${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap.log + + + ${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap_%d.log + + 5 + + + %date %level [%thread] %logger{40} %msg%n + + + + + + %date %level [%thread] %logger{40} %msg%n + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/deployment/tz-single-server-deployment/reporting/config/services/nifi/libs/postgresql-42.2.3.jar b/deployment/tz-single-server-deployment/reporting/config/services/nifi/libs/postgresql-42.2.3.jar new file mode 100755 index 0000000..228fa9f Binary files /dev/null and b/deployment/tz-single-server-deployment/reporting/config/services/nifi/libs/postgresql-42.2.3.jar differ diff --git a/deployment/tz-single-server-deployment/reporting/config/services/nifi/scripts/download-toolkit.sh b/deployment/tz-single-server-deployment/reporting/config/services/nifi/scripts/download-toolkit.sh new file mode 100755 index 0000000..075d0f6 --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/nifi/scripts/download-toolkit.sh @@ -0,0 +1,56 @@ +#!/bin/bash +export NIFI_TOOLKIT_DOWNLOAD_MAX_RETRIES=3 +export NIFI_DOCKER_TMP_DIR=/tmp/nifi-docker-cache + +downloadNifiToolkit() { + local nifiVersion=$1 + local retryCount=$2 + local downloadDir=$NIFI_DOCKER_TMP_DIR + if ! [[ -w "$downloadDir" ]]; then + echo "$downloadDir is not writable. Setting the download dir to /tmp" + downloadDir=/tmp + fi + local archivePath=${downloadDir}/nifi-toolkit-${nifiVersion}.tar.gz + + if [ "$retryCount" -gt "$NIFI_TOOLKIT_DOWNLOAD_MAX_RETRIES" ]; then + echo "Failed to download NiFi Toolkit $NIFI_TOOLKIT_DOWNLOAD_MAX_RETRIES times. Exiting!" + return 2 + fi + + mkdir -p ${downloadDir} + if ! [[ -e $archivePath ]]; then + echo "Downloading the NiFi Toolkit archive (${retryCount})." + curl -f -o $archivePath http://archive.apache.org/dist/nifi/${nifiVersion}/nifi-toolkit-${nifiVersion}-bin.tar.gz + if [ $? -ne 0 ]; then + return 1 + fi + fi + + extractNifiToolkit "$nifiVersion" "$archivePath" "$retryCount" + + return $? +} + +extractNifiToolkit() { + local nifiVersion=$1 + local archivePath=$2 + local retryCount=$3 + local destDir=/tmp + echo "Extracting the NiFi Toolkit archive to $destDir/nifi-toolkit-$nifiVersion." + rm -rf $destDir/nifi-toolkit-$nifiVersion + tar -xf $archivePath -C $destDir + + if [ $? -ne 0 ]; then + rm $archivePath + echo "Could not extract NiFi Toolkit. Restarting the download." + retryCount=$((retryCount + 1)) + downloadNifiToolkit "$nifiVersion" "$retryCount" + + return $? + fi + + return 0 +} + +downloadNifiToolkit "$1" 0 +exit $? diff --git a/deployment/tz-single-server-deployment/reporting/config/services/nifi/scripts/hideUsernameWithinNifi.sh b/deployment/tz-single-server-deployment/reporting/config/services/nifi/scripts/hideUsernameWithinNifi.sh new file mode 100755 index 0000000..cae67b7 --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/nifi/scripts/hideUsernameWithinNifi.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +CSS_PATH="/opt/nifi/nifi-current/work/jetty/nifi-web-ui-$1.war/webapp/css" + +while [ ! -d $CSS_PATH ] ; +do + sleep 2 +done + +cp $CSS_PATH/nf-canvas-all.css /nf-canvas-all.css.backup +echo "#current-user-container{display:none}" >> $CSS_PATH/nf-canvas-all.css +echo "#login-link-container{display:none}" >> $CSS_PATH/nf-canvas-all.css + +echo "Username and sign-in menu have been hidden" diff --git a/deployment/tz-single-server-deployment/reporting/config/services/nifi/scripts/preload.sh b/deployment/tz-single-server-deployment/reporting/config/services/nifi/scripts/preload.sh new file mode 100755 index 0000000..e752799 --- /dev/null +++ b/deployment/tz-single-server-deployment/reporting/config/services/nifi/scripts/preload.sh @@ -0,0 +1,215 @@ +#!/bin/bash + +export NIFI_BASE_URL="http://nifi:8080" +export WORKING_DIR="/config/nifi/scripts" +export TEMPLATES_DIR="$WORKING_DIR/templates" +export NIFI_UP_RETRY_COUNT=240 + +: ${POSTGRES_PASSWORD:?"Need to set POSTGRES_PASSWORD"} +: ${AUTH_SERVER_CLIENT_ID:?"Need to set AUTH_SERVER_CLIENT_ID"} +: ${AUTH_SERVER_CLIENT_SECRET:?"Need to set AUTH_SERVER_CLIENT_SECRET"} +: ${TRUSTED_HOSTNAME:?"Need to set TRUSTED_HOSTNAME"} +: ${OL_ADMIN_USERNAME:?"Need to set OL_ADMIN_USERNAME"} +: ${OL_ADMIN_PASSWORD:?"Need to set OL_ADMIN_PASSWORD"} +: ${OL_BASE_URL:?"Need to set OL_BASE_URL"} + +main() { + if waitNifiAvailable ${NIFI_UP_RETRY_COUNT}; then + local subCommand=$1 + + if [ "$subCommand" == "init" ]; then + initialize "${@:2}" + else + return 1 + fi + else + return 1 + fi + + return $? +} + +waitNifiAvailable() { + echo "PRELOAD Waiting for NiFi to be available" + local maxTries=$1 + local retryCount=1 + + while ! curl -s -f "$NIFI_BASE_URL/nifi"; do + sleep 10 + retryCount=$((retryCount + 1)) + if [[ "$retryCount" -gt "$maxTries" ]]; then + echo "PRELOAD ERROR, too many retries waiting for NiFi to be available" + return 1 + fi + done + + return 0 +} + +initialize() { + uploadAndImportTemplates "$@" + restartFlows "$@" +} + +uploadAndImportTemplates() { + echo "PRELOAD Uploading and importing templates" + local returnCode=0 + local templateCount=0 + + for templateFile in ${TEMPLATES_DIR}/*.xml; do + local filename=$(basename ${templateFile}) + if [ -e "${templateFile}" ]; then + echo "PRELOAD Uploading template defined in ${templateFile}" + templateId=$(curl -s $NIFI_BASE_URL/nifi-api/process-groups/root/templates/upload -F template=@${templateFile} | xmlstarlet sel -t -v '/templateEntity/template/id') + #echo "PRELOAD templateId = ${templateId}" + + if [ ! "$templateId" == "" ]; then + echo "PRELOAD Creating process group by importing template defined in ${templateFile}" + local processGroupOffsetY=$((templateCount * 200 + 50)) + curl -s -X POST -H 'Content-Type: application/json' $NIFI_BASE_URL/nifi-api/process-groups/root/template-instance -d '{"templateId":"'"${templateId}"'","originX":400.0,"originY":"'"${processGroupOffsetY}"'"}' > /dev/null + templateCount=$((templateCount + 1)) + else + echo "PRELOAD ERROR Uploading template was not successful" + returnCode=2 + fi + fi + done + + return $returnCode +} + +restartFlows() { + echo "PRELOAD Starting Flows" + + curl -s -X GET $NIFI_BASE_URL/nifi-api/process-groups/root/process-groups | jq '.[]|keys[]' | while read key ; + do + processorGroupId=$(curl -s -X GET $NIFI_BASE_URL/nifi-api/process-groups/root/process-groups | jq -r ".[][$key].component.id") + # Get process group version number + processGroupVersionNumber=$(curl -s -X GET -H 'Content-Type: application/json' $NIFI_BASE_URL/nifi-api/process-groups/${processorGroupId}/variable-registry | jq -r '.processGroupRevision.version') + # Set the global variables + curl --trace-ascii /dev/stdout -X PUT -H 'Content-Type: application/json' -d '{"processGroupRevision":{"version":"'"${processGroupVersionNumber}"'"},"variableRegistry":{"variables":[{"variable":{"name":"baseUrl","value":"'$OL_BASE_URL'","processGroupId":"'"${processorGroupId}"'"},"canWrite":true},{"variable":{"name":"username","value":"'$OL_ADMIN_USERNAME'","processGroupId":"'"${processorGroupId}"'"},"canWrite":true},{"variable":{"name":"password","value":"'$OL_ADMIN_PASSWORD'","processGroupId":"'"${processorGroupId}"'"},"canWrite":true}],"processGroupId":"'"${processorGroupId}"'"}}' $NIFI_BASE_URL/nifi-api/process-groups/${processorGroupId}/variable-registry + + # Enter sensitive values + curl -s -X GET $NIFI_BASE_URL/nifi-api/flow/process-groups/${processorGroupId}/controller-services | jq '.controllerServices|keys[]' | while read key ; + do + controllerServiceId=$(curl -s -X GET $NIFI_BASE_URL/nifi-api/flow/process-groups/${processorGroupId}/controller-services | jq -r ".controllerServices[$key].component.id") + controllerServiceName=$(curl -s -X GET $NIFI_BASE_URL/nifi-api/flow/process-groups/${processorGroupId}/controller-services | jq -r ".controllerServices[$key].component.name") + + if [ "$controllerServiceName" == "DBCPConnectionPool" ]; + then + curl --trace-ascii /dev/stdout -X PUT -H 'Content-Type: application/json' -d '{"revision":{"clientId":"random", "version":"0"},"component":{"id":"'"${controllerServiceId}"'","properties":{"Password":"'"$SUPERSET_POSTGRES_PASSWORD"'"}}}' $NIFI_BASE_URL/nifi-api/controller-services/${controllerServiceId} + else + continue + fi + done + + # Enable connector service + curl -s -X GET $NIFI_BASE_URL/nifi-api/flow/process-groups/${processorGroupId}/controller-services | jq '.controllerServices|keys[]' | while read key ; + do + controllerServiceId=$(curl -s -X GET $NIFI_BASE_URL/nifi-api/flow/process-groups/${processorGroupId}/controller-services | jq -r ".controllerServices[$key].component.id") + curl --trace-ascii /dev/stdout -X PUT -H 'Content-Type: application/json' -d '{"revision":{"clientId":"random", "version":"1"},"component":{"id":"'"${controllerServiceId}"'","state":"ENABLED"}}' $NIFI_BASE_URL/nifi-api/controller-services/${controllerServiceId} + done + + # find invokehttp processors and update password + curl -s -X GET $NIFI_BASE_URL/nifi-api/process-groups/${processorGroupId}/process-groups | jq '.[]|keys[]' | while read key ; + do + searchKey=$(curl -s -X GET $NIFI_BASE_URL/nifi-api/process-groups/${processorGroupId}/process-groups | jq -r ".processGroups[$key].component.name") + if [ "$searchKey" == "Create Token" ] || [ "$searchKey" == "Create token" ]; + then + createTokenId=$(curl -s -X GET $NIFI_BASE_URL/nifi-api/process-groups/${processorGroupId}/process-groups | jq -r ".processGroups[$key].component.id") + curl -s -X GET $NIFI_BASE_URL/nifi-api/process-groups/${createTokenId}/processors | jq '.[]|keys[]' | while read key ; + do + processorName=$(curl -s -X GET $NIFI_BASE_URL/nifi-api/process-groups/${createTokenId}/processors | jq -r ".processors[$key].component.name") + if [ "$processorName" == "Get Access token" ] ; + then + invokeHttpId=$(curl -s -X GET $NIFI_BASE_URL/nifi-api/process-groups/${createTokenId}/processors | jq -r ".processors[$key].component.id") + versionNumber=$(curl -s -X GET $NIFI_BASE_URL/nifi-api/processors/${invokeHttpId} | jq -r ".revision.version") + curl --trace-ascii /dev/stdout -X PUT -H 'Content-Type: application/json' -d '{"revision":{"clientId":"randomId","version":"'"${versionNumber}"'"},"component":{"id":"'"${invokeHttpId}"'","config":{"properties":{"Basic Authentication Username":"'"$AUTH_SERVER_CLIENT_ID"'","Basic Authentication Password":"'"$AUTH_SERVER_CLIENT_SECRET"'"}}}}}' $NIFI_BASE_URL/nifi-api/processors/${invokeHttpId} + break + fi + done + elif [ "$searchKey" == "Get Measures" ]; + then + checkMeasureId=$(curl -s -X GET $NIFI_BASE_URL/nifi-api/process-groups/${processorGroupId}/process-groups | jq -r ".processGroups[$key].component.id") + curl -s -X GET $NIFI_BASE_URL/nifi-api/process-groups/${checkMeasureId}/processors | jq '.[]|keys[]' | while read key ; + do + processorName=$(curl -s -X GET $NIFI_BASE_URL/nifi-api/process-groups/${checkMeasureId}/processors | jq -r ".processors[$key].component.name") + if [ "$processorName" == "Invoke FHIR token" ] ; + then + invokeHttpId=$(curl -s -X GET $NIFI_BASE_URL/nifi-api/process-groups/${checkMeasureId}/processors | jq -r ".processors[$key].component.id") + versionNumber=$(curl -s -X GET $NIFI_BASE_URL/nifi-api/processors/${invokeHttpId} | jq -r ".revision.version") + curl --trace-ascii /dev/stdout -X PUT -H 'Content-Type: application/json' -d '{"revision":{"clientId":"randomId","version":"'"${versionNumber}"'"},"component":{"id":"'"${invokeHttpId}"'","config":{"properties":{"Basic Authentication Username":"'"$FHIR_ID"'","Basic Authentication Password":"'"$FHIR_PASSWORD"'"}}}}}' $NIFI_BASE_URL/nifi-api/processors/${invokeHttpId} + break + fi + done + elif [ "$searchKey" == "Check MeasureReports" ]; + then + checkMeasureId=$(curl -s -X GET $NIFI_BASE_URL/nifi-api/process-groups/${processorGroupId}/process-groups | jq -r ".processGroups[$key].component.id") + curl -s -X GET $NIFI_BASE_URL/nifi-api/process-groups/${checkMeasureId}/processors | jq '.[]|keys[]' | while read key ; + do + processorName=$(curl -s -X GET $NIFI_BASE_URL/nifi-api/process-groups/${checkMeasureId}/processors | jq -r ".processors[$key].component.name") + if [ "$processorName" == "Invoke FHIR token" ] ; + then + invokeHttpId=$(curl -s -X GET $NIFI_BASE_URL/nifi-api/process-groups/${checkMeasureId}/processors | jq -r ".processors[$key].component.id") + versionNumber=$(curl -s -X GET $NIFI_BASE_URL/nifi-api/processors/${invokeHttpId} | jq -r ".revision.version") + curl --trace-ascii /dev/stdout -X PUT -H 'Content-Type: application/json' -d '{"revision":{"clientId":"randomId","version":"'"${versionNumber}"'"},"component":{"id":"'"${invokeHttpId}"'","config":{"properties":{"Basic Authentication Username":"'"$FHIR_ID"'","Basic Authentication Password":"'"$FHIR_PASSWORD"'"}}}}}' $NIFI_BASE_URL/nifi-api/processors/${invokeHttpId} + break + fi + done + elif [ "$searchKey" == "Generate products and measure list" ]; + then + curl -s -X GET $NIFI_BASE_URL/nifi-api/flow/process-groups/${processorGroupId}/controller-services | jq '.controllerServices|keys[]' | while read key ; + do + controllerServiceId=$(curl -s -X GET $NIFI_BASE_URL/nifi-api/flow/process-groups/${processorGroupId}/controller-services | jq -r ".controllerServices[$key].component.id") + # Enable connector service + curl --trace-ascii /dev/stdout -X PUT -H 'Content-Type: application/json' -d '{"revision":{"clientId":"random", "version":"0"},"component":{"id":"'"${controllerServiceId}"'","state":"ENABLED"}}' $NIFI_BASE_URL/nifi-api/controller-services/${controllerServiceId} + done + + generateProductMeasuresId=$(curl -s -X GET $NIFI_BASE_URL/nifi-api/process-groups/${processorGroupId}/process-groups | jq -r ".processGroups[$key].component.id") + curl -s -X GET $NIFI_BASE_URL/nifi-api/process-groups/${generateProductMeasuresId}/process-groups | jq '.[]|keys[]' | while read key ; + do + searchKey=$(curl -s -X GET $NIFI_BASE_URL/nifi-api/process-groups/${generateProductMeasuresId}/process-groups | jq -r ".processGroups[$key].component.name") + if [ "$searchKey" == "Get products" ]; + then + getProductsId=$(curl -s -X GET $NIFI_BASE_URL/nifi-api/process-groups/${generateProductMeasuresId}/process-groups | jq -r ".processGroups[$key].component.id") + curl -s -X GET $NIFI_BASE_URL/nifi-api/process-groups/${getProductsId}/processors | jq '.[]|keys[]' | while read key ; + do + processorName=$(curl -s -X GET $NIFI_BASE_URL/nifi-api/process-groups/${getProductsId}/processors | jq -r ".processors[$key].component.name") + if [ "$processorName" == "Get access token" ] ; + then + getAccessTokenId=$(curl -s -X GET $NIFI_BASE_URL/nifi-api/process-groups/${getProductsId}/processors | jq -r ".processors[$key].component.id") + versionNumber=$(curl -s -X GET $NIFI_BASE_URL/nifi-api/processors/${getAccessTokenId} | jq -r ".revision.version") + curl --trace-ascii /dev/stdout -X PUT -H 'Content-Type: application/json' -d '{"revision":{"clientId":"randomId","version":"'"${versionNumber}"'"},"component":{"id":"'"${getAccessTokenId}"'","config":{"properties":{"Basic Authentication Username":"'"$AUTH_SERVER_CLIENT_ID"'","Basic Authentication Password":"'"$AUTH_SERVER_CLIENT_SECRET"'"}}}}}' $NIFI_BASE_URL/nifi-api/processors/${getAccessTokenId} + break + fi + done + fi + done + else + continue + fi + done + + # Restart flows + sleep 5 # necessary to ensure all changes have taken effect + + # Start all processor groups except 'materialized view' process group. + # Save its id for reference to start it after 3 mins when data has been loaded into the table + processorGroupName=$(curl -s -X GET $NIFI_BASE_URL/nifi-api/process-groups/root/process-groups | jq -r ".[][$key].component.name") + if [ "$processorGroupName" == "Materialized Views" ]; + then + echo ${processorGroupId} > tempFileforMatViewId.txt + else + curl -s -X PUT -H 'Content-Type: application/json' -d '{"id":"'"${processorGroupId}"'","state":"RUNNING"}' $NIFI_BASE_URL/nifi-api/flow/process-groups/${processorGroupId} + fi + done + + sleep 180 + materializedViewProcessorGroupId=$( +