Skip to content

Commit 6c60b2f

Browse files
mkaranasoumkaranasou
committed
single multistage Dockerfile for dashboard
1 parent a86256a commit 6c60b2f

10 files changed

Lines changed: 213 additions & 146 deletions

File tree

‎README.md‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,4 +130,28 @@ docker-compose exec kafka bash
130130

131131
# the following will consume / display a few messages, just to make sure all is well
132132
/opt/bitnami/kafka/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic deflect.logs --offset 6131 --partition 0
133-
```
133+
```
134+
135+
### Dashboard
136+
Notes for the Baskerville dashboard:
137+
- The Dockerfile is heavy, as it is a multistage Dockerfile. It uses Nginx internally but, since we already have an nginx service
138+
it would be nice to have a common volume for the front-end to be served and proper networking for the backend to be served also (only for the web-sockets)
139+
- It has Baskerville as a dependency (with all that this entails, like esretriever, iforest pyspark etc, which means different pyspark versions with conflicts and a lot of build time)
140+
141+
142+
143+
### Misc
144+
In case baskerville_preprocessing and baskerville_postprocessing fail to start because `baskerville` database does not exist:
145+
```bash
146+
docker-compose exec postgres bash
147+
psql
148+
CREATE DATABASE baskerville;
149+
\q
150+
exit
151+
152+
docker-compose restart baskerville_preprocessing baskerville_postprocessing
153+
```
154+
155+
### Firewall
156+
- open 29092 port for Kafka connections
157+

‎containers/baskerville_dashboard/Dockerfile‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
# GLOBAL ARGS:
2+
ARG DOCKER_KAFKA_HOST
3+
ARG DASHBOARD_BRANCH
4+
# for front-end
5+
ARG API_BASE_URL
6+
ARG SOCKET_URL
7+
# for backend:
8+
ARG BASKERVILLE_BRANCH
9+
ARG REDIS_HOST
10+
11+
#FROM nginx:1.17.1-alpine AS NGINX
12+
FROM node:14.8.0-alpine AS FRONTEND
13+
ARG DASHBOARD_BRANCH
14+
ARG API_BASE_URL
15+
ARG SOCKET_URL
16+
17+
ENV DASHBOARD_BRANCH $DASHBOARD_BRANCH
18+
ENV API_BASE_URL $API_BASE_URL
19+
ENV SOCKET_URL $SOCKET_URL
20+
21+
RUN apk update \
22+
# && apk upgrade \
23+
&& apk add git \
24+
&& mkdir /app && cd /app \
25+
&& git clone --branch $DASHBOARD_BRANCH https://github.com/deflect-ca/baskerville_dashboard.git \
26+
&& cd baskerville_dashboard/front-end \
27+
&& npm install \
28+
&& npm install -g @angular/cli@11.1.0
29+
30+
RUN echo $SOCKET_URL, $API_BASE_URL && echo "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<"
31+
32+
# this builds the front-end with provided configuration and copies the result in /var/www for nginx
33+
RUN cd /app/baskerville_dashboard/front-end \
34+
&& npm run config \
35+
&& ng build --prod
36+
37+
FROM openjdk:8 AS OJDK8
38+
FROM python:3.6 AS BACKEND
39+
40+
ARG DOCKER_KAFKA_HOST
41+
ARG BASKERVILLE_BRANCH
42+
ARG DASHBOARD_BRANCH
43+
ARG REDIS_HOST
44+
45+
ENV DOCKER_KAFKA_HOST $DOCKER_KAFKA_HOST
46+
ENV DASHBOARD_BRANCH $DASHBOARD_BRANCH
47+
ENV BASKERVILLE_BRANCH $BASKERVILLE_BRANCH
48+
ENV REDIS_HOST $REDIS_HOST
49+
ENV BASKERVILLE_ROOT '/app/baskerville'
50+
ENV BASKERVILLE_DASH_ROOT '/app/baskerville_dashboard'
51+
52+
53+
# Get jdk8 from previous stage https://docs.docker.com/develop/develop-images/multistage-build/
54+
COPY --from=OJDK8 /usr/local/openjdk-8 /usr/local/openjdk-8
55+
COPY --from=FRONTEND /app/baskerville_dashboard/front-end/dist/baskerville_dashboard_frontend/ /var/www/baskerville_dashboard_frontend/
56+
57+
# Set java path
58+
ENV JAVA_HOME /usr/local/openjdk-8
59+
ENV PATH $PATH:$JAVA_HOME/bin
60+
61+
# TODO: The following installs spark-iforest, esretriever and baskerville
62+
# which takes a long time - mostly because of the different pyspark versions.
63+
RUN apt-get clean && apt-get update \
64+
# && apt-get -y upgrade \
65+
&& apt-get install -y nginx=1.18.* \
66+
&& apt-get install git \
67+
&& pip install --upgrade pip \
68+
&& mkdir /app && cd /app \
69+
&& git clone https://github.com/titicaca/spark-iforest.git \
70+
&& cd spark-iforest/python \
71+
&& pip install . \
72+
&& cd /app \
73+
&& git clone https://github.com/equalitie/esretriever.git \
74+
&& cd esretriever \
75+
&& pip install . \
76+
&& cd /app \
77+
&& git clone --branch $BASKERVILLE_BRANCH https://github.com/deflect-ca/baskerville.git \
78+
&& cd baskerville \
79+
&& pip install . \
80+
&& cd /app \
81+
&& git clone --branch $DASHBOARD_BRANCH https://github.com/deflect-ca/baskerville_dashboard.git \
82+
&& cd baskerville_dashboard/backend \
83+
&& pip install . \
84+
&& cd /app/baskerville_dashboard/backend/src/baskerville_dashboard
85+
86+
# copy all related configurations
87+
COPY ./nginx.conf /etc/nginx/nginx.conf
88+
COPY ./config.yaml /app/baskerville_dashboard/backend/conf
89+
COPY ./baskerville.yaml /app/baskerville_dashboard/backend/conf
90+
91+
WORKDIR /app/baskerville_dashboard/backend/src/baskerville_dashboard
92+
93+
# socketio for Python includes a production grade web server.
94+
# start.sh starts nginx and runs dashboard backend.
95+
COPY start.sh start.sh
96+
CMD ["sh", "./start.sh"]
97+
EXPOSE 80 81 5000

‎containers/baskerville_dashboard/backend/Dockerfile‎

Lines changed: 0 additions & 51 deletions
This file was deleted.

containers/baskerville_dashboard/backend/baskerville.yaml renamed to containers/baskerville_dashboard/baskerville.yaml

Lines changed: 4 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
database:
33
name: baskerville
44
user: !ENV ${DB_USER:postgres}
5-
host: !ENV ${DB_HOST:postgres}
5+
host: !ENV ${DB_HOST:127.0.0.1}
66
port: !ENV ${DB_PORT:5432}
77
password: !ENV ${DB_PASS:secret}
88
type: 'postgres'
@@ -91,7 +91,7 @@ engine:
9191

9292
kafka:
9393
connection:
94-
bootstrap_servers: !ENV '${KAFKA_HOST:kafka}'
94+
bootstrap_servers: !ENV '${DOCKER_KAFKA_HOST:0.0.0.0}'
9595
# bootstrap_servers: !ENV ${KAFKA_HOST}
9696
auto_offset_reset: 'smallest'
9797
data_topic: 'feedback'
@@ -111,10 +111,10 @@ spark:
111111
parallelism: -1
112112
log_conf: 'true'
113113
log_level: 'ERROR'
114-
redis_host: !ENV ${REDIS_HOST:localhost}
114+
redis_host: !ENV ${REDIS_HOST:redis}
115115
redis_password: !ENV ${REDIS_PASSWORD:""}
116116
redis_port: '6379'
117-
jars: !ENV '${BASKERVILLE_ROOT}/data/jars/spark-redis_2.11-2.5.0-SNAPSHOT-jar-with-dependencies.jar,${BASKERVILLE_ROOT}/data/jars/spark-iforest-3.0.1.jar,${BASKERVILLE_ROOT}/data/jars/postgresql-42.2.4.jar,${BASKERVILLE_ROOT}/data/jars/spark-streaming-kafka-0-8-assembly_2.11-2.4.0.jar'
117+
jars: !ENV '${BASKERVILLE_ROOT}/data/jars/spark-redis_2.11-2.5.0-SNAPSHOT-jar-with-dependencies.jar,${BASKERVILLE_ROOT}/data/jars/spark-iforest-2.4.0.99.jar,${BASKERVILLE_ROOT}/data/jars/postgresql-42.2.4.jar,${BASKERVILLE_ROOT}/data/jars/spark-streaming-kafka-0-8-assembly_2.11-2.4.0.jar'
118118
session_timezone: 'UTC'
119119
shuffle_partitions: 12
120120
spark_driver_memory: '8G'
@@ -135,20 +135,3 @@ spark:
135135
driver_java_options: '-Dio.netty.noPreferDirect=true -Dio.netty.allocator.type=unpooled -XX:+UseCompressedOops -XX:G1HeapRegionSize=10 -XX:+UseG1GC -XX:ParallelGCThreads=8 -XX:ConcGCThreads=2 -XX:InitiatingHeapOccupancyPercent=25 -XX:+UnlockDiagnosticVMOptions -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=1098'
136136
executor_extra_java_options: '-Dio.netty.noPreferDirect=true -Dio.netty.allocator.type=unpooled -XX:+UseCompressedOops -XX:G1HeapRegionSize=10 -XX:+UseG1GC -XX:ParallelGCThreads=8 -XX:ConcGCThreads=2 -XX:InitiatingHeapOccupancyPercent=25 -XX:+PrintFlagsFinal -XX:+PrintReferenceGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+UnlockDiagnosticVMOptions -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=1098'
137137

138-
139-
#spark:
140-
# ssl_enabled: True
141-
# ssl_truststore: '/root/keys/truststore/kafka.truststore.jks'
142-
# ssl_truststore_password: 'B1^ZRUUVoIuKND7t2HiJ8fwRg0kdMo4zdh8m8eRzgXw!'
143-
# ssl_keystore: '/root/keys/keystore_client/kafka.keystore.jks'
144-
# ssl_keystore_password: 'B1^ZRUUVoIuKND7t2HiJ8fwRg0kdMo4zdh8m8eRzgXw!'
145-
# ssl_keypassword: 'B1^ZRUUVoIuKND7t2HiJ8fwRg0kdMo4zdh8m8eRzgXw!'
146-
# auth_secret: 'B1^ZRUUVoIuKND7t2HiJ8fwRg0kdMo4zdh8m8eRzgXw!'
147-
# redis_host: 'bnode1.deflect.ca'
148-
# redis_password: 'B1^ZRUUVoIuKND7t2HiJ8fwRg0kdMo4zdh8m8eRzgXw!'
149-
# app_name: 'Postprocessing'
150-
# master: !ENV 'spark://${SPARK_MASTER_HOST}:7077'
151-
# parallelism: -1
152-
# log_conf: 'true'
153-
# log_level: 'ERROR'
154-

containers/baskerville_dashboard/backend/config.yaml renamed to containers/baskerville_dashboard/config.yaml

File renamed without changes.

‎containers/baskerville_dashboard/front-end/Dockerfile‎

Lines changed: 0 additions & 34 deletions
This file was deleted.

‎containers/baskerville_dashboard/front-end/nginx.conf‎

Lines changed: 0 additions & 26 deletions
This file was deleted.

‎containers/baskerville_dashboard/nginx.conf‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
events{}
2+
http {
3+
include /etc/nginx/mime.types;
4+
server {
5+
listen 80;
6+
server_name _;
7+
server_name localhost;
8+
9+
location / {
10+
root /var/www/baskerville_dashboard_frontend/;
11+
index index.html;
12+
}
13+
location /api {
14+
rewrite ^/api/(.*) /$1 break;
15+
proxy_pass http://127.0.0.1:5000;
16+
proxy_set_header Host $host;
17+
proxy_set_header X-Real-IP ip_address;
18+
}
19+
20+
location /socket.io {
21+
include proxy_params;
22+
proxy_http_version 1.1;
23+
proxy_buffering off;
24+
proxy_set_header Upgrade $http_upgrade;
25+
proxy_set_header Connection "Upgrade";
26+
proxy_pass http://127.0.0.1:5000/socket.io;
27+
}
28+
# location /api {
29+
# rewrite ^/api/(.*) /$1 break;
30+
# proxy_pass http://127.0.0.0:5000;
31+
# proxy_set_header Host $host;
32+
# proxy_set_header X-Real-IP ip_address;
33+
# }
34+
# location ~* \.io {
35+
# proxy_set_header X-Real-IP $remote_addr;
36+
# proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
37+
# proxy_set_header Host $http_host;
38+
# proxy_set_header X-NginX-Proxy false;
39+
#
40+
# proxy_pass http://127.0.0.0:5000;
41+
# proxy_redirect off;
42+
#
43+
# proxy_http_version 1.1;
44+
# proxy_set_header Upgrade $http_upgrade;
45+
# proxy_set_header Connection "upgrade";
46+
# }
47+
48+
}
49+
50+
# server {
51+
# listen 81;
52+
# server_name "dashboard_backend";
53+
# # server_name $hostname;
54+
# location /api {
55+
# rewrite ^/api/(.*) /$1 break;
56+
# proxy_pass http://localhost:5000;
57+
# proxy_set_header Host $host;
58+
# proxy_set_header X-Real-IP ip_address;
59+
# }
60+
# location ~* \.io {
61+
# proxy_set_header X-Real-IP $remote_addr;
62+
# proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
63+
# proxy_set_header Host $http_host;
64+
# proxy_set_header X-NginX-Proxy false;
65+
#
66+
# proxy_pass http://localhost:5000;
67+
# proxy_redirect off;
68+
#
69+
# proxy_http_version 1.1;
70+
# proxy_set_header Upgrade $http_upgrade;
71+
# proxy_set_header Connection "upgrade";
72+
# }
73+
#
74+
# }
75+
# include /etc/nginx/sites.d/*.conf;
76+
}

‎containers/baskerville_dashboard/start.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
nginx && python app.py

0 commit comments

Comments
 (0)