docker-compose.yaml

version: '3.2'

services:
  elasticsearch:
    build:
      context: ./containers/elasticsearch
      args:
        ELK_VERSION: $ELK_VERSION
    env_file:
      - .env
    volumes:
      - type: bind
        source: ./containers/elasticsearch/config/elasticsearch.yml
        target: /usr/share/elasticsearch/config/elasticsearch.yml
        read_only: true
    expose:
      - "9200"
      - "9300"
    environment:
      ES_JAVA_OPTS: "-Xmx256m -Xms256m"
      ELASTIC_PASSWORD: changeme
      # Use single node discovery in order to disable production mode and avoid bootstrap checks.
      # see: https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html
      discovery.type: single-node

  logstash:
    build:
      context: ./containers/logstash
      args:
        ELK_VERSION: $ELK_VERSION
    env_file:
      - .env
    volumes:
      - type: bind
        source: ./containers/logstash/config/logstash.yml
        target: /usr/share/logstash/config/logstash.yml
        read_only: true
      - type: bind
        source: ./containers/logstash/pipeline/logstash.conf
        target: /usr/share/logstash/pipeline/logstash.conf
        read_only: true
    command: logstash -f /usr/share/logstash/pipeline/logstash.conf
    expose:
     - "5044"
     - "5000/tcp"
     - "5000/udp"
     - "9600"
    environment:
      LS_JAVA_OPTS: "-Xmx256m -Xms256m"
    depends_on:
      - elasticsearch
      - kafka

  filebeat:
    build:
      context: ./containers/filebeat
      args:
        ELK_VERSION: ${ELK_VERSION}
        DEFLECT_DNET: dnet1
    volumes:
      - type: bind
        source: ./containers/filebeat/config/filebeat.docker.yml
        target: /usr/share/filebeat/filebeat.yml
        read_only: true
      - type: bind
        source: /var/log/
        target: /var/log/
        read_only: false
      - type: bind
        source: /var/run/
        target: /var/run/
        read_only: false
      - type: bind
        source: /var/lib/docker/containers/
        target: /var/lib/docker/containers/
        read_only: true
    command: filebeat -e -strict.perms=false
    depends_on:
      - logstash
#      - kafka

  zookeeper:
    image: bitnami/zookeeper:3.6.3
    environment:
      ALLOW_ANONYMOUS_LOGIN: "yes"
      ZOO_TLS_CLIENT_AUTH: "none"
    expose:
      - "2181-2182"

  kafka:
    image: bitnami/kafka:2.8.0
#    build:
#      context: ./containers/kafka
    ports:
      - "9092:9092"
      - "29092:29092"
    environment:
      ALLOW_PLAINTEXT_LISTENER: "yes"
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,EXTERNAL:SSL
      KAFKA_LISTENERS: PLAINTEXT://:9092,EXTERNAL://:29092
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://${DOCKER_KAFKA_HOST}:9092, EXTERNAL://${KAFKA_HOST}:29092
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_BROKER_ID: 1
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "true"
      KAFKA_CFG_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM:
      KAFKA_CFG_ZOOKEEPER_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM:
      KAFKA_CFG_ZOOKEEPER_PROTOCOL: PLAINTEXT
      KAFKA_SECURITY_PROTOCOL: PLAINTEXT
      KAFKA_SECURITY_INTER_BROKER_PROTOCOL: PLAINTEXT
      KAFKA_CFG_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_CFG_SSL_KEYSTORE_LOCATION: /bitnami/kafka/config/certs/kafka.keystore.jks
      KAFKA_CFG_SSL_KEYSTORE_PASSWORD: ${KAFKA_KEYSTORE_PASSWORD}
      KAFKA_CFG_SSL_TRUSTSTORE_LOCATION: /bitnami/kafka/config/certs/kafka.truststore.jks
      KAFKA_CFG_SSL_TRUSTSTORE_PASSWORD: ${KAFKA_TRUSTSTORE_PASSWORD}

    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - ./conf/kafka_local:/bitnami/kafka/config/certs

    depends_on:
      - zookeeper

  postgres:
    build:
      context: ./containers/postgres
      dockerfile: Dockerfile
    env_file:
      - .env
    shm_size: 2g
    user: postgres
    environment:
      - TZ=GMT
      - PGTZ=GMT
      - POSTGRES_USER=${BASKERVILLE_POSTGRES_USER}
      - POSTGRES_PASSWORD=${BASKERVILLE_POSTGRES_PASSWORD}
    command: postgres -p ${BASKERVILLE_POSTGRES_PORT} -c log_destination='csvlog' -c logging_collector='on' -c log_filename='postgresql.log' -c log_statement='all' -c synchronous_commit=off
    expose:
      - "${BASKERVILLE_POSTGRES_PORT}"
    volumes:
      - postgres-data3:/var/lib/postgresql/data/

  grafana:
      build:
          context: ./containers/grafana
          dockerfile: Dockerfile
          args:
            - BASKERVILLE_DB=baskerville
      env_file:
        - .env
      user: "0"
      ports:
          - "3000:3000"
      depends_on:
        - postgres
      volumes:
        - volume-grafana3:/var/lib/grafana:Z

  redis:
    image: redis
    env_file:
      - .env
    expose:
      - "6379"
    volumes:
      - /tmp:/data

  baskerville_preprocessing:
    image: "${BASKERVILLE_IMAGE}"
    env_file:
      - .env
    environment:
      - DB_USER=${BASKERVILLE_POSTGRES_USER}
      - DB_PASSWORD=${BASKERVILLE_POSTGRES_PASSWORD}
      - DB_HOST=postgres
      - DB_PORT=${BASKERVILLE_POSTGRES_PORT}
      - KAFKA_HOST=${DOCKER_KAFKA_HOST}:9092
      - REDIS_PASSWORD=""
      - CLEARING_HOUSE_KAFKA=${CLEARING_HOUSE_KAFKA}
    command: python3 /usr/local/baskerville/src/baskerville/main.py -c /app/baskerville/conf/preprocessing.yaml preprocessing
    volumes:
      - type: bind
        source: ./conf
        target: /app/baskerville/conf
        read_only: true
      - type: bind
        source: ./clearing_house_connection
        target: /app/baskerville/clearing_house_connection
        read_only: true
    depends_on:
      - postgres
      - kafka
      - redis
      - filebeat
    expose:
      - "4040-4041"
      - "8081-8080"

  baskerville_postprocessing:
    image: "${BASKERVILLE_IMAGE}"
    env_file:
      - .env
    environment:
      - DB_USER=${BASKERVILLE_POSTGRES_USER}
      - DB_PASSWORD=${BASKERVILLE_POSTGRES_PASSWORD}
      - DB_HOST=postgres
      - DB_PORT=${BASKERVILLE_POSTGRES_PORT}
      - KAFKA_HOST=${DOCKER_KAFKA_HOST}:9092
      - REDIS_PASSWORD=""
      - CLEARING_HOUSE_KAFKA=${CLEARING_HOUSE_KAFKA}
    command: python3 /usr/local/baskerville/src/baskerville/main.py -c /app/baskerville/conf/postprocessing.yaml postprocessing
    volumes:
      - type: bind
        source: ./conf
        target: /app/baskerville/conf
        read_only: true
      - type: bind
        source: ./clearing_house_connection
        target: /app/baskerville/clearing_house_connection
        read_only: true
    depends_on:
      - postgres
      - kafka
      - redis
      - filebeat

  banjax-next:
    build:
      context: ./containers/banjax-next
      dockerfile: Dockerfile
    # this uses the nginx network namespace so it can set iptables rules
    # in the right place. the go app listens on port 8081
    network_mode: "service:nginx"
    user: root
    cap_add:
      - NET_ADMIN  # we use iptables for banning
    volumes:
      - type: bind
        source: /var/log/banjax-next/
        target: /var/log/banjax-next/
        read_only: false
    depends_on:
      - nginx

  nginx:
    build:
      context: ./containers/nginx
      dockerfile: Dockerfile
    ports:
      - "80:80"
    volumes:
      - type: bind
        source: /var/log/banjax-next/
        target: /var/log/banjax-next/
        read_only: false
      - type: bind
        source: /var/log/nginx/
        target: /var/log/nginx/
        read_only: false
    networks:
      - local

  test-origin:
    build:
      context: ./containers/test-origin
      dockerfile: Dockerfile
#    ports:
#      - "8080:8080"
    network_mode: "service:nginx"


volumes:
  postgres-data3:

  volume-grafana3:

networks:
  local:
    driver: bridge