open-ecpds/run/bin/ecpds/Linux-ecpds/docker-compose.yml at master · ecmwf/open-ecpds · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
#
# File docker-compose.yml
#
# Purpose: Docker compose for deploying the OpenECPDS infrastructure in the Linux environment
#
# In adition to the options configured in this file please also check the following:
#
# - etc/ecpds/database/mysqld.cnf
# - etc/ecpds/master/ecmwf.properties
# - etc/ecpds/monitor/ecmwf.properties
# - etc/ecpds/mover/ecmwf.properties
#
# (c) Copyright ECMWF 2019-2024 - Laurent Gougeon (syi@ecmwf.int)
#

services:
  # The OpenECPDS Database (MariaDB) enables persistent storage for configurations, metadata and history
  database:
    container_name: database
    hostname: ecpds-database
    network_mode: host
    image: ${ECPDS_REPOSITORY:-ecpds}/database:${TAG:-latest}
    restart: unless-stopped
    # Database interface/port defined in 'etc/ecpds/database/mysqld.cnf'
    volumes:
      - "${ECPDS_ROOT_PATH:-}/var/lib/ecpds/database:/var/lib/mysql"
      - "${ECPDS_ROOT_PATH:-}/etc/ecpds/database:/etc/mysql/conf.d"

  # The AI backend (Ollama) provides language model capabilities for OpenECPDS
  ai:
    profiles: ["ai"]   # <-- disabled unless profile is enabled
    container_name: ai
    hostname: ecpds-ai
    network_mode: host
    image: ${ECPDS_REPOSITORY:-ecpds}/ai:${TAG:-latest}
    restart: unless-stopped
    environment:
      # Optional: limit memory/threads if desired
      OLLAMA_MAX_MEMORY: 10g
    volumes:
      - "${ECPDS_ROOT_PATH:-}/var/lib/ecpds/ai:/var/lib/ai" # persistent models & metadata

  # The ECPDS Master-Server (backend) implements the business logic of the application
  master:
    container_name: master
    hostname: ecpds-master
    network_mode: host
    environment:
      # Setting initial, soft and maximum heap size
      INITIAL_MEMORY: 128M
      SOFT_MAX_MEMORY: 256M
      MAX_MEMORY: 1G
      # Exposed IP address used to bind the public services (ecpds, ftp)
      EXTERNAL_ADDRESS: 0.0.0.0
      # Dedicated to data traffic between the internal components (jmx, rmi)
      INTERNAL_ADDRESS: 127.0.0.1
      # Ports for the Master-Server services
      PORT_JMX: 2062 # Java Management Extensions (monitoring & management)
      PORT_CALLBACK: 2600 # Java Remote Method Invocation (communication between components)
      PORT_ECPDS: 2640 # Dedicated service for ecpds command
      PORT_FTP: 2021 # Virtual FTP Server
      # Database related parameters (database interface/port must be the same as in 'etc/ecpds/database/mysqld.cnf')
      DATABASE_ALIAS: sequential://localhost:3309/ecpds
      DATABASE_MIN_SPOOL_SIZE: 5
      DATABASE_MAX_SPOOL_SIZE: 50
      # Define rollover strategy for master.log
      LOG_ROLLOVER_SIZE_MAX: 100MB # Maximum size the log file is allowed to reach before being rolled over
      LOG_ROLLOVER_SIZE_KEEP: 20 # Maximum number of backup files to keep
      # Define rollover strategy for request.log, operator.log, event.log & splunk.log
      LOG_ROLLOVER_DATE_PATTERN: dd-MM-yyyy # Date pattern for renaming current log file after rollover
      LOG_ROLLOVER_DATE_KEEP: 10d # Keep backup files which are newer than the specified duration
      # Log levels for log4j2 (off, fatal, error, warn, info, debug, trace, all)
      LOG_LEVEL: debug
      # Interactions with various services (smtp, opsview)
      METADATA_MISSING_CONTACTS_MAIL:
      SMTP_HOST:
      SMTP_FROM:
      OPSVIEW_HOST:
      # Miscellaneous parameters
      RETRIEVAL_MAX_SPOOL_SIZE: 50 # Maximum number of parallel retrievals
      # Java Virtual Machine options
      JAVA_OPTS: -XX:+UseG1GC # The Garbage Collector
    image: ${ECPDS_REPOSITORY:-ecpds}/master:${TAG:-latest}
    restart: unless-stopped
    volumes:
      - "${ECPDS_ROOT_PATH:-}/var/log/ecpds/master:/var/log/ecpds/master"
      - "${ECPDS_ROOT_PATH:-}/var/tmp/ecpds/master:/var/tmp/ecpds/master"
      - "${ECPDS_ROOT_PATH:-}/var/lib/ecpds/master:/var/lib/ecpds/master"
      - "${ECPDS_ROOT_PATH:-}/etc/ecpds:/etc/ecpds"
    depends_on:
      - database

  # The Monitoring interface (frontend) allow monitoring, controlling and administrating the application
  monitor:
    container_name: monitor
    hostname: ecpds-monitor
    network_mode: host
    environment:
      # Setting initial, soft and maximum heap size
      INITIAL_MEMORY: 128M
      SOFT_MAX_MEMORY: 256M
      MAX_MEMORY: 1G
      # Exposed IP address used to bind the public services (http/s)
      EXTERNAL_ADDRESS: 0.0.0.0
      # Dedicated to data traffic between the internal components (jmx, rmi)
      INTERNAL_ADDRESS: 127.0.0.1
      # Ports for the Monitor-Server services
      PORT_JMX: 3062 # Java Management Extensions (monitoring & management)
      PORT_CALLBACK: 3600 # Java Remote Method Invocation (communication between components)
      PORT_HTTPS: 3443 # Web monitoring interface (https)
      # Connectivity to the Master-Server
      MASTER_ADDRESS: localhost
      PORT_MASTER: 2600
      # Define rollover strategy for monitor.log
      LOG_ROLLOVER_SIZE_MAX: 100MB # Maximum size the log file is allowed to reach before being rolled over
      LOG_ROLLOVER_SIZE_KEEP: 20 # Maximum number of backup files to keep
      # Log levels for log4j2 (off, fatal, error, warn, info, debug, trace, all)
      LOG_LEVEL: debug
      # Password to unlock the 'etc/ecpds/monitor/ecpds-monitor.pfx' keystore
      KEYSTORE_PASSWORD: s5w8g771
      # Personalisation of monitoring web interface
      MONITOR_NICKNAME: "PDS"
      MONITOR_TITLE: "Personal Data Store"
      MONITOR_COLOR: "#0079d3"
      # Miscellaneous parameters
      USE_CACHE: "no"
      MASTER_URL_LIST: "localhost:2600=Loopback export for testing"
      DEFAULT_SOURCE_HOST_ID:
      # Java Virtual Machine options
      JAVA_OPTS: -XX:+UseG1GC # The Garbage Collector
    image: ${ECPDS_REPOSITORY:-ecpds}/monitor:${TAG:-latest}
    restart: unless-stopped
    volumes:
      - "${ECPDS_ROOT_PATH:-}/var/log/ecpds/monitor:/var/log/ecpds/monitor"
      - "${ECPDS_ROOT_PATH:-}/var/tmp/ecpds/monitor:/var/tmp/ecpds/monitor"
      - "${ECPDS_ROOT_PATH:-}/etc/ecpds:/etc/ecpds"
    depends_on:
      - master

  # The Data-Mover (frontend) make it possible to store objects and to perform incoming and outgoing
  # data transfers
  mover:
    container_name: mover
    hostname: ecpds-mover
    network_mode: host
    environment:
      # Setting initial, soft and maximum heap size
      INITIAL_MEMORY: 128M
      SOFT_MAX_MEMORY: 256M
      MAX_MEMORY: 1G
      # Exposed IP address used to bind the public services (http/s, ftp, sftp, ecproxy)
      EXTERNAL_ADDRESS: 0.0.0.0
      # Dedicated to data traffic between the internal components (jmx, rmi)
      INTERNAL_ADDRESS: 127.0.0.1
      # Ports for the Monitor-Server services
      PORT_JMX: 4062 # Java Management Extensions (monitoring & management)
      PORT_CALLBACK: 4600 # Java Remote Method Invocation (communication between components)
      PORT_HTTP: 4080 # Web data-portal interface (http)
      PORT_HTTPS: 4443 # Web data-portal interface (https)
      PORT_MQTT: 4183 # MQTT interface (mqtt)
      PORT_MQTTS: 4883 # MQTT interface (mqtts)
      PORT_FTP: 4021 # Data-portal interface (ftp)
      PORT_SSH: 4022 # Data-portal interface (sftp/scp)
      PORT_ECPROXY: 4640 # Dedicated service for ecpds command
      # Connectivity to the Master-Server
      MASTER_ADDRESS: localhost
      PORT_MASTER: 2600
      # Define rollover strategy for mover.log
      LOG_ROLLOVER_SIZE_MAX: 100MB # Maximum size the log file is allowed to reach before being rolled over
      LOG_ROLLOVER_SIZE_KEEP: 20 # Maximum number of backup files to keep
      # Log levels for log4j2 (off, fatal, error, warn, info, debug, trace, all)
      LOG_LEVEL: debug
      # Password to unlock the 'etc/ecpds/mover/ecpds-mover.pfx' keystore
      KEYSTORE_PASSWORD: s5w8g771
      # Personalisation of monitoring web interface
      MOVER_TITLE: "Personal Data Store (PDS)"
      MOVER_COLOR: "#0079d3"
      # Miscellaneous parameters
      OPERATION: "yes" # Set to 'no' to produce random data for missing files (for testing purpose only)
      # Java Virtual Machine options
      JAVA_OPTS: -XX:+UseG1GC # The Garbage Collector
    image: ${ECPDS_REPOSITORY:-ecpds}/mover:${TAG:-latest}
    restart: unless-stopped
    volumes:
      - "${ECPDS_ROOT_PATH:-}/var/log/ecpds/mover:/var/log/ecpds/mover"
      - "${ECPDS_ROOT_PATH:-}/var/tmp/ecpds/mover:/var/tmp/ecpds/mover"
      - "${ECPDS_ROOT_PATH:-}/var/lib/ecpds/mover:/var/lib/ecpds/mover"
      - "${ECPDS_ROOT_PATH:-}/etc/ecpds:/etc/ecpds"
    depends_on:
      - master