Skip to content

Commit 8756c31

Browse files
JohTJohT
committed
Modularize Neo4j configuration and support profile changes
1 parent 51240f0 commit 8756c31

9 files changed

Lines changed: 257 additions & 97 deletions

‎init.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ mkdir -p "./${SOURCE_DIRECTORY}"
7979
createForwardingScript "./../../scripts/analysis/analyze.sh"
8080
createForwardingScript "./../../scripts/startNeo4j.sh"
8181
createForwardingScript "./../../scripts/stopNeo4j.sh"
82+
createForwardingScript "./../../scripts/useNeo4jHighMemoryProfile.sh"
8283

8384
source "${SCRIPTS_DIR}/scripts/checkCompatibility.sh"
8485

‎scripts/configuration/template-neo4j-high-memory.conf‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
2-
# The following static configuration entries were taken from "template-neo4j.conf".
3-
41
# Anonymous usage data reporting
52
dbms.usage_report.enabled=false
63

‎scripts/configuration/template-neo4j-low-memory.conf‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
2-
# The following static configuration entries were taken from "template-neo4j.conf".
3-
41
# Anonymous usage data reporting
52
dbms.usage_report.enabled=false
63

‎scripts/configuration/template-neo4j-v4-low-memory.conf‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
2-
# The following static configuration entries were taken from "template-neo4j.conf".
3-
41
# List of procedures and user defined functions that are allowed
52
# full access to the database through unsupported/insecure internal APIs.
63
dbms.security.procedures.unrestricted=apoc.*,gds.*

‎scripts/configuration/template-neo4j-v4.conf‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
2-
# The following static configuration entries were taken from "template-neo4j.conf".
3-
41
# List of procedures and user defined functions that are allowed
52
# full access to the database through unsupported/insecure internal APIs.
63
dbms.security.procedures.unrestricted=apoc.*,gds.*

‎scripts/configuration/template-neo4j.conf‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
2-
# The following static configuration entries were taken from "template-neo4j.conf".
3-
41
# Anonymous usage data reporting
52
dbms.usage_report.enabled=false
63

‎scripts/configureNeo4j.sh‎

Lines changed: 227 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,227 @@
1+
#!/usr/bin/env bash
2+
3+
# Configures a (local) Neo4j Community Edition Graph Database (https://neo4j.com/download-center/#community).
4+
#
5+
# Main input is the environment variable NEO4J_CONFIG_TEMPLATE:
6+
# - The name of the template file ("scripts/configuration" folder) for the Neo4j configuration.
7+
# - Defaults to "template-neo4j.conf".
8+
# - The template configuration will be appended to the end of the Neo4j configuration file.
9+
#
10+
# Prerequisites:
11+
# - Intended to be sourced from within another script like the setupNeo4j.sh script
12+
# - Neo4j has been installed
13+
# - The tools directory has been created
14+
#
15+
# Example Usage:
16+
# - NEO4J_CONFIG_TEMPLATE=template-neo4j-high-memory.conf ./../../scripts/configureNeo4j.sh
17+
#
18+
# When run for the first time, the original configuration will be backed up and all configuration entries will be set.
19+
# When run again (re-configuration), the template configuration at the end if the script is executed again.
20+
#
21+
# Requires operatingSystemFunctions.sh
22+
23+
# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
24+
set -o errexit -o pipefail
25+
26+
NEO4J_EDITION=${NEO4J_EDITION:-"community"} # Choose "community" or "enterprise"
27+
NEO4J_VERSION=${NEO4J_VERSION:-"2026.01.4"}
28+
29+
DATA_DIRECTORY=${DATA_DIRECTORY:-"$( pwd -P )/data"} # Path where Neo4j writes its data to (outside tools dir)
30+
RUNTIME_DIRECTORY=${RUNTIME_DIRECTORY:-"$( pwd -P )/runtime"} # Path where Neo4j puts runtime data to (e.g. logs) (outside tools dir)
31+
TOOLS_DIRECTORY=${TOOLS_DIRECTORY:-"tools"} # Get the tools directory (defaults to "tools")
32+
IMPORT_DIRECTORY=${IMPORT_DIRECTORY:-"$( pwd -P )/import"} # The name of the directory that is used to import data (e.g. CSV) files. Defaults to "import".
33+
SHARED_DOWNLOADS_DIRECTORY="${SHARED_DOWNLOADS_DIRECTORY:-$(dirname "$( pwd )")/downloads}"
34+
35+
NEO4J_HTTP_PORT=${NEO4J_HTTP_PORT:-"7474"} # Neo4j HTTP API port for executing queries
36+
NEO4J_HTTPS_PORT=${NEO4J_HTTPS_PORT:-"7473"} # Neo4j HTTPS port for encrypted querying
37+
NEO4J_BOLT_PORT=${NEO4J_BOLT_PORT:-"7687"} # Neo4j's own "Bolt Protocol" port
38+
39+
NEO4J_CONFIG_TEMPLATE=${NEO4J_CONFIG_TEMPLATE:-"template-neo4j.conf"} # Name of the template file ("configuration" folder) for the Neo4j configuration. Defaults to "template-neo4j.conf".
40+
41+
## Get this "scripts" directory if not already set
42+
# Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution.
43+
# CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.
44+
# This way non-standard tools like readlink aren't needed.
45+
SCRIPTS_DIR=${SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} # Repository directory containing the shell scripts
46+
SCRIPT_NAME="configureNeo4j"
47+
#echo "${SCRIPT_NAME}: SCRIPTS_DIR=${SCRIPTS_DIR}"
48+
49+
# Internal constants
50+
NEO4J_INSTALLATION_NAME="neo4j-${NEO4J_EDITION}-${NEO4J_VERSION}"
51+
NEO4J_INSTALLATION_DIRECTORY="${TOOLS_DIRECTORY}/${NEO4J_INSTALLATION_NAME}"
52+
NEO4J_CONFIG="${NEO4J_INSTALLATION_DIRECTORY}/conf/neo4j.conf"
53+
NEO4J_MAJOR_VERSION_NUMBER=$(echo "${NEO4J_VERSION}" | cut -d'.' -f1) # First part of the version number (=major version number)
54+
NEO4J_CONFIG_TEMPLATE_PATH="${SCRIPTS_DIR}/configuration/${NEO4J_CONFIG_TEMPLATE}"
55+
56+
# Include operation system functions like "convertPosixToWindowsPathIfNecessary".
57+
source "${SCRIPTS_DIR}/operatingSystemFunctions.sh"
58+
59+
fail() {
60+
local ERROR_COLOR='\033[0;31m' # red
61+
local DEFAULT_COLOR='\033[0m'
62+
local errorMessage="${1}"
63+
echo -e "${ERROR_COLOR}${SCRIPT_NAME}: Error: ${errorMessage}${DEFAULT_COLOR}" >&2
64+
exit 1
65+
}
66+
67+
insert_hint_that_the_configuration_is_script_managed() {
68+
local file="$1"
69+
local tempFile
70+
tempFile="$(mktemp)"
71+
72+
awk '
73+
BEGIN {
74+
separator_count = 0
75+
separator_pattern = "^#\\*{11,}$"
76+
}
77+
{
78+
if ($0 ~ separator_pattern) {
79+
separator_count++
80+
if (separator_count == 2) {
81+
print "#"
82+
print "# This file had been configured with the configureNeo4j.sh script of code-graph-analysis-pipeline."
83+
print "# Manual changes to this file might be overwritten when the script is executed again since the script is designed to update the configuration according to the template configuration."
84+
print "# Please check the documentation of the configureNeo4j.sh script for more details."
85+
}
86+
}
87+
88+
print
89+
}
90+
' "${file}" > "${tempFile}" && mv "${tempFile}" "$file"
91+
}
92+
93+
append_configuration_from_template() {
94+
echo "${SCRIPT_NAME}: Appending configuration template ${NEO4J_CONFIG_TEMPLATE} (memory, procedure permissions, ...)"
95+
local EYE_CATCHER_COMMENT_PREFIX="# The following configuration entries were taken from"
96+
97+
# Check if a template configuration had already been appended by looking for the eye-catcher comment.
98+
# If it is already there, then delete the old template configuration including the eye-catcher and append the new one again to update it.
99+
# This way this function is idempotent, the configuration is always up to date with the template and there are no duplicate entries.
100+
if grep -q "^${EYE_CATCHER_COMMENT_PREFIX}" "${NEO4J_CONFIG}"; then
101+
echo "${SCRIPT_NAME}: Deleting old configuration that had been appended from the template before since the eye-catcher comment was found. This ensures that there are no duplicate entries and that the configuration is up to date with the template."
102+
sed -i.edit.backup "/^${EYE_CATCHER_COMMENT_PREFIX}/,/^# End of configuration from ${NEO4J_CONFIG_TEMPLATE}/d" "${NEO4J_CONFIG}"
103+
if grep -q '^$' "${NEO4J_CONFIG}"; then
104+
sed -i.edit.backup '$d' "${NEO4J_CONFIG}" # Delete the last line of the config if it is an empty line
105+
fi
106+
rm -f "${NEO4J_CONFIG}.edit.backup" # The backup is created even if not needed to be compatible with both GNU sed and BSD sed (e.g. on
107+
fi
108+
109+
# Append first an eye-catcher comment to the configuration file to indicate that the configuration is script-managed
110+
# and that manual changes might be overwritten.
111+
{
112+
echo ""
113+
echo "${EYE_CATCHER_COMMENT_PREFIX} '${NEO4J_CONFIG_TEMPLATE}' by the script ${SCRIPT_NAME}."
114+
echo "# WARNING: Manual changes below this line might be overwritten when the script is executed again."
115+
echo ""
116+
} >> "${NEO4J_CONFIG}"
117+
118+
cat "${NEO4J_CONFIG_TEMPLATE_PATH}" >> "${NEO4J_CONFIG}"
119+
}
120+
121+
if [ -z "${TOOLS_DIRECTORY}" ]; then
122+
fail "Requires variable TOOLS_DIRECTORY to be set. If it is the current directory, then use a dot to reflect that."
123+
fi
124+
if [ ! -d "${TOOLS_DIRECTORY}" ]; then
125+
fail "Tools directory <${TOOLS_DIRECTORY}> does not exist. Please install Neo4j first by running the setupNeo4j.sh script which will create the tools directory if necessary."
126+
fi
127+
128+
# Check if SHARED_DOWNLOADS_DIRECTORY variable is set
129+
if [ -z "${SHARED_DOWNLOADS_DIRECTORY}" ]; then
130+
fail "Requires variable SHARED_DOWNLOADS_DIRECTORY to be set. If it is the current directory, then use a dot to reflect that."
131+
fi
132+
if [ ! -d "${SHARED_DOWNLOADS_DIRECTORY}" ]; then
133+
fail "Shared downloads directory <${SHARED_DOWNLOADS_DIRECTORY}> does not exist. Please install Neo4j first by running the setupNeo4j.sh script which will create the shared downloads directory if necessary."
134+
fi
135+
136+
# Fail if Neo4j hadn't been downloaded yet
137+
if [ ! -d "${NEO4J_INSTALLATION_DIRECTORY}" ] ; then
138+
fail "${NEO4J_INSTALLATION_NAME} is not installed into ${TOOLS_DIRECTORY} yet. Please run the setupNeo4j.sh script first to download and install Neo4j."
139+
fi
140+
141+
if [ ! -f "${NEO4J_CONFIG_TEMPLATE_PATH}" ]; then
142+
fail "Configuration template file ${NEO4J_CONFIG_TEMPLATE} does not exist in the scripts/configuration folder. Please make sure it exists and is correctly named:\n${NEO4J_CONFIG_TEMPLATE_PATH}"
143+
fi
144+
145+
# Create a backup of the original configuration file before any modification in case there is none yet.
146+
if [ ! -f "${NEO4J_CONFIG}.original.backup" ]; then
147+
cp "${NEO4J_CONFIG}" "${NEO4J_CONFIG}.original.backup"
148+
echo "${SCRIPT_NAME}: First time configuration. Created backup of the original configuration file at ${NEO4J_CONFIG}.original.backup"
149+
else
150+
echo "${SCRIPT_NAME}: Re-Configuration because ${NEO4J_CONFIG}.original.backup already exists."
151+
append_configuration_from_template
152+
echo "${SCRIPT_NAME}: Configuration template replaced successfully."
153+
return 0
154+
fi
155+
156+
echo "${SCRIPT_NAME}: Commenting out configuration properties that will later be replaced or are not needed"
157+
if [ "${NEO4J_MAJOR_VERSION_NUMBER}" -ge 5 ]; then
158+
sed -i.edit.backup '/^server\.directories\.import=/ s/^/# defined in the directory section further below #/' "${NEO4J_CONFIG}"
159+
sed -i.edit.backup '/^db\.tx_log\.rotation\.retention_policy=/ s/^/# defined in the transaction section further below #/' "${NEO4J_CONFIG}"
160+
else
161+
sed -i.edit.backup '/^dbms\.directories\.import=/ s/^/# defined in the directory section further below #/' "${NEO4J_CONFIG}"
162+
sed -i.edit.backup '/^dbms\.tx_log\.rotation\.retention_policy=/ s/^/# defined in the transaction section further below #/' "${NEO4J_CONFIG}"
163+
fi
164+
# Remove the edit backup file
165+
rm -f "${NEO4J_CONFIG}.edit.backup"
166+
167+
# Configure all paths with data that changes (database data, logs, ...) to be in the outside "data" directory
168+
# instead of inside the neo4j directory
169+
echo "${SCRIPT_NAME}: Configuring dynamic settings (data directories, ports, ...)"
170+
171+
neo4jDataPath=$(convertPosixToWindowsPathIfNecessary "${DATA_DIRECTORY}")
172+
neo4jLogsPath=$(convertPosixToWindowsPathIfNecessary "${RUNTIME_DIRECTORY}/logs")
173+
neo4jDumpsPath=$(convertPosixToWindowsPathIfNecessary "${RUNTIME_DIRECTORY}/dumps")
174+
neo4jRunPath=$(convertPosixToWindowsPathIfNecessary "${RUNTIME_DIRECTORY}/run")
175+
neo4jTransactionsPath=$(convertPosixToWindowsPathIfNecessary "${DATA_DIRECTORY}/transactions")
176+
neo4jImportPath=$(convertPosixToWindowsPathIfNecessary "${IMPORT_DIRECTORY}")
177+
178+
# Create import directory in case it doesn't exist.
179+
# The import needs to be configured even if its not used since it will be configured below and validated by Neo4j.
180+
mkdir -p "${IMPORT_DIRECTORY}"
181+
182+
if [ "${NEO4J_MAJOR_VERSION_NUMBER}" -ge 5 ]; then
183+
echo "setupNeo4j: Neo4j v5 or higher detected"
184+
{
185+
echo ""
186+
echo "# Paths of data directories in the installation (> v5)"
187+
echo "server.directories.data=${neo4jDataPath}"
188+
echo "server.directories.logs=${neo4jLogsPath}"
189+
echo "server.directories.dumps.root=${neo4jDumpsPath}"
190+
echo "server.directories.run=${neo4jRunPath}"
191+
echo "server.directories.transaction.logs.root=${neo4jTransactionsPath}"
192+
echo "server.directories.import=${neo4jImportPath}"
193+
echo ""
194+
echo "# Ports Configuration (> v5)"
195+
echo "server.bolt.listen_address=:${NEO4J_BOLT_PORT}"
196+
echo "server.bolt.advertised_address=:${NEO4J_BOLT_PORT}"
197+
echo "server.http.listen_address=:${NEO4J_HTTP_PORT}"
198+
echo "server.http.advertised_address=:${NEO4J_HTTP_PORT}"
199+
echo "server.https.listen_address=:${NEO4J_HTTPS_PORT}"
200+
echo "server.https.advertised_address=:${NEO4J_HTTPS_PORT}"
201+
} >> "${NEO4J_CONFIG}"
202+
else
203+
echo "setupNeo4j: Neo4j v4 or lower detected"
204+
{
205+
echo ""
206+
echo "# Paths of data directories in the installation"
207+
echo "dbms.directories.data=${neo4jDataPath}"
208+
echo "dbms.directories.logs=${neo4jLogsPath}"
209+
echo "dbms.directories.dumps.root=${neo4jDumpsPath}"
210+
echo "dbms.directories.run=${neo4jRunPath}"
211+
echo "dbms.directories.transaction.logs.root=${neo4jTransactionsPath}"
212+
echo "dbms.directories.import=${neo4jImportPath}"
213+
echo ""
214+
echo "# Ports Configuration"
215+
echo "dbms.connector.bolt.listen_address=:${NEO4J_BOLT_PORT}"
216+
echo "dbms.connector.bolt.advertised_address=:${NEO4J_BOLT_PORT}"
217+
echo "dbms.connector.http.listen_address=:${NEO4J_HTTP_PORT}"
218+
echo "dbms.connector.http.advertised_address=:${NEO4J_HTTP_PORT}"
219+
echo "dbms.connector.https.listen_address=:${NEO4J_HTTPS_PORT}"
220+
echo "dbms.connector.https.advertised_address=:${NEO4J_HTTPS_PORT}"
221+
} >> "${NEO4J_CONFIG}"
222+
fi
223+
224+
append_configuration_from_template
225+
insert_hint_that_the_configuration_is_script_managed "${NEO4J_CONFIG}"
226+
227+
echo "${SCRIPT_NAME}: Configuration successfully updated."

0 commit comments

Comments
 (0)