Skip to content

Commit 064cd50

Browse files
robertMileaNirobertMileaNi
committed
feedback
1 parent 593dbe4 commit 064cd50

4 files changed

Lines changed: 121 additions & 205 deletions

File tree

‎scripts/conf/confignode-env.sh‎

Lines changed: 2 additions & 104 deletions
Original file line numberDiff line numberDiff line change
@@ -64,109 +64,6 @@ esac
6464
# whether we allow enable heap dump files
6565
IOTDB_ALLOW_HEAP_DUMP="true"
6666

67-
calculate_memory_sizes()
68-
{
69-
case "`uname`" in
70-
Linux)
71-
system_memory_in_mb=`free -m| sed -n '2p' | awk '{print $2}'`
72-
# When running in a container/pod, use cgroup memory limit instead of host memory
73-
if [ -f /sys/fs/cgroup/memory.max ]; then
74-
# cgroup v2
75-
cgroup_mem=`cat /sys/fs/cgroup/memory.max`
76-
if [ "$cgroup_mem" != "max" ]; then
77-
cgroup_mem_in_mb=`expr $cgroup_mem / 1024 / 1024`
78-
if [ "$cgroup_mem_in_mb" -lt "$system_memory_in_mb" ]; then
79-
system_memory_in_mb=$cgroup_mem_in_mb
80-
fi
81-
fi
82-
elif [ -f /sys/fs/cgroup/memory/memory.limit_in_bytes ]; then
83-
# cgroup v1
84-
cgroup_mem=`cat /sys/fs/cgroup/memory/memory.limit_in_bytes`
85-
cgroup_mem_in_mb=`expr $cgroup_mem / 1024 / 1024`
86-
if [ "$cgroup_mem_in_mb" -lt "$system_memory_in_mb" ]; then
87-
system_memory_in_mb=$cgroup_mem_in_mb
88-
fi
89-
fi
90-
system_cpu_cores=`egrep -c 'processor([[:space:]]+):.*' /proc/cpuinfo`
91-
;;
92-
FreeBSD)
93-
system_memory_in_bytes=`sysctl hw.physmem | awk '{print $2}'`
94-
system_memory_in_mb=`expr $system_memory_in_bytes / 1024 / 1024`
95-
system_cpu_cores=`sysctl hw.ncpu | awk '{print $2}'`
96-
;;
97-
SunOS)
98-
system_memory_in_mb=`prtconf | awk '/Memory size:/ {print $3}'`
99-
system_cpu_cores=`psrinfo | wc -l`
100-
;;
101-
Darwin)
102-
system_memory_in_bytes=`sysctl hw.memsize | awk '{print $2}'`
103-
system_memory_in_mb=`expr $system_memory_in_bytes / 1024 / 1024`
104-
system_cpu_cores=`sysctl hw.ncpu | awk '{print $2}'`
105-
;;
106-
*)
107-
# assume reasonable defaults for e.g. a modern desktop or
108-
# cheap server
109-
system_memory_in_mb="2048"
110-
system_cpu_cores="2"
111-
;;
112-
esac
113-
114-
# some systems like the raspberry pi don't report cores, use at least 1
115-
if [ "$system_cpu_cores" -lt "1" ]
116-
then
117-
system_cpu_cores="1"
118-
fi
119-
120-
# suggest using memory, system memory 3 / 10
121-
suggest_using_memory_in_mb=`expr $system_memory_in_mb / 10 \* 3`
122-
123-
if [ -n "$MEMORY_SIZE" ]
124-
then
125-
if [ "${MEMORY_SIZE%"G"}" != "$MEMORY_SIZE" ] || [ "${MEMORY_SIZE%"M"}" != "$MEMORY_SIZE" ]
126-
then
127-
if [ "${MEMORY_SIZE%"G"}" != "$MEMORY_SIZE" ]
128-
then
129-
memory_size_in_mb=`expr ${MEMORY_SIZE%"G"} "*" 1024`
130-
else
131-
memory_size_in_mb=`expr ${MEMORY_SIZE%"M"}`
132-
fi
133-
else
134-
echo "Invalid format of MEMORY_SIZE, please use the format like 2048M or 2G"
135-
exit 1
136-
fi
137-
else
138-
# set memory size to suggest using memory, if suggest using memory is greater than 8GB, set memory size to 8GB
139-
if [ "$suggest_using_memory_in_mb" -gt "8192" ]
140-
then
141-
memory_size_in_mb="8192"
142-
else
143-
memory_size_in_mb=$suggest_using_memory_in_mb
144-
fi
145-
fi
146-
147-
# set on heap memory size
148-
# when memory_size_in_mb is less than 4 * 1024, we will set on heap memory size to memory_size_in_mb / 4 * 3
149-
# when memory_size_in_mb is greater than 4 * 1024 and less than 16 * 1024, we will set on heap memory size to memory_size_in_mb / 5 * 4
150-
# when memory_size_in_mb is greater than 16 * 1024 and less than 128 * 1024, we will set on heap memory size to memory_size_in_mb / 8 * 7
151-
# when memory_size_in_mb is greater than 128 * 1024, we will set on heap memory size to memory_size_in_mb - 16 * 1024
152-
if [ "$memory_size_in_mb" -lt "4096" ]
153-
then
154-
on_heap_memory_size_in_mb=`expr $memory_size_in_mb / 4 \* 3`
155-
elif [ "$memory_size_in_mb" -lt "16384" ]
156-
then
157-
on_heap_memory_size_in_mb=`expr $memory_size_in_mb / 5 \* 4`
158-
elif [ "$memory_size_in_mb" -lt "131072" ]
159-
then
160-
on_heap_memory_size_in_mb=`expr $memory_size_in_mb / 8 \* 7`
161-
else
162-
on_heap_memory_size_in_mb=`expr $memory_size_in_mb - 16384`
163-
fi
164-
off_heap_memory_size_in_mb=`expr $memory_size_in_mb - $on_heap_memory_size_in_mb`
165-
166-
ON_HEAP_MEMORY="${on_heap_memory_size_in_mb}M"
167-
OFF_HEAP_MEMORY="${off_heap_memory_size_in_mb}M"
168-
}
169-
17067
get_cn_system_dir() {
17168
local config_file="$1"
17269
local cn_system_dir=""
@@ -253,7 +150,8 @@ illegal_access_params="$illegal_access_params --add-opens=java.base/java.io=ALL-
253150
illegal_access_params="$illegal_access_params --add-opens=java.base/java.net=ALL-UNNAMED"
254151

255152

256-
calculate_memory_sizes
153+
# ConfigNode: suggest 30% of system memory (3/10), cap at 8 GB.
154+
calculate_memory_sizes 3 10 8192
257155

258156
# on heap memory size
259157
#ON_HEAP_MEMORY="2G"

‎scripts/conf/datanode-env.sh‎

Lines changed: 2 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -65,104 +65,6 @@ esac
6565
# whether we allow enable heap dump files
6666
IOTDB_ALLOW_HEAP_DUMP="true"
6767

68-
calculate_memory_sizes()
69-
{
70-
case "`uname`" in
71-
Linux)
72-
system_memory_in_mb=`free -m| sed -n '2p' | awk '{print $2}'`
73-
# When running in a container/pod, use cgroup memory limit instead of host memory
74-
if [ -f /sys/fs/cgroup/memory.max ]; then
75-
# cgroup v2
76-
cgroup_mem=`cat /sys/fs/cgroup/memory.max`
77-
if [ "$cgroup_mem" != "max" ]; then
78-
cgroup_mem_in_mb=`expr $cgroup_mem / 1024 / 1024`
79-
if [ "$cgroup_mem_in_mb" -lt "$system_memory_in_mb" ]; then
80-
system_memory_in_mb=$cgroup_mem_in_mb
81-
fi
82-
fi
83-
elif [ -f /sys/fs/cgroup/memory/memory.limit_in_bytes ]; then
84-
# cgroup v1
85-
cgroup_mem=`cat /sys/fs/cgroup/memory/memory.limit_in_bytes`
86-
cgroup_mem_in_mb=`expr $cgroup_mem / 1024 / 1024`
87-
if [ "$cgroup_mem_in_mb" -lt "$system_memory_in_mb" ]; then
88-
system_memory_in_mb=$cgroup_mem_in_mb
89-
fi
90-
fi
91-
system_cpu_cores=`egrep -c 'processor([[:space:]]+):.*' /proc/cpuinfo`
92-
;;
93-
FreeBSD)
94-
system_memory_in_bytes=`sysctl hw.physmem | awk '{print $2}'`
95-
system_memory_in_mb=`expr $system_memory_in_bytes / 1024 / 1024`
96-
system_cpu_cores=`sysctl hw.ncpu | awk '{print $2}'`
97-
;;
98-
SunOS)
99-
system_memory_in_mb=`prtconf | awk '/Memory size:/ {print $3}'`
100-
system_cpu_cores=`psrinfo | wc -l`
101-
;;
102-
Darwin)
103-
system_memory_in_bytes=`sysctl hw.memsize | awk '{print $2}'`
104-
system_memory_in_mb=`expr $system_memory_in_bytes / 1024 / 1024`
105-
system_cpu_cores=`sysctl hw.ncpu | awk '{print $2}'`
106-
;;
107-
*)
108-
# assume reasonable defaults for e.g. a modern desktop or
109-
# cheap server
110-
system_memory_in_mb="2048"
111-
system_cpu_cores="2"
112-
;;
113-
esac
114-
115-
# some systems like the raspberry pi don't report cores, use at least 1
116-
if [ "$system_cpu_cores" -lt "1" ]
117-
then
118-
system_cpu_cores="1"
119-
fi
120-
121-
# suggest using memory, system memory 1 / 2
122-
suggest_using_memory_in_mb=`expr $system_memory_in_mb / 2`
123-
124-
if [ -n "$MEMORY_SIZE" ]
125-
then
126-
if [ "${MEMORY_SIZE%"G"}" != "$MEMORY_SIZE" ] || [ "${MEMORY_SIZE%"M"}" != "$MEMORY_SIZE" ]
127-
then
128-
if [ "${MEMORY_SIZE%"G"}" != "$MEMORY_SIZE" ]
129-
then
130-
memory_size_in_mb=`expr ${MEMORY_SIZE%"G"} "*" 1024`
131-
else
132-
memory_size_in_mb=`expr ${MEMORY_SIZE%"M"}`
133-
fi
134-
else
135-
echo "Invalid format of MEMORY_SIZE, please use the format like 2048M or 2G"
136-
exit 1
137-
fi
138-
else
139-
memory_size_in_mb=$suggest_using_memory_in_mb
140-
fi
141-
142-
# set on heap memory size
143-
# when memory_size_in_mb is less than 4 * 1024, we will set on heap memory size to memory_size_in_mb / 4 * 3
144-
# when memory_size_in_mb is greater than 4 * 1024 and less than 16 * 1024, we will set on heap memory size to memory_size_in_mb / 5 * 4
145-
# when memory_size_in_mb is greater than 16 * 1024 and less than 128 * 1024, we will set on heap memory size to memory_size_in_mb / 8 * 7
146-
# when memory_size_in_mb is greater than 128 * 1024, we will set on heap memory size to memory_size_in_mb - 16 * 1024
147-
if [ "$memory_size_in_mb" -lt "4096" ]
148-
then
149-
on_heap_memory_size_in_mb=`expr $memory_size_in_mb / 4 \* 3`
150-
elif [ "$memory_size_in_mb" -lt "16384" ]
151-
then
152-
on_heap_memory_size_in_mb=`expr $memory_size_in_mb / 5 \* 4`
153-
elif [ "$memory_size_in_mb" -lt "131072" ]
154-
then
155-
on_heap_memory_size_in_mb=`expr $memory_size_in_mb / 8 \* 7`
156-
else
157-
on_heap_memory_size_in_mb=`expr $memory_size_in_mb - 16384`
158-
fi
159-
off_heap_memory_size_in_mb=`expr $memory_size_in_mb - $on_heap_memory_size_in_mb`
160-
161-
ON_HEAP_MEMORY="${on_heap_memory_size_in_mb}M"
162-
OFF_HEAP_MEMORY="${off_heap_memory_size_in_mb}M"
163-
}
164-
165-
16668
# find first dir of dn_data_dirs from properties file
16769
get_first_data_dir() {
16870
local config_file="$1"
@@ -261,7 +163,8 @@ illegal_access_params="$illegal_access_params --add-opens=java.base/java.nio=ALL
261163
illegal_access_params="$illegal_access_params --add-opens=java.base/java.io=ALL-UNNAMED"
262164
illegal_access_params="$illegal_access_params --add-opens=java.base/java.net=ALL-UNNAMED"
263165

264-
calculate_memory_sizes
166+
# DataNode: suggest 50% of system memory (1/2), no cap.
167+
calculate_memory_sizes 1 2 0
265168

266169
# on heap memory size
267170
#ON_HEAP_MEMORY="2G"

‎scripts/conf/iotdb-common.sh‎

Lines changed: 116 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -392,4 +392,119 @@ get_iotdb_include() {
392392
esac
393393
done
394394
echo "$VARS"
395-
}
395+
}
396+
397+
# Calculate ON_HEAP_MEMORY and OFF_HEAP_MEMORY from system resources.
398+
#
399+
# $1 numerator of the suggested memory fraction (e.g. 1 for DataNode, 3 for ConfigNode)
400+
# $2 denominator of the suggested memory fraction (e.g. 2 for DataNode, 10 for ConfigNode)
401+
# $3 max suggested memory in MB; 0 means no cap (e.g. 0 for DataNode, 8192 for ConfigNode)
402+
#
403+
# Respects the MEMORY_SIZE environment variable (e.g. '2G' or '2048M') when set.
404+
calculate_memory_sizes()
405+
{
406+
local suggest_numerator=${1:-1}
407+
local suggest_denominator=${2:-2}
408+
local max_suggest_mb=${3:-0}
409+
410+
case "`uname`" in
411+
Linux)
412+
system_memory_in_mb=`free -m| sed -n '2p' | awk '{print $2}'`
413+
# When running in a container/pod, use cgroup memory limit instead of host memory.
414+
# This reads the root cgroup, which is what containers see; it does NOT walk
415+
# /proc/self/cgroup, so cgroup-restricted processes on bare metal are not covered.
416+
if [ -f /sys/fs/cgroup/memory.max ]; then
417+
# cgroup v2
418+
cgroup_mem=`cat /sys/fs/cgroup/memory.max`
419+
if [ "$cgroup_mem" != "max" ]; then
420+
cgroup_mem_in_mb=`expr $cgroup_mem / 1024 / 1024`
421+
if [ "$cgroup_mem_in_mb" -lt "$system_memory_in_mb" ]; then
422+
system_memory_in_mb=$cgroup_mem_in_mb
423+
fi
424+
fi
425+
elif [ -f /sys/fs/cgroup/memory/memory.limit_in_bytes ]; then
426+
# v1 has no "max" sentinel — when unset, this file holds a huge integer (~PAGE_COUNTER_MAX);
427+
# the -lt check below naturally rejects it, so no explicit guard is needed.
428+
cgroup_mem=`cat /sys/fs/cgroup/memory/memory.limit_in_bytes`
429+
cgroup_mem_in_mb=`expr $cgroup_mem / 1024 / 1024`
430+
if [ "$cgroup_mem_in_mb" -lt "$system_memory_in_mb" ]; then
431+
system_memory_in_mb=$cgroup_mem_in_mb
432+
fi
433+
fi
434+
system_cpu_cores=`egrep -c 'processor([[:space:]]+):.*' /proc/cpuinfo`
435+
;;
436+
FreeBSD)
437+
system_memory_in_bytes=`sysctl hw.physmem | awk '{print $2}'`
438+
system_memory_in_mb=`expr $system_memory_in_bytes / 1024 / 1024`
439+
system_cpu_cores=`sysctl hw.ncpu | awk '{print $2}'`
440+
;;
441+
SunOS)
442+
system_memory_in_mb=`prtconf | awk '/Memory size:/ {print $3}'`
443+
system_cpu_cores=`psrinfo | wc -l`
444+
;;
445+
Darwin)
446+
system_memory_in_bytes=`sysctl hw.memsize | awk '{print $2}'`
447+
system_memory_in_mb=`expr $system_memory_in_bytes / 1024 / 1024`
448+
system_cpu_cores=`sysctl hw.ncpu | awk '{print $2}'`
449+
;;
450+
*)
451+
# assume reasonable defaults for e.g. a modern desktop or
452+
# cheap server
453+
system_memory_in_mb="2048"
454+
system_cpu_cores="2"
455+
;;
456+
esac
457+
458+
# some systems like the raspberry pi don't report cores, use at least 1
459+
if [ "$system_cpu_cores" -lt "1" ]
460+
then
461+
system_cpu_cores="1"
462+
fi
463+
464+
suggest_using_memory_in_mb=`expr $system_memory_in_mb / $suggest_denominator \* $suggest_numerator`
465+
466+
if [ -n "$MEMORY_SIZE" ]
467+
then
468+
if [ "${MEMORY_SIZE%"G"}" != "$MEMORY_SIZE" ] || [ "${MEMORY_SIZE%"M"}" != "$MEMORY_SIZE" ]
469+
then
470+
if [ "${MEMORY_SIZE%"G"}" != "$MEMORY_SIZE" ]
471+
then
472+
memory_size_in_mb=`expr ${MEMORY_SIZE%"G"} "*" 1024`
473+
else
474+
memory_size_in_mb=`expr ${MEMORY_SIZE%"M"}`
475+
fi
476+
else
477+
echo "Invalid format of MEMORY_SIZE, please use the format like 2048M or 2G"
478+
exit 1
479+
fi
480+
else
481+
if [ "$max_suggest_mb" -gt "0" ] && [ "$suggest_using_memory_in_mb" -gt "$max_suggest_mb" ]
482+
then
483+
memory_size_in_mb=$max_suggest_mb
484+
else
485+
memory_size_in_mb=$suggest_using_memory_in_mb
486+
fi
487+
fi
488+
489+
# set on heap memory size
490+
# when memory_size_in_mb is less than 4 * 1024, we will set on heap memory size to memory_size_in_mb / 4 * 3
491+
# when memory_size_in_mb is greater than 4 * 1024 and less than 16 * 1024, we will set on heap memory size to memory_size_in_mb / 5 * 4
492+
# when memory_size_in_mb is greater than 16 * 1024 and less than 128 * 1024, we will set on heap memory size to memory_size_in_mb / 8 * 7
493+
# when memory_size_in_mb is greater than 128 * 1024, we will set on heap memory size to memory_size_in_mb - 16 * 1024
494+
if [ "$memory_size_in_mb" -lt "4096" ]
495+
then
496+
on_heap_memory_size_in_mb=`expr $memory_size_in_mb / 4 \* 3`
497+
elif [ "$memory_size_in_mb" -lt "16384" ]
498+
then
499+
on_heap_memory_size_in_mb=`expr $memory_size_in_mb / 5 \* 4`
500+
elif [ "$memory_size_in_mb" -lt "131072" ]
501+
then
502+
on_heap_memory_size_in_mb=`expr $memory_size_in_mb / 8 \* 7`
503+
else
504+
on_heap_memory_size_in_mb=`expr $memory_size_in_mb - 16384`
505+
fi
506+
off_heap_memory_size_in_mb=`expr $memory_size_in_mb - $on_heap_memory_size_in_mb`
507+
508+
ON_HEAP_MEMORY="${on_heap_memory_size_in_mb}M"
509+
OFF_HEAP_MEMORY="${off_heap_memory_size_in_mb}M"
510+
}

‎scripts/conf/windows/confignode-env.bat‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919

2020
@echo off
2121

22-
@REM You can set datanode memory size, example '2G' or '2048M'
22+
@REM You can set confignode memory size, example '2G' or '2048M'
2323
@REM If the MEMORY_SIZE environment variable is already set, its value will be used.
2424
if not defined MEMORY_SIZE set MEMORY_SIZE=
2525

0 commit comments

Comments
 (0)