From 4e1405a18c807f3c58fe2720cb00f2a581bd26a5 Mon Sep 17 00:00:00 2001 From: Alexandre Vazquez Date: Sat, 8 Jun 2019 12:53:59 +0200 Subject: [PATCH 1/2] Included he option to set the Xmx based on the configured memory to the container. It uses the introduce new variable MEMORY_LIMIT that wasn't in use at the moment, using the following approach. - We get the amount of memory reserved for the container and substract a little bit for the rest of the container and everyhing else is set as Xmx. We have two way to set the amount of memory that is going to use for the system (not BW) based on variables: - MEMORY_FIXED_LIMIT (in MB): in case you want to substract a fixed amount of memory like 128 or similar. - MEMORY_DYAMIC_LIMIT in (percentage from 0-100): in case you want to substract a percentage of the assigned memory. In both cases if the memory that is dedicated is less than 128 M it is ignored and use a failsafe memory (memory assigned - 128). IN case only MEMORY_LIMIT is set, do same thing (memory assigned - 128) Also added a DEBUG trace that shows the amount of memory that is going to be assigned: --- reducedStartupTime/setup.sh | 30 +++++++++++++++++++++++++++--- scripts/setup.sh | 32 ++++++++++++++++++++++++++++---- 2 files changed, 55 insertions(+), 7 deletions(-) diff --git a/reducedStartupTime/setup.sh b/reducedStartupTime/setup.sh index e88df7b..b2bb6cf 100644 --- a/reducedStartupTime/setup.sh +++ b/reducedStartupTime/setup.sh @@ -289,9 +289,33 @@ done memoryCalculator() { - if [[ ${MEMORY_LIMIT} ]]; then - memory_Number=`echo $MEMORY_LIMIT | sed 's/m$//I'` - configured_MEM=$((($memory_Number*67+50)/100)) + if [[ ${MEMORY_LIMIT} ]]; then + + configured_MEM=$(expr `cat /sys/fs/cgroup/memory/memory.limit_in_bytes` / 1024 / 1024) + configured_MEM_orig=$configured_MEM + + if [[ ${MEMORY_DYNAMIC_LIMIT} ]]; then + sys_MEM=$(expr $configured_MEM \* ${MEMORY_DYNAMIC_LIMIT} / 100) + system_MEM=`printf "%.0f" $sys_MEM` + configured_MEM_temp=$(expr $configured_MEM - $system_MEM) + if [[ $configured_MEM_temp -gt 128 ]]; then + configured_MEM=$configured_MEM_temp + fi + elif [[ ${MEMORY_FIXED_LIMIT} ]]; then + configured_MEM_temp=$(expr $configured_MEM - ${MEMORY_FIXED_LIMIT} ) + if [[ $configured_MEM_temp -gt 128 ]]; then + configured_MEM=$configured_MEM_temp + fi + fi + + if [[ $configured_MEM -eq $configured_MEM_orig ]]; then + configured_MEM_temp=$(expr $configured_MEM - 128) + if [[ $configured_MEM_temp -gt 128 ]]; then + configured_MEM=$configured_MEM_temp + fi + fi + + print_Debug "Maximum memory calculated in a dynamic way to a value [$configured_MEM]" thread_Stack=$((memory_Number)) JAVA_PARAM="-Xmx"$configured_MEM"M -Xms128M -Xss512K" export BW_JAVA_OPTS=$JAVA_PARAM" "$BW_JAVA_OPTS diff --git a/scripts/setup.sh b/scripts/setup.sh index af0da6a..3ce33bb 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -289,13 +289,37 @@ done memoryCalculator() { - if [[ ${MEMORY_LIMIT} ]]; then - memory_Number=`echo $MEMORY_LIMIT | sed 's/m$//I'` - configured_MEM=$((($memory_Number*67+50)/100)) + if [[ ${MEMORY_LIMIT} ]]; then + + configured_MEM=$(expr `cat /sys/fs/cgroup/memory/memory.limit_in_bytes` / 1024 / 1024) + configured_MEM_orig=$configured_MEM + + if [[ ${MEMORY_DYNAMIC_LIMIT} ]]; then + sys_MEM=$(expr $configured_MEM \* ${MEMORY_DYNAMIC_LIMIT} / 100) + system_MEM=`printf "%.0f" $sys_MEM` + configured_MEM_temp=$(expr $configured_MEM - $system_MEM) + if [[ $configured_MEM_temp -gt 128 ]]; then + configured_MEM=$configured_MEM_temp + fi + elif [[ ${MEMORY_FIXED_LIMIT} ]]; then + configured_MEM_temp=$(expr $configured_MEM - ${MEMORY_FIXED_LIMIT} ) + if [[ $configured_MEM_temp -gt 128 ]]; then + configured_MEM=$configured_MEM_temp + fi + fi + + if [[ $configured_MEM -eq $configured_MEM_orig ]]; then + configured_MEM_temp=$(expr $configured_MEM - 128) + if [[ $configured_MEM_temp -gt 128 ]]; then + configured_MEM=$configured_MEM_temp + fi + fi + + print_Debug "Maximum memory calculated in a dynamic way to a value [$configured_MEM]" thread_Stack=$((memory_Number)) JAVA_PARAM="-Xmx"$configured_MEM"M -Xms128M -Xss512K" export BW_JAVA_OPTS=$JAVA_PARAM" "$BW_JAVA_OPTS - fi + fi } applyDefaultJVMHeapParams(){ From 0afaf7cf2dbe4115d7bb5503226c7997df7c97da Mon Sep 17 00:00:00 2001 From: Alexandre Vazquez Date: Sat, 8 Jun 2019 23:37:34 +0200 Subject: [PATCH 2/2] Introducing new property BW_COMPONENT_JOB_FLOWLIMIT that include process level flowlimit with the following syntax "version.componentName:value1;version.componentName2:value2;" to be able to specify flowlimit at the component level. --- reducedStartupTime/setup.sh | 13 +++++++++++++ scripts/setup.sh | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/reducedStartupTime/setup.sh b/reducedStartupTime/setup.sh index b2bb6cf..72f88c6 100644 --- a/reducedStartupTime/setup.sh +++ b/reducedStartupTime/setup.sh @@ -182,6 +182,19 @@ checkEnvSubstituteConfig() print_Debug "set BW_APPLICATION_JOB_FLOWLIMIT to $BW_APPLICATION_JOB_FLOWLIMIT" fi fi + if [[ ${BW_COMPONENT_JOB_FLOWLIMIT} ]]; then + if [ -e ${appnodeConfigFile} ]; then + IFS=';' # space is set as delimiter + read -ra processConfigurationList <<< "${BW_COMPONENT_JOB_FLOWLIMIT}" # str is read into an array as tokens separated by IFS + for process in "${processConfigurationList[@]}"; do # access each element of array + echo "Setting flow limit for $process" + IFS=':' # space is set as delimiter + read -ra processConfiguration <<< "$process" # str is read into an array as tokens separated by IFS + printf '%s\n' "bw.application.job.flowlimit.$bwBundleAppName.${processConfiguration[0]}=${processConfiguration[1]}" >> $appnodeConfigFile + print_Debug "set bw.application.job.flowlimit.$bwBundleAppName.${processConfiguration[0]} to ${processConfiguration[1]}" + done + fi + fi if [[ ${BW_APP_MONITORING_CONFIG} ]]; then if [ -e ${appnodeConfigFile} ]; then sed -i 's/bw.frwk.event.subscriber.metrics.enabled=false/bw.frwk.event.subscriber.metrics.enabled=true/g' $appnodeConfigFile diff --git a/scripts/setup.sh b/scripts/setup.sh index 3ce33bb..d23ba21 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -182,6 +182,19 @@ checkEnvSubstituteConfig() print_Debug "set BW_APPLICATION_JOB_FLOWLIMIT to $BW_APPLICATION_JOB_FLOWLIMIT" fi fi + if [[ ${BW_COMPONENT_JOB_FLOWLIMIT} ]]; then + if [ -e ${appnodeConfigFile} ]; then + IFS=';' # space is set as delimiter + read -ra processConfigurationList <<< "${BW_COMPONENT_JOB_FLOWLIMIT}" # str is read into an array as tokens separated by IFS + for process in "${processConfigurationList[@]}"; do # access each element of array + echo "Setting flow limit for $process" + IFS=':' # space is set as delimiter + read -ra processConfiguration <<< "$process" # str is read into an array as tokens separated by IFS + printf '%s\n' "bw.application.job.flowlimit.$bwBundleAppName.${processConfiguration[0]}=${processConfiguration[1]}" >> $appnodeConfigFile + print_Debug "set bw.application.job.flowlimit.$bwBundleAppName.${processConfiguration[0]} to ${processConfiguration[1]}" + done + fi + fi if [[ ${BW_APP_MONITORING_CONFIG} ]]; then if [ -e ${appnodeConfigFile} ]; then sed -i 's/bw.frwk.event.subscriber.metrics.enabled=false/bw.frwk.event.subscriber.metrics.enabled=true/g' $appnodeConfigFile