Skip to content

Commit 1466e39

Browse files
committed
Merge branch 'better_init' of github.com:ocaisa/software-layer-scripts into better_init
2 parents c3d3159 + 6b057e8 commit 1466e39

3 files changed

Lines changed: 146 additions & 19 deletions

File tree

.github/workflows/test_eessi_container_script.yml

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ jobs:
1717
- help
1818
- listrepos_default
1919
- listrepos_custom
20+
- no_cvmfs_mounts
21+
- exec
2022
- run
2123
- shell
2224
- container
@@ -70,14 +72,33 @@ jobs:
7072
./eessi_container.sh --verbose --list-repos | tee ${outfile2}
7173
grep "EESSI/20AB.CD" ${outfile2}
7274
73-
# test use of --mode run
75+
# test use of --repository None
76+
elif [[ ${{matrix.SCRIPT_TEST}} == 'no_cvmfs_mounts' ]]; then
77+
outfile=out_nocvmfs.txt
78+
./eessi_container.sh --verbose --repository None --mode exec mount | tee ${outfile}
79+
# make sure that there are no CVMFS mounts in the container, i.e. mount does not have any lines line:
80+
# cvmfs2 on /cvmfs/...
81+
! grep "^cvmfs" ${outfile}
82+
83+
# test use of --mode exec
84+
elif [[ ${{matrix.SCRIPT_TEST}} == 'exec' ]]; then
85+
outfile=out_exec.txt
86+
echo "${test_cmd}" > test_script.sh
87+
chmod u+x test_script.sh
88+
export SINGULARITY_BIND="$PWD:/test"
89+
./eessi_container.sh --verbose --mode exec /test/test_script.sh | tee ${outfile}
90+
grep "${out_pattern}" ${outfile}
91+
92+
# test use of --mode run, which should now print a warning
7493
elif [[ ${{matrix.SCRIPT_TEST}} == 'run' ]]; then
7594
outfile=out_run.txt
7695
echo "${test_cmd}" > test_script.sh
7796
chmod u+x test_script.sh
7897
export SINGULARITY_BIND="$PWD:/test"
7998
./eessi_container.sh --verbose --mode run /test/test_script.sh | tee ${outfile}
8099
grep "${out_pattern}" ${outfile}
100+
warn_message="Note: the behaviour of the run mode has changed."
101+
grep "${warn_message}" ${outfile}
81102
82103
# test use of --mode shell
83104
elif [[ ${{matrix.SCRIPT_TEST}} == 'shell' ]]; then
@@ -100,7 +121,7 @@ jobs:
100121
echo "touch /cvmfs/software.eessi.io/${fn}" > test_script.sh
101122
chmod u+x test_script.sh
102123
export SINGULARITY_BIND="$PWD:/test"
103-
./eessi_container.sh --verbose --access rw --mode run /test/test_script.sh > ${outfile}
124+
./eessi_container.sh --verbose --access rw --mode exec /test/test_script.sh > ${outfile}
104125
105126
tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g")
106127
# note: must use '--access rw' again here, since touched file is in overlay upper dir
@@ -143,7 +164,7 @@ jobs:
143164
export SINGULARITY_BIND="$PWD:/test"
144165
echo 'ls -ld /cvmfs*/software.eessi.io/*' > test_script.sh
145166
chmod u+x test_script.sh
146-
./eessi_container.sh --verbose --container ${container} --access rw --overlay-tool unionfs --mode run /test/test_script.sh 2>&1 | tee ${outfile}
167+
./eessi_container.sh --verbose --container ${container} --access rw --overlay-tool unionfs --mode exec /test/test_script.sh 2>&1 | tee ${outfile}
147168
for pattern in "/cvmfs/software.eessi.io/versions" "/cvmfs_ro/software.eessi.io/versions"; do
148169
grep "${pattern}" ${outfile} || (echo "Pattern '${pattern}' not found in ${outfile}"; exit 1)
149170
done

.github/workflows/tests_scripts.yml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ jobs:
8585
8686
# run wrapper script + capture & check output
8787
out="${PWD}/eb-${EB_VERSION}.out"
88-
./eessi_container.sh --access rw --mode run --verbose /software-layer-scripts/run_in_compat_layer_env.sh /software-layer-scripts/eb-${EB_VERSION}.sh 2>&1 | tee ${out}
88+
./eessi_container.sh --access rw --mode exec --verbose /software-layer-scripts/run_in_compat_layer_env.sh /software-layer-scripts/eb-${EB_VERSION}.sh 2>&1 | tee ${out}
8989
pattern="^This is EasyBuild ${EB_VERSION} "
9090
grep "${pattern}" ${out} || (echo "Pattern '${pattern}' not found in output!" && exit 1)
9191
done
@@ -103,7 +103,7 @@ jobs:
103103
# make sure that correct EESSI version is used (required because default is a placeholder version)
104104
export EESSI_VERSION_OVERRIDE="${{matrix.EESSI_VERSION}}"
105105
106-
./eessi_container.sh --mode run --verbose /software-layer-scripts/install_software_layer.sh
106+
./eessi_container.sh --mode exec --verbose /software-layer-scripts/install_software_layer.sh
107107
108108
- name: test create_directory_tarballs.sh script
109109
run: |
@@ -116,7 +116,7 @@ jobs:
116116
117117
# scripts need to be copied to /tmp,
118118
# since create_directory_tarballs.sh must be accessible from within build container
119-
./eessi_container.sh --mode run --verbose /software-layer-scripts/create_directory_tarballs.sh "${{matrix.EESSI_VERSION}}"
119+
./eessi_container.sh --mode exec --verbose /software-layer-scripts/create_directory_tarballs.sh "${{matrix.EESSI_VERSION}}"
120120
# check if tarballs have been produced
121121
ls -l *.tar.gz
122122
@@ -140,7 +140,7 @@ jobs:
140140
export EESSI_VERSION_OVERRIDE="${{matrix.EESSI_VERSION}}"
141141
142142
out="${PWD}/test_create_lmodsitepackage.out"
143-
./eessi_container.sh --mode run --verbose /software-layer-scripts/run_in_compat_layer_env.sh /software-layer-scripts/test_lmod_sitepackage.sh 2>&1 | tee ${out}
143+
./eessi_container.sh --mode exec --verbose /software-layer-scripts/run_in_compat_layer_env.sh /software-layer-scripts/test_lmod_sitepackage.sh 2>&1 | tee ${out}
144144
for pattern in "^Site Pkg location.*/software-layer-scripts/.lmod/SitePackage.lua" "LMOD_SITEPACKAGE_LOCATION.*/software-layer-scripts/.lmod/SitePackage.lua"; do
145145
grep "${pattern}" ${out} || (echo "Pattern '${pattern}' not found in output!" && exit 1)
146146
done
@@ -160,6 +160,6 @@ jobs:
160160
export EESSI_VERSION_OVERRIDE="${{matrix.EESSI_VERSION}}"
161161
162162
out="${PWD}/eb-${EB_VERSION}.out"
163-
./eessi_container.sh --repository software.eessi.io,access=rw,mount=bind --mode run --verbose /software-layer-scripts/run_in_compat_layer_env.sh ls 2>&1 | tee ${out}
163+
./eessi_container.sh --repository software.eessi.io,access=rw,mount=bind --mode exec --verbose /software-layer-scripts/run_in_compat_layer_env.sh ls 2>&1 | tee ${out}
164164
echo $(grep "SINGULARITY_BIND" ${out})
165165
grep "SINGULARITY_BIND" ${out} | grep "software.eessi.io" || (echo "software.eessi.io did not seem to be bind mounted!" && exit 1)

eessi_container.sh

Lines changed: 117 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ display_help() {
8686
echo " [default: /..storage../opt-eessi]"
8787
echo " -l | --list-repos - list available repository identifiers [default: false]"
8888
echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or"
89-
echo " MODE==run (run a script or command) [default: shell]"
89+
echo " MODE==exec/run (run a script or command) [default: shell]"
9090
echo " -n | --nvidia MODE - configure the container to work with NVIDIA GPUs,"
9191
echo " MODE==install for a CUDA installation, MODE==run to"
9292
echo " attach a GPU, MODE==all for both [default: false]"
@@ -98,7 +98,8 @@ display_help() {
9898
echo " -r | --repository CFG - configuration file or identifier defining the"
9999
echo " repository to use; can be given multiple times;"
100100
echo " CFG may include suffixes ',access={ro,rw},mode={bind,fuse}' to"
101-
echo " overwrite the global access and/or mount mode for this repository"
101+
echo " overwrite the global access and/or mount mode for this repository;"
102+
echo " use 'None' to not mount any repositories"
102103
echo " [default: software.eessi.io via CVMFS config available"
103104
echo " via default container, see --container]"
104105
echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball,"
@@ -118,11 +119,79 @@ display_help() {
118119
echo " -y | --https-proxy URL - provides URL for the env variable https_proxy"
119120
echo " [default: not set]; uses env var \$https_proxy if set"
120121
echo
121-
echo " If value for --mode is 'run', the SCRIPT/COMMAND provided is executed. If"
122+
echo " If value for --mode is 'exec' or 'run', the SCRIPT/COMMAND provided is executed. If"
122123
echo " arguments to the script/command start with '-' or '--', use the flag terminator"
123124
echo " '--' to let eessi_container.sh stop parsing arguments."
124125
}
125126

127+
# function to parse and check bind paths
128+
# returns:
129+
# 0 if target found with matching source (no conflict)
130+
# 1 if target found with different source (conflict)
131+
# 2 if target not found
132+
check_bind_paths_for_target() {
133+
local search_target="$1"
134+
local search_src="$2"
135+
local bind_paths="$3" # typically used to pass value of $BIND_PATHS
136+
137+
# handle empty BIND_PATHS
138+
if [[ -z "${bind_paths}" ]]; then
139+
return 2
140+
fi
141+
142+
# split by comma and process each entry
143+
IFS=',' read -ra BIND_ENTRIES <<< "${bind_paths}"
144+
145+
for entry in "${BIND_ENTRIES[@]}"; do
146+
# skip empty entries
147+
[[ -z "${entry}" ]] && continue
148+
149+
local bind_src bind_target
150+
151+
# split entry by ':'
152+
IFS=':' read -ra PARTS <<< "${entry}"
153+
154+
bind_src="${PARTS[0]}"
155+
156+
if [[ ${#PARTS[@]} -ge 2 ]]; then
157+
bind_target="${PARTS[1]}"
158+
else
159+
# no target given, use src
160+
bind_target=${bind_src}
161+
fi
162+
163+
# trim any possible whitespace
164+
bind_src=$(echo "${bind_src}" | xargs)
165+
bind_target=$(echo "${bind_target}" | xargs)
166+
167+
[[ ${VERBOSE} -eq 1 ]] && echo "Parsed bind entry: src='${bind_src}' target='${bind_target}'"
168+
169+
# check if this entry matches our target
170+
if [[ "${bind_target}" == "${search_target}" ]]; then
171+
# found target -> need to compare normalised sources
172+
# try to normalise source paths, but don't fail if they don't exist (yet)
173+
174+
bind_src_normalised=$(readlink -f "${bind_src}" 2>/dev/null)
175+
search_src_normalised=$(readlink -f "${search_src}" 2>/dev/null)
176+
177+
# decide which path to use - normalised or original
178+
local bind_src_compare="${bind_src_normalised:-${bind_src}}"
179+
local search_src_compare="${search_src_normalised:-${search_src}}"
180+
181+
[[ ${VERBOSE} -eq 1 ]] && echo "Comparing: '${bind_src_compare}' vs '${search_src_compare}'"
182+
183+
if [[ "${bind_src_compare}" == "${search_src_compare}" ]]; then
184+
return 0 # found target with same source (all good)
185+
else
186+
echo "${bind_src}" # return the conflicting source for error message
187+
return 1 # found target with different source (conflict)
188+
fi
189+
fi
190+
done
191+
192+
return 2 # target not found in bind paths
193+
}
194+
126195
# set defaults for command line arguments
127196
ACCESS="ro"
128197
CONTAINER="docker://ghcr.io/eessi/build-node:debian12"
@@ -285,6 +354,14 @@ if [[ ${#REPOSITORIES[@]} -eq 0 ]]; then
285354
REPOSITORIES+=(${eessi_default_cvmfs_repo})
286355
fi
287356

357+
# if the first element of REPOSITORIES is "none" (case-insensitive),
358+
# make sure it is an empty list from here on, i.e. no repositories will be mounted
359+
if [[ ${REPOSITORIES[0],,} == "none" ]]; then
360+
REPOSITORIES=()
361+
# also prevent the cvmfs-config repo from being mounted
362+
EESSI_DO_NOT_MOUNT_CVMFS_CONFIG_CERN_CH=1
363+
fi
364+
288365
# 1. check if argument values are valid
289366
# (arg -a|--access) check if ACCESS is supported
290367
# use the value as global setting, suffix to --repository can specify an access mode per repository
@@ -300,10 +377,18 @@ fi
300377
# HOST_STORAGE_ERROR_EXITCODE
301378

302379
# (arg -m|--mode) check if MODE is known
303-
if [[ "${MODE}" != "shell" && "${MODE}" != "run" ]]; then
380+
if [[ "${MODE}" != "shell" && "${MODE}" != "exec" && "${MODE}" != "run" ]]; then
304381
fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}"
305382
fi
306383

384+
# the run mode should actually call "apptainer exec", so simply override run to exec
385+
if [[ "${MODE}" == "run" ]]; then
386+
echo_yellow "Note: the behaviour of the run mode has changed."
387+
echo_yellow "Previously, it mistakenly ran 'apptainer/singularity run', but it now runs 'apptainer/singularity exec' instead."
388+
echo_yellow "You can silence this message by using --mode exec instead of --mode run."
389+
MODE="exec"
390+
fi
391+
307392
# Also validate the NVIDIA GPU mode (if present)
308393
if [[ ${SETUP_NVIDIA} -eq 1 ]]; then
309394
if [[ "${NVIDIA_MODE}" != "run" && "${NVIDIA_MODE}" != "install" && "${NVIDIA_MODE}" != "all" ]]; then
@@ -366,8 +451,8 @@ done
366451
# TODO (arg -y|--https-proxy) check if https proxy is accessible
367452
# HTTPS_PROXY_ERROR_EXITCODE
368453

369-
# check if a script is provided if mode is 'run'
370-
if [[ "${MODE}" == "run" ]]; then
454+
# check if a script is provided if mode is 'exec'
455+
if [[ "${MODE}" == "exec" ]]; then
371456
if [[ $# -eq 0 ]]; then
372457
fatal_error "no command specified to run?!" "${RUN_SCRIPT_MISSING_EXITCODE}"
373458
fi
@@ -719,17 +804,38 @@ if [[ ! -z ${http_proxy} ]]; then
719804
HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST})
720805
[[ ${VERBOSE} -eq 1 ]] && echo "HTTP_PROXY_IPV4='${HTTP_PROXY_IPV4}'"
721806
echo "CVMFS_HTTP_PROXY=\"${http_proxy}|http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \
722-
>> ${EESSI_TMPDIR}/repos_cfg/default.local
807+
>> ${EESSI_TMPDIR}/repos_cfg/default.local
723808
[[ ${VERBOSE} -eq 1 ]] && echo "contents of default.local"
724809
[[ ${VERBOSE} -eq 1 ]] && cat ${EESSI_TMPDIR}/repos_cfg/default.local
725810

726811
# if default.local is not BIND mounted into container, add it to BIND_PATHS
727812
src=${EESSI_TMPDIR}/repos_cfg/default.local
728813
target=/etc/cvmfs/default.local
729-
if [[ ${BIND_PATHS} =~ "${target}" ]]; then
730-
fatal_error "BIND target in '${src}:${target}' is already in paths to be bind mounted into the container ('${BIND_PATHS}')" ${REPOSITORY_ERROR_EXITCODE}
731-
fi
732-
BIND_PATHS="${BIND_PATHS},${src}:${target}"
814+
815+
# check if target already exists in BIND_PATHS, and, if so, if sources are
816+
# the same
817+
conflict_src=$(check_bind_paths_for_target "${target}" "${src}" "${BIND_PATHS}")
818+
check_result=$?
819+
820+
case ${check_result} in
821+
0)
822+
# target already bound with same source - no action needed
823+
[[ ${VERBOSE} -eq 1 ]] && echo "Bind mount already configured: ${src}:${target}"
824+
;;
825+
1)
826+
# target already bound with different source - conflict!
827+
fatal_error "BIND target '${target}' conflict: already bound from '${conflict_src}', cannot bind from '${src}'" ${REPOSITORY_ERROR_EXITCODE}
828+
;;
829+
2)
830+
# target not found - safe to add
831+
if [[ -z ${BIND_PATH} ]]; then
832+
BIND_PATHS="${src}:${target}"
833+
else
834+
BIND_PATHS="${BIND_PATHS},${src}:${target}"
835+
fi
836+
[[ ${VERBOSE} -eq 1 ]] && echo "Added bind mount: ${src}:${target}"
837+
;;
838+
esac
733839
fi
734840

735841
# 4. set up vars and dirs specific to a scenario

0 commit comments

Comments
 (0)