diff --git a/internal/remote/firecrest/controller.go b/internal/remote/firecrest/controller.go index e198856b..9cb029af 100644 --- a/internal/remote/firecrest/controller.go +++ b/internal/remote/firecrest/controller.go @@ -648,11 +648,11 @@ func addSessionMountsToScript(sessionScript string, fileSystems *[]FileSystem, s mounts = append(mounts, fmt.Sprintf("%s:/secrets:ro", secretsPath)) } - // Format mount list for the environment.toml file + // Format mount list for i := range mounts { - mounts[i] = fmt.Sprintf(" \"%s\",", mounts[i]) + mounts[i] = fmt.Sprintf("\"%s\"", mounts[i]) } - mountsStr := fmt.Sprintf("mounts = [\n%s\n]", strings.Join(mounts, "\n")) + mountsStr := fmt.Sprintf("--container-mounts=%s", strings.Join(mounts, ",")) return strings.Replace(sessionScript, "#{{SESSION_MOUNTS_PLACEHOLDER}}", mountsStr, 1) } diff --git a/internal/remote/firecrest/controller_test.go b/internal/remote/firecrest/controller_test.go index 7c5583eb..6b5e0106 100644 --- a/internal/remote/firecrest/controller_test.go +++ b/internal/remote/firecrest/controller_test.go @@ -70,7 +70,13 @@ func TestRenderSessionScriptStatic(t *testing.T) { assert.Contains(t, sessionScriptFinal, "#SBATCH --partition=my-partition") // Check the mounts - mountsRegExp := regexp.MustCompile(`mounts(?:\s*)=(?:\s*)[[]([^]]*)]`) + // From `srun --help`: + // --container-mounts=SRC:DST[:FLAGS][,SRC:DST...] + // [pyxis] bind mount[s] inside the container. Mount + // flags are separated with "+", e.g. "ro+rprivate" + flagsRegExp := `:(?:ro|rprivate)(?:[+](?:ro|rprivate))*` + mountRegExp := `"[^:,"]+:[^:,"]+(?:` + flagsRegExp + `)?"` + mountsRegExp := regexp.MustCompile(`--container-mounts=(` + mountRegExp + `(?:,` + mountRegExp + `)*)`) matches := mountsRegExp.FindStringSubmatch(sessionScriptFinal) assert.Len(t, matches, 2) foundMounts := matches[1] diff --git a/internal/remote/firecrest/session_script.sh b/internal/remote/firecrest/session_script.sh index 5d56856b..53da0808 100644 --- a/internal/remote/firecrest/session_script.sh +++ b/internal/remote/firecrest/session_script.sh @@ -9,11 +9,16 @@ set -e -o pipefail +: ${REMOTE_SESSION_IMAGE:?'not set, aborting!'} + : ${ARCH:=$(uname -m)} : ${RENKU_PKG:="${HOME}/.renku/${ARCH}/pkg"} +: ${GIT_PROXY_PORT:=65480} +: ${GIT_PROXY_HEALTH_PORT:=65481} : ${GIT_PROXY_WAIT_SLEEP_SECONDS:=10} : ${GIT_PROXY_WAIT_RETRIES:=10} : ${RCLONE_VERSION:="1.70.2"} +: ${WSTUNNEL_PATH_PREFIX:="sessions/my-session/wstunnel"} : ${WSTUNNEL_VERSION:="10.5.5"} case ${ARCH} in @@ -29,10 +34,15 @@ case ${ARCH} in ;; esac +: ${SESSION_DIR:="${PWD}"} +: ${SESSION_WORK_DIR:="${SESSION_DIR}/work"} +: ${SECRETS_DIR:="${SESSION_DIR}/secrets"} +: ${LOGS_DIR:="${SESSION_DIR}/logs"} + # Installs rclone # # Usage: -# rclone="$(install_rclone $version $gh_arch)" +# rclone="$(install_rclone "$version" "$gh_arch")" # "$rclone" version function install_rclone() { rclone_version=${1:?"install_rclone: Version missing"} @@ -56,12 +66,13 @@ function install_rclone() { mkdir -p "${rclone_pkg}" tmp="$(mktemp -d)" - cwd="$(pwd)" - cd "${tmp}" - curl -Lo "rclone.zip" "${rclone_url}" - >&2 unzip "rclone.zip" - rm -r "${rclone_pkg}" - mv ./rclone-v"${rclone_version}"-* "${rclone_pkg}" + (# Run in a subshell to prevent changing the working directory of the caller + cd "${tmp}" + curl -Lo "rclone.zip" "${rclone_url}" + >&2 unzip "rclone.zip" + rm -rf "${rclone_pkg}" + mv ./rclone-v"${rclone_version}"-* "${rclone_pkg}" + ) rm -r "${tmp}" chmod a+x "${rclone_bin}" @@ -71,7 +82,7 @@ function install_rclone() { # Installs wstunnel # # Usage: -# wstunnel="$(install_wstunnel $version $gh_arch)" +# wstunnel="$(install_wstunnel "$version" "$gh_arch")" # "$wstunnel" --version function install_wstunnel() { wstunnel_version=${1:?"wstunnel_version: Version missing"} @@ -96,32 +107,24 @@ function install_wstunnel() { mkdir -p "${wstunnel_pkg}" tmp="$(mktemp -d)" - cwd="$(pwd)" - cd "${tmp}" - curl -Lo "wstunnel.tar.gz" "${wstunnel_url}" - tar xf "wstunnel.tar.gz" -C "${wstunnel_pkg}" - cd "${cwd}" + (# Run in a sub shell to prevent changing the working directory of the caller + cd "${tmp}" + curl -Lo "wstunnel.tar.gz" "${wstunnel_url}" + rm -rf "${wstunnel_pkg}" + mkdir -p ${wstunnel_pkg} # the folder has to exist for tar -C + tar xf "wstunnel.tar.gz" -C "${wstunnel_pkg}" + ) rm -r "${tmp}" chmod a+x "${wstunnel_bin}" echo "${wstunnel_bin}" } -if [ -z "${REMOTE_SESSION_IMAGE}" ]; then - echo "REMOTE_SESSION_IMAGE is not set, aborting!" - exit 1 -fi - -SESSION_DIR="$(pwd)" -SESSION_WORK_DIR="${SESSION_DIR}/work" -SECRETS_DIR="${SESSION_DIR}/secrets" -LOGS_DIR="${SESSION_DIR}/logs" -echo "SESSION_DIR: ${SESSION_DIR}" -echo "SESSION_WORK_DIR: ${SESSION_WORK_DIR}" - -mkdir -p "${SESSION_WORK_DIR}" -mkdir -p "${SECRETS_DIR}" -mkdir -p "${LOGS_DIR}" +for d in SESSION_WORK_DIR SECRETS_DIR LOGS_DIR +do + echo "${d}: ${!d}" + mkdir -p "${!d}" +done # # Install rclone # rclone="$(install_rclone "${RCLONE_VERSION}" "${gh_arch}")" @@ -131,7 +134,7 @@ mkdir -p "${LOGS_DIR}" wstunnel="$(install_wstunnel "${WSTUNNEL_VERSION}" "${gh_arch}")" echo "wstunnel: ${wstunnel}" -# Ensure NVIDIA_VISIBLE_DEVICES is set to void +# Ensure NVIDIA_VISIBLE_DEVICES is set to void # so that cuda enabled images work on eiger if !(nvidia-smi 2>&1 >/dev/null); then export NVIDIA_VISIBLE_DEVICES=void @@ -140,16 +143,14 @@ fi # Create the environment.toml file to run the session EDF_FILE="${SESSION_DIR}/environment.toml" cat <"${EDF_FILE}" -image = "${REMOTE_SESSION_IMAGE}" - -#{{SESSION_MOUNTS_PLACEHOLDER}} - -workdir = "${SESSION_WORK_DIR}" - [annotations] com.hooks.cxi.enabled = "false" EOF +srun_param_container_image="--container-image ${REMOTE_SESSION_IMAGE}" +srun_param_workdir="--container-workdir ${SESSION_WORK_DIR}" +srun_param_mounts=#{{SESSION_MOUNTS_PLACEHOLDER}} + export RENKU_MOUNT_DIR="${SESSION_WORK_DIR}" export RENKU_WORKING_DIR="${SESSION_WORK_DIR}" # Force the frontend to listen on 127.0.0.1 @@ -173,9 +174,6 @@ echo "TODO: setup rclone mounts..." # "${rclone}" mount --config "${RCLONE_CONFIG}" --daemon --read-only era5: "${SESSION_WORK_DIR}/era5" # echo "Starting tunnel..." -GIT_PROXY_PORT="${GIT_PROXY_PORT:-65480}" -GIT_PROXY_HEALTH_PORT="${GIT_PROXY_HEALTH_PORT:-65481}" -WSTUNNEL_PATH_PREFIX="${WSTUNNEL_PATH_PREFIX:-sessions/my-session/wstunnel}" echo "wstunnel client \ -R tcp://0.0.0.0:${RENKU_SESSION_PORT}:localhost:${RENKU_SESSION_PORT} \ -L tcp://${GIT_PROXY_PORT}:localhost:${GIT_PROXY_PORT} \ @@ -249,6 +247,12 @@ echo "Starting session..." # Start session while listening to EXIT signals pid= trap 'exit_script && [[ $pid ]] && kill -TERM "$pid" && exit_script' EXIT -srun --environment "${EDF_FILE}" --no-container-entrypoint sh /etc/rc & pid=$! +srun \ + --environment "${EDF_FILE}" \ + ${srun_param_container_image} \ + ${srun_param_workdir} \ + ${srun_param_mounts} \ + --no-container-entrypoint sh /etc/rc \ + & pid=$! wait pid=