Improve workflow and documentation

Author: Tom-Notch

.gitignore

@@ -354,3 +354,6 @@ AMENT_IGNORE
 .ionide
 
 # End of https://www.toptal.com/developers/gitignore/api/visualstudiocode,c++,python,jupyternotebooks,ros,ros2,matlab,git
+
+# Singularity images
+*.sif

README.md

@@ -13,7 +13,7 @@ TLDR: Search for `todo` and update all occurrences to your desired name
 
 Docker and singularity is not a must unless you cannot install some dependencies locally on HPC shell environment due to permission issue
 
-### Base Repo
+### Base Repository
 
 1. Change [LICENSE](LICENSE) if necessary
 
@@ -23,6 +23,8 @@ Docker and singularity is not a must unless you cannot install some dependencies
 
 ### Docker Config
 
+Continue on a machine where you have docker permission, HPC clusters usually restrict docker access for security reasons
+
 1. Modify `todo-docker-user`, `todo-image-name`, `todo-image-user` in [.env](.env)
 
    - [.env](env) will be loaded when you use docker compose for build/run/push/...
@@ -38,11 +40,10 @@ Docker and singularity is not a must unless you cannot install some dependencies
 
 1. [build_docker_image.sh](scripts/build_docker_image.sh) to build and test the image locally in your machine's architecture
 
-   - Do this on a machine where you have docker permission, HPC clusters usually restrict docker access for security reasons
    - The scripts uses buildx to build multi-arch image, you can disable this by removing redundant archs in [docker-compose.yml](docker-compose.yml)
    - Building stage does not have GPU access, if some of your dependencies need GPU, build them inside a running container and commit to the final image
 
-1. To run and test a built image, use [run_docker_container.sh](scripts/run_docker_container.sh) or `docker compose up -d`
+1. [run_docker_container.sh](scripts/run_docker_container.sh) or `docker compose up -d` to run and test a built image
 
    - The service by default will mount the whole repository onto `CODE_FOLDER` inside the container so any modification inside also takes effect outside, which is useful when you use vscode remote extension to develop inside a running container with remote docker context
 
@@ -52,25 +53,31 @@ Docker and singularity is not a must unless you cannot install some dependencies
 
 ### Singularity Config
 
+Continue on the actual HPC cluster environment
+
 1. [pull_singularity_image.sh](scripts/pull_singularity_image.sh) to build the singularity image locally
 
    - Singularity image can be built upon existing docker image
+   - You should see the image `todo-image-name_latest.def` after successfully built
 
 1. [run_singularity_instance.sh](scripts/run_singularity_instance.sh) to test the image
 
    - Add additional volume binding options to the script such as dataset directories, best practice is to define in [.env](.env) then export in [variables.sh](scripts/variables.sh) with `resolve_host_path` to turn relative path into absolute real path
-   - Singularity instances by default has less environment isolation than docker containers unless you specify the additional options like the script
+   - Singularity instances by default have less environment isolation than docker containers unless you specify the additional options like the script
 
 ### Job Config
 
-1. Modify job specifications under [jobs/](jobs/)
+1. Modify job specifications under `jobs/`
 
    - Each (HPC) Slurm environment has different partition definitions, which are often heterogeneous, you can query this by `sinfo` with some options
-   - All the jobs has `-l`(login) options in shebang so that any command working in your current shell environment should also run as a job
+   - `--ntasks-per-node` specifies number of parallelization, and it's convenient to tie other resources to task, e.g., `--gpus-per-task`, `--cpus-per-task`, `--mem-per-gpu`, so that you only need to increase ntasks to scale up on a node
+   - All the jobs have `-l`(login) options in shebang so that any command working in your current shell environment should also run as a job
+
+1. `sbatch jobs/your-cluster/your-job.job` or `jobs/your-cluster/your-job.job` to submit jobs
 
-1. Submit job by `sbatch jobs/your-cluster/your-job.job` or `jobs/your-cluster/your-job.job`
+   - You should see a file `todo_your_job_name_slurm_job_id.out` in the base folder of this repository, which contains job logs
 
-1. Recommend [turm](https://github.com/kabouzeid/turm) for job monitor, use `turm -u your-slurm-user` after installation
+1. Recommend [turm](https://github.com/kabouzeid/turm) for job monitor outside the job, use `turm -u your-slurm-user` after installation
 
 ## Developer Quick Start
 

docker/build-context/home-folder-config/.p10k.zsh

@@ -473,7 +473,7 @@
   typeset -g POWERLEVEL9K_VCS_MAX_INDEX_SIZE_DIRTY=-1
 
   # Don't show Git status in prompt for repositories whose workdir matches this pattern.
-  # For example, if set to '~', the Git repository at $HOME/.git will be ignored.
+  # For example, if set to '~', the Git repository at ${HOME}/.git will be ignored.
   # Multiple patterns can be combined with '|': '~(|/foo)|/bar/baz/*'.
   typeset -g POWERLEVEL9K_VCS_DISABLED_WORKDIR_PATTERN='~'
 

docker/latest/Dockerfile

@@ -1,14 +1,14 @@
 # Do not add --platform=linux/blabla since this is intended for multiplatform builds
 FROM nvidia/cuda:13.0.0-cudnn-devel-ubuntu24.04
-ENV HOME=$HOME_FOLDER
-WORKDIR $HOME_FOLDER/
+ENV HOME=${HOME_FOLDER}
+WORKDIR ${HOME_FOLDER}/
 
 # Fix apt install stuck problem
 ENV DEBIAN_FRONTEND=noninteractive
 RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
 
 # Copy home folder config
-COPY --from=home-folder-config . $HOME_FOLDER/
+COPY --from=home-folder-config . ${HOME_FOLDER}/
 
 # Remove cuda source list
 # E: Failed to fetch https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/Packages.gz File has unexpected size (853674 != 833365). Mirror sync in progress? [IP: 23.58.157.10 443]
@@ -68,7 +68,7 @@ RUN wget https://apt-get.kitware.com/kitware-archive.sh -O- | sh -s && \
 
 # # Add a new group and user
 # RUN addgroup --gid 1000 $USER && \
-#     adduser --uid 1000 --ingroup $USER --home $HOME_FOLDER --shell /bin/zsh --disabled-password --gecos "" $USER && \
+#     adduser --uid 1000 --ingroup $USER --home ${HOME_FOLDER} --shell /bin/zsh --disabled-password --gecos "" $USER && \
 #     echo "$USER ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
 
 # # Fix UID/GID when mounting from host using this: https://github.com/boxboat/fixuid
@@ -84,12 +84,12 @@ RUN sh -c "$(wget -O- https://github.com/deluan/zsh-in-docker/releases/latest/do
     -p https://github.com/zsh-users/zsh-autosuggestions \
     -p https://github.com/zsh-users/zsh-completions \
     -p https://github.com/zsh-users/zsh-syntax-highlighting \
-    -a "[[ ! -f $HOME_FOLDER/.p10k.zsh ]] || source $HOME_FOLDER/.p10k.zsh" \
+    -a "[[ ! -f ${HOME_FOLDER}/.p10k.zsh ]] || source ${HOME_FOLDER}/.p10k.zsh" \
     -a "POWERLEVEL9K_DISABLE_GITSTATUS=true" \
     -a "bindkey -M emacs '^[[3;5~' kill-word" \
     -a "bindkey '^H' backward-kill-word" \
     -a "autoload -U compinit && compinit" \
-    -a "export PATH=~/.local/bin:$PATH"
+    -a "export PATH=~/.local/bin:${PATH}"
 
 # change default shell for the $USER in the image building process for extra environment safety
 RUN chsh -s $(which zsh)
@@ -103,7 +103,7 @@ RUN wget "https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-$(uname -m
     rm ~/miniconda.sh
 
 # Add conda to PATH
-ENV PATH=$PATH:/opt/conda/bin
+ENV PATH=${PATH}:/opt/conda/bin
 
 # Init conda for zsh
 RUN conda init zsh
@@ -113,12 +113,12 @@ RUN conda init zsh
 # ENV OPENCV_VERSION=4.2.0
 # RUN pip3 uninstall -y opencv && \
 #     apt install -y --no-install-recommends libavcodec-dev libavformat-dev libswscale-dev libgstreamer-plugins-base1.0-dev libgstreamer1.0-dev libgtk-3-dev libpng-dev libjpeg-dev && \
-#     git clone --depth 1 --recursive https://github.com/opencv/opencv.git $HOME_FOLDER/opencv -b $OPENCV_VERSION && \
+#     git clone --depth 1 --recursive https://github.com/opencv/opencv.git ${HOME_FOLDER}/opencv -b $OPENCV_VERSION && \
 #     #  follow the error information，replace all the “ipcp-unit-growth” with “ipa-cp-unit-growth” in 3rdparty/carotene/CMakeLists.txt and 3rdparty/carotene/hal/CMakeLists.txt
-#     perl -pi -e 's/ipcp-unit-growth/ipa-cp-unit-growth/g' $HOME_FOLDER/opencv/3rdparty/carotene/CMakeLists.txt $HOME_FOLDER/opencv/3rdparty/carotene/hal/CMakeLists.txt && \
-#     git clone --depth 1 --recursive https://github.com/opencv/opencv_contrib.git $HOME_FOLDER/opencv_contrib -b $OPENCV_VERSION && \
-#     mkdir -p $HOME_FOLDER/opencv/build && \
-#     cd $HOME_FOLDER/opencv/build && \
+#     perl -pi -e 's/ipcp-unit-growth/ipa-cp-unit-growth/g' ${HOME_FOLDER}/opencv/3rdparty/carotene/CMakeLists.txt ${HOME_FOLDER}/opencv/3rdparty/carotene/hal/CMakeLists.txt && \
+#     git clone --depth 1 --recursive https://github.com/opencv/opencv_contrib.git ${HOME_FOLDER}/opencv_contrib -b $OPENCV_VERSION && \
+#     mkdir -p ${HOME_FOLDER}/opencv/build && \
+#     cd ${HOME_FOLDER}/opencv/build && \
 #     cmake \
 #     -D CMAKE_CXX_STANDARD=20 \
 #     -D EIGEN_INCLUDE_PATH=/usr/include/eigen3 \
@@ -142,8 +142,8 @@ RUN conda init zsh
 #     make install -j$(($(nproc)-1)) && \
 #     echo "export OpenCV_DIR=/usr/local/lib/cmake/opencv4/" >> ${HOME_FOLDER}/.zshrc && \
 #     echo "export OpenCV_DIR=/usr/local/lib/cmake/opencv4/" >> ${HOME_FOLDER}/.bashrc && \
-#     rm -rf $HOME_FOLDER/opencv && \
-#     rm -rf $HOME_FOLDER/opencv_contrib
+#     rm -rf ${HOME_FOLDER}/opencv && \
+#     rm -rf ${HOME_FOLDER}/opencv_contrib
 
 # Add git safe directory
 RUN git config --global --add safe.directory "*"
@@ -156,16 +156,16 @@ RUN apt full-upgrade -y && \
     rm -rf /var/lib/apt/lists/*
 
 # change owner of home folder
-RUN chown -R $IMAGE_USER:$IMAGE_USER $HOME_FOLDER
+RUN chown -R ${IMAGE_USER}:${IMAGE_USER} ${HOME_FOLDER}
 
 # change user
-USER $IMAGE_USER
+USER ${IMAGE_USER}
 
 # Set the default shell to zsh
 SHELL [ "/bin/zsh", "-c" ]
 
 # # move fixuid config
-# RUN mv $HOME_FOLDER/fixuid-config.yml /etc/fixuid/config.yml
+# RUN mv ${HOME_FOLDER}/fixuid-config.yml /etc/fixuid/config.yml
 
 # Entrypoint command
 # ENTRYPOINT [ "/bin/sh" , "-c", "fixuid; /bin/zsh" ]

jobs/psc/gpu-singularity.job

@@ -17,4 +17,5 @@ set -x
 
 cd todo-your-code-directory || exit # this directory refers to the absolute path outside any docker/singularity container/instance
 scripts/run_singularity_instance.sh
-srun singularity exec instance://todo-your-container-name zsh -c "source ~/.zshrc && conda run -n todo-your-conda-env-name todo-your-code-entrypoint"
+source scripts/variables.sh
+srun singularity exec instance://"${CONTAINER_NAME}" zsh -c "source ~/.zshrc && conda run -n todo-your-conda-env-name todo-your-code-entrypoint"

scripts/dev_setup.sh

@@ -15,14 +15,14 @@ echo "Please enter your sudo password:"
 read -rs SUDO_PASSWORD # Use -r to avoid mangling backslashes, -s for silent input
 
 # Keep sudo alive during the script execution
-echo "$SUDO_PASSWORD" | sudo -v -S
+echo "${SUDO_PASSWORD}" | sudo -v -S
 
 # Update package lists
-echo "$SUDO_PASSWORD" | sudo -S apt update
+echo "${SUDO_PASSWORD}" | sudo -S apt update
 
 # Install dependencies
 echo "Installing clang-format clang-tidy python3 python3-pip"
-echo "$SUDO_PASSWORD" | sudo -S apt install -y clang-format clang-tidy libpython3-dev python3-pip
+echo "${SUDO_PASSWORD}" | sudo -S apt install -y clang-format clang-tidy libpython3-dev python3-pip
 
 # Install/uprade pre-commit
 echo "Installing/upgrading pre-commit"
@@ -32,7 +32,7 @@ pip3 install --upgrade pre-commit
 DEFAULT_SHELL=$(basename "$SHELL")
 
 # Determine the shell configuration file
-case "$DEFAULT_SHELL" in
+case "${DEFAULT_SHELL}" in
 bash)
 	SHELL_RC="${HOME}/.bashrc"
 	;;
@@ -46,24 +46,24 @@ ksh)
 	SHELL_RC="${HOME}/.kshrc"
 	;;
 *)
-	echo "Unsupported shell: $DEFAULT_SHELL"
+	echo "Unsupported shell: ${DEFAULT_SHELL}"
 	exit 1
 	;;
 esac
 
 # Add pre-commit executable to PATH if not already present
-if ! grep -q 'export PATH=~/.local/bin:$PATH' "$SHELL_RC"; then
-	echo "Adding pre-commit (actually python3-pip packages') executable to path in $SHELL_RC"
-	echo 'export PATH=~/.local/bin:$PATH' >>"$SHELL_RC"
+if ! grep -q 'export PATH=~/.local/bin:${PATH}' "${SHELL_RC}"; then
+	echo "Adding pre-commit (actually python3-pip packages') executable to path in ${SHELL_RC}"
+	echo 'export PATH=~/.local/bin:${PATH}' >>"${SHELL_RC}"
 	# Source the .zshrc file using zsh
-	if [ "$DEFAULT_SHELL" = "zsh" ]; then
-		zsh -c "source $SHELL_RC"
+	if [ "${DEFAULT_SHELL}" = "zsh" ]; then
+		zsh -c "source ${SHELL_RC}"
 	else
 		# shellcheck source=/dev/null
-		. "$SHELL_RC"
+		. "${SHELL_RC}"
 	fi
 else
-	echo "PATH already updated in $SHELL_RC"
+	echo "PATH already updated in ${SHELL_RC}"
 fi
 
 # Perform pre-installation of pre-commit hooks and dry run on all files

scripts/kill_docker_container.sh

@@ -10,6 +10,6 @@
 
 . "$(dirname "$0")"/variables.sh
 
-docker exec --privileged -it "$CONTAINER_NAME" pkill -f zsh
+docker exec --privileged -it "${CONTAINER_NAME}" pkill -f zsh
 
-docker rm -f "$CONTAINER_NAME"
+docker rm -f "${CONTAINER_NAME}"

scripts/kill_singularity_instance.sh

@@ -11,4 +11,4 @@
 set -euo pipefail
 . "$(dirname "$0")"/variables.sh
 
-singularity instance stop "$CONTAINER_NAME"
+singularity instance stop "${CONTAINER_NAME}"

scripts/pull_docker_image.sh

@@ -10,4 +10,4 @@
 
 . "$(dirname "$0")"/variables.sh
 
-docker pull "$DOCKER_USER"/"$IMAGE_NAME":"$IMAGE_TAG"
+docker pull "${DOCKER_USER}"/"${IMAGE_NAME}":"${IMAGE_TAG}"

scripts/push_docker_image.sh

@@ -56,6 +56,6 @@ echo "----------------------------------------"
 cat "${tmp_dockerfile}"
 echo "----------------------------------------"
 
-# docker push "$DOCKER_USER"/"$IMAGE_NAME":"$IMAGE_TAG"
+# docker push "${DOCKER_USER}"/"${IMAGE_NAME}":"${IMAGE_TAG}"
 
 docker buildx bake --file "$(dirname "$0")"/../docker-compose.yml --push --set "*.dockerfile=${tmp_dockerfile}"