From dcaf053598c34ab9ef52b648573109023299692c Mon Sep 17 00:00:00 2001 From: Calvin Pieters Date: Mon, 28 Jul 2025 13:19:17 +0300 Subject: [PATCH 1/5] Refactors Dockerfile for multi-stage build Migrates to a multi-stage Dockerfile to reduce image size by separating the build environment from the runtime environment. This approach optimizes the final image by only including necessary components, improving efficiency and security. Includes ARC installation by copying from a prior build stage. Also includes adding aliases and entrypoint, and sets permissions. --- Dockerfile | 215 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 143 insertions(+), 72 deletions(-) diff --git a/Dockerfile b/Dockerfile index 143d083b13..643470ad58 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,7 +46,7 @@ RUN apt-get update && apt-get install -y \ && apt-get clean \ && apt-get autoclean \ && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/*\ + && rm -rf /var/lib/apt/lists/* \ && echo "${NEW_MAMBA_USER} ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers # Change user to the non-root user USER $MAMBA_USER @@ -57,68 +57,52 @@ RUN mkdir -p /home/rmguser/Code # Change working directory to Code WORKDIR /home/rmguser/Code -# Clone the RMG base and database repositories. The pulled branches are only the main branches. -RUN git clone -b main https://github.com/ReactionMechanismGenerator/RMG-Py.git \ - && git clone -b main https://github.com/ReactionMechanismGenerator/RMG-database.git +# ------------------------------------------------------------------ clone & checkout +WORKDIR /home/rmguser/Code +RUN git clone --filter=blob:none --no-checkout https://github.com/ReactionMechanismGenerator/RMG-Py.git RMG-Py \ + && git -C RMG-Py checkout --detach 55464c54d1fa61b531e865682df598d33718597d \ + && git clone --filter=blob:none --depth 1 https://github.com/ReactionMechanismGenerator/RMG-database.git RMG-database -# cd into RMG-Py -WORKDIR /home/rmguser/Code/RMG-Py -# Install RMG-Py and then clean up the micromamba cache -RUN micromamba create -y -f environment.yml && \ - micromamba install -n rmg_env -c conda-forge conda && \ - micromamba clean --all -f -y -# Activate the RMG environment -ARG MAMBA_DOCKERFILE_ACTIVATE=1 -ENV ENV_NAME=rmg_env -# Set environment variables -# These need to be set in the Dockerfile so that they are available to the build process -ENV PATH /opt/conda/envs/rmg_env/bin:$PATH -ENV PYTHONPATH /home/rmguser/Code/RMG-Py:$PYTHONPATH -ENV PATH /home/rmguser/Code/RMG-Py:$PATH +ENV PATH=/opt/conda/envs/rmg_env/bin:/home/rmguser/Code/RMG-Py:$PATH \ + PYTHONPATH=/home/rmguser/Code/RMG-Py -# Build RMG -RUN make \ - && echo "export PYTHONPATH=/home/rmguser/Code/RMG-Py" >> ~/.bashrc \ - && echo "export PATH=/home/rmguser/Code/RMG-Py:$PATH" >> ~/.bashrc ENV JULIA_DEPOT_PATH="/home/rmguser/julia-ver/packages" -ENV JULIA_HISTORY /home/rmguser/repl_history.jl +ENV JULIA_HISTORY=/home/rmguser/repl_history.jl # Since 1.9.0 Julia, the CPU target is set to "native" by default. This is not ideal for a Docker image, so we set it to a list of common CPU targets # This avoids the need to compile the Julia packages for the specific CPU architecture of the host machine ENV JULIA_CPU_TARGET="x86-64,haswell,skylake,broadwell,znver1,znver2,znver3,cascadelake,icelake-client,cooperlake,generic,native" - # Install RMS # The extra arguments are required to install PyCall and RMS in this Dockerfile. Will not work without them. # Final command is to compile the RMS during Docker build - This will reduce the time it takes to run RMS for the first time -RUN touch /opt/conda/envs/rmg_env/condarc-julia.yml -RUN CONDA_JL_CONDA_EXE=/bin/micromamba julia -e 'ENV["CONDA_JL_CONDA_EXE"]="/opt/conda/envs/rmg_env/bin/conda";using Pkg;Pkg.add(PackageSpec(name="PyCall", rev="master")); Pkg.build("PyCall"); Pkg.add(PackageSpec(name="ReactionMechanismSimulator", rev="main")), using ReactionMechanismSimulator' \ - && python -c "import julia; julia.install(); import diffeqpy; diffeqpy.install()" \ - && python-jl -c "from pyrms import rms" - -RUN python-jl /home/rmguser/Code/RMG-Py/rmg.py /home/rmguser/Code/RMG-Py/examples/rmg/minimal/input.py \ - # delete the results, preserve input.py - && mv /home/rmguser/Code/RMG-Py/examples/rmg/minimal/input.py /home/rmguser/Code/RMG-Py/examples/input.py \ - && rm -rf /home/rmguser/Code/RMG-Py/examples/rmg/minimal/* \ - && mv /home/rmguser/Code/RMG-Py/examples/input.py /home/rmguser/Code/RMG-Py/examples/rmg/minimal/input.py - -# Add alias to bashrc - rmge to activate the environment -# These commands are not necessary for the Docker image to run, but they are useful for the user -RUN echo "alias rmge='micromamba activate rmg_env'" >> ~/.bashrc \ - && echo "alias arce='micromamba activate arc_env'" >> ~/.bashrc \ - && echo "alias rmg='python-jl /home/rmguser/Code/RMG-Py/rmg.py input.py'" >> ~/.bashrc \ - && echo "alias deact='micromamba deactivate'" >> ~/.bashrc \ - && echo "export rmgpy_path='/home/rmguser/Code/RMG-Py/'" >> ~/.bashrc \ - && echo "export rmgdb_path='/home/rmguser/Code/RMG-database/'" >> ~/.bashrc \ - && echo "alias rmgcode='cd \$rmgpy_path'" >> ~/.bashrc \ - && echo "alias rmgdb='cd \$rmgdb_path'" >> ~/.bashrc \ - && echo "alias arcode='cd /home/rmguser/Code/ARC'" >> ~/.bashrc \ - && echo "alias conda='micromamba'" >> ~/.bashrc \ - && echo "alias mamba='micromamba'" >> ~/.bashrc \ - && echo "export PYTHONPATH=$PYTHONPATH:/home/rmguser/Code/ARC" >> ~/.bashrc \ - && echo "export PATH=$PATH:/home/rmguser/Code/ARC" >> ~/.bashrc +# Julia + PyCall + RMS in rmg_env +ENV CONDA_JL_CONDA_EXE=/opt/conda/envs/rmg_env/bin/conda +# ------------------------------------------------------------------ env create +WORKDIR /home/rmguser/Code/RMG-Py +RUN micromamba create -y -n rmg_env -f environment.yml \ + && touch /opt/conda/envs/rmg_env/condarc-julia.yml \ + && micromamba install -y -n rmg_env -c conda-forge conda \ + && micromamba clean -a -y \ + && echo "export PYTHONPATH=/home/rmguser/Code/RMG-Py" >> ~/.bashrc \ + && echo "export PATH=/home/rmguser/Code/RMG-Py:$PATH" >> ~/.bashrc \ + && micromamba run -n rmg_env make -j"$(nproc)" \ + && micromamba run -n rmg_env bash -lc "\ + julia -e 'ENV[\"CONDA_JL_CONDA_EXE\"]=\"${CONDA_JL_CONDA_EXE}\"; using Pkg; \ + Pkg.add(PackageSpec(name=\"PyCall\", rev=\"master\")); Pkg.build(\"PyCall\"); \ + Pkg.add(PackageSpec(name=\"ReactionMechanismSimulator\", url=\"https://github.com/ReactionMechanismGenerator/ReactionMechanismSimulator.jl\", rev=\"8dbb07eedebaacf9b9f77081623c572d054b9a6f\")); \ + using ReactionMechanismSimulator'; \ + python -c 'import julia, diffeqpy; julia.install(); diffeqpy.install()'; \ + python-jl -c 'from pyrms import rms' \ + " + +RUN micromamba run -n rmg_env python-jl /home/rmguser/Code/RMG-Py/rmg.py /home/rmguser/Code/RMG-Py/examples/rmg/minimal/input.py \ +# delete the results, preserve input.py +&& mv /home/rmguser/Code/RMG-Py/examples/rmg/minimal/input.py /home/rmguser/Code/RMG-Py/examples/input.py \ +&& rm -rf /home/rmguser/Code/RMG-Py/examples/rmg/minimal/* \ +&& mv /home/rmguser/Code/RMG-Py/examples/input.py /home/rmguser/Code/RMG-Py/examples/rmg/minimal/input.py # Installing ARC # Change directory to Code @@ -132,23 +116,100 @@ WORKDIR /home/rmguser/Code/ARC ENV PYTHONPATH="${PYTHONPATH}:/home/rmguser/Code/ARC" ENV PYTHONPATH="${PYTHONPATH}:/home/rmguser/Code/AutoTST" ENV PYTHONPATH="${PYTHONPATH}:/home/rmguser/Code/TS-GCN" -ENV PATH /home/rmguser/Code/ARC:$PATH +ENV PATH=/home/rmguser/Code/ARC:$PATH # Install ARC Environment -RUN micromamba create -y -f environment.yml && \ +COPY --chown=rmguser:rmguser ./environment.yml /home/rmguser/Code/ARC/environment.yml +RUN micromamba create -y -n arc_env -f environment.yml --channel-priority flexible && \ micromamba clean --all -f -y && \ - rm -rf /home/rmguser/.cache/yarn \ - rm -rf /home/rmguser/.cache/pip &&\ - rm -rf /home/rmguser/.cache/pip && \ - find -name '*.a' -delete && \ - find -name '*.pyc' -delete && \ - find -name '__pycache__' -type d -exec rm -rf '{}' '+' && \ - find /opt/conda/envs/arc_env/lib/python3.7/site-packages/scipy -name 'tests' -type d -exec rm -rf '{}' '+' && \ - find /opt/conda/envs/arc_env/lib/python3.7/site-packages/numpy -name 'tests' -type d -exec rm -rf '{}' '+' && \ - find /opt/conda/envs/arc_env/lib/python3.7/site-packages/pandas -name 'tests' -type d -exec rm -rf '{}' '+' && \ - find /opt/conda/envs/arc_env/lib/python3.7/site-packages -name '*.pyx' -delete && \ - rm -rf /opt/conda/envs/arc_env/lib/python3.7/site-packages/uvloop/loop.c && \ - make clean +bash -euxo pipefail <<'EOF' +PYDIR=$(echo /opt/conda/envs/arc_env/lib/python*/site-packages) + +# cache +rm -rf /home/rmguser/.cache/pip /home/rmguser/.cache/yarn + +# strip compiled cruft +find "$PYDIR" -type f \( -name "*.a" -o -name "*.py[co]" \) -delete +find "$PYDIR" -type d -name "__pycache__" -prune -exec rm -rf {} + + +# drop test directories in one traversal +find "$PYDIR" -type d \( -path "*/scipy/tests" -o -path "*/numpy/tests" -o -path "*/pandas/tests" \) \ + -prune -exec rm -rf {} + + +# remove cython sources +find "$PYDIR" -name "*.pyx" -delete +rm -f "$PYDIR/uvloop/loop.c" +EOF + +FROM --platform=linux/amd64 mambaorg/micromamba@sha256:20fb02f2d1160265f7fabaf1601707a902ae65c6dc9e053d305441182450c368 AS arc-stage +USER root +ARG NEW_MAMBA_USER=rmguser +ARG NEW_MAMBA_USER_ID=1000 +ARG NEW_MAMBA_USER_GID=1000 +RUN usermod "--login=${NEW_MAMBA_USER}" "--home=/home/${NEW_MAMBA_USER}" \ + --move-home "-u ${NEW_MAMBA_USER_ID}" "${MAMBA_USER}" && \ + groupmod "--new-name=${NEW_MAMBA_USER}" \ + "-g ${NEW_MAMBA_USER_GID}" "${MAMBA_USER}" && \ + echo "${NEW_MAMBA_USER}" > "/etc/arg_mamba_user" && \ + : + +# Set the environment variables +ARG MAMBA_ROOT_PREFIX=/opt/conda +ENV MAMBA_USER=$NEW_MAMBA_USER +ENV BASE=$MAMBA_ROOT_PREFIX + +# Install system dependencies +# +# List of deps and why they are needed: +# - make, gcc, g++ for building RMG +# - git for downloading RMG respoitories +# - wget for downloading conda install script +# - libxrender1 required by RDKit +# Clean up the apt cache to reduce the size of the image +RUN apt-get update && apt-get install -y \ + git \ + gcc \ + g++ \ + make \ + libgomp1\ + libxrender1 \ + sudo \ + nano \ + && apt-get clean \ + && apt-get autoclean \ + && apt-get autoremove -y \ + && rm -rf /var/lib/apt/lists/* \ + && echo "${NEW_MAMBA_USER} ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers \ + && printf '\n# print cheat-sheet once per interactive Bash\nif [[ $- == *i* && $SHLVL -eq 1 ]]; then\n aliases\nfi\n' \ + >> /etc/bash.bashrc +# Change user to the non-root user +USER $MAMBA_USER + +# Make directory for RMG-Py and RMG-database +RUN mkdir -p /home/rmguser/Code + +COPY --from=rmg-stage --chown=rmguser:rmguser /opt/conda /opt/conda +COPY --from=rmg-stage --chown=rmguser:rmguser /home/rmguser/Code/RMG-Py /home/rmguser/Code/RMG-Py +COPY --from=rmg-stage --chown=rmguser:rmguser /home/rmguser/Code/RMG-database /home/rmguser/Code/RMG-database +COPY --from=rmg-stage --chown=rmguser:rmguser /home/rmguser/Code/ARC /home/rmguser/Code/ARC + + +ENV PYTHONPATH="${PYTHONPATH}:/home/rmguser/Code/ARC" +ENV PYTHONPATH="${PYTHONPATH}:/home/rmguser/Code/AutoTST" +ENV PYTHONPATH="${PYTHONPATH}:/home/rmguser/Code/TS-GCN" +ENV PATH=/home/rmguser/Code/ARC:/home/rmguser/Code/RMG-Py:/home/rmguser/Code/RMG-database:$PATH +ENV PYTHONPATH=/home/rmguser/Code/ARC:/home/rmguser/Code/RMG-Py:/home/rmguser/Code/RMG-database:$PYTHONPATH + +ENV JULIA_DEPOT_PATH="/home/rmguser/julia-ver/packages" +ENV JULIA_HISTORY=/home/rmguser/repl_history.jl +# Since 1.9.0 Julia, the CPU target is set to "native" by default. This is not ideal for a Docker image, so we set it to a list of common CPU targets +# This avoids the need to compile the Julia packages for the specific CPU architecture of the host machine +ENV JULIA_CPU_TARGET="x86-64,haswell,skylake,broadwell,znver1,znver2,znver3,cascadelake,icelake-client,cooperlake,generic" +# Install RMS +# The extra arguments are required to install PyCall and RMS in this Dockerfile. Will not work without them. +# Final command is to compile the RMS during Docker build - This will reduce the time it takes to run RMS for the first time +# Julia + PyCall + RMS in rmg_env +ENV CONDA_JL_CONDA_EXE=/opt/conda/envs/rmg_env/bin/conda WORKDIR /home/rmguser/ @@ -157,15 +218,25 @@ RUN mkdir -p /home/rmguser/.arc && \ cp /home/rmguser/Code/ARC/arc/settings/submit.py /home/rmguser/.arc/submit.py # Copy alias_print.sh and entrywrapper.sh to the container -COPY --chown=rmguser:rmguser ./dockerfiles/alias_print.sh /home/rmguser/alias_print.sh -COPY --chown=rmguser:rmguser ./dockerfiles/entrywrapper.sh /home/rmguser/entrywrapper.sh - -# Make the scripts executable -RUN chmod +x /home/rmguser/alias_print.sh \ - && chmod +x /home/rmguser/entrywrapper.sh +# Copy your login‐wide aliases into /etc/profile.d +COPY --chown=rmguser:rmguser dockerfiles/aliases.sh /etc/profile.d/99-rmg-aliases.sh + +# Copy the cheat‐sheet and entrypoint +COPY --chown=rmguser:rmguser dockerfiles/aliases_print.sh /usr/local/bin/aliases +COPY --chown=rmguser:rmguser dockerfiles/entrywrapper.sh /home/rmguser/entrywrapper.sh + +# Fix permissions & make the scripts executable & ensure rms is run once +RUN chmod 644 /etc/profile.d/99-rmg-aliases.sh \ + && chmod +x /home/rmguser/entrywrapper.sh \ + && chmod +x /usr/local/bin/aliases \ + && micromamba run -n rmg_env python-jl /home/rmguser/Code/RMG-Py/rmg.py /home/rmguser/Code/RMG-Py/examples/rmg/minimal/input.py \ + # delete the results, preserve input.py + && mv /home/rmguser/Code/RMG-Py/examples/rmg/minimal/input.py /home/rmguser/Code/RMG-Py/examples/input.py \ + && rm -rf /home/rmguser/Code/RMG-Py/examples/rmg/minimal/* \ + && mv /home/rmguser/Code/RMG-Py/examples/input.py /home/rmguser/Code/RMG-Py/examples/rmg/minimal/input.py -# Set the wrapper script as the entrypoint -ENTRYPOINT ["/home/rmguser/entrywrapper.sh"] +# Use a login shell so /etc/profile and /etc/profile.d/* are sourced automatically +ENTRYPOINT ["bash","-l","/home/rmguser/entrywrapper.sh"] # Activate the ARC environment ARG MAMBA_DOCKERFILE_ACTIVATE=1 From 67712633a06878bf69f4a1aa7ff66e39081e9a59 Mon Sep 17 00:00:00 2001 From: Calvin Pieters Date: Mon, 28 Jul 2025 13:19:27 +0300 Subject: [PATCH 2/5] Refactors entrypoint and alias handling Consolidates alias definitions into a dedicated script for better organization. Replaces the previous `alias_print.sh` with a new version that is more readable and uses environment variables for paths. Simplifies the entrypoint script to handle interactive and non-interactive modes more cleanly, including direct execution of `rmg` and `arc` commands with environment activation. The removal of the CONTAINER_MODE check and the permission adjustment logic streamlines the startup process. --- dockerfiles/alias_print.sh | 34 -------------- dockerfiles/aliases.sh | 28 ++++++++++++ dockerfiles/aliases_print.sh | 31 +++++++++++++ dockerfiles/entrywrapper.sh | 87 +++++++++++++----------------------- 4 files changed, 89 insertions(+), 91 deletions(-) delete mode 100755 dockerfiles/alias_print.sh create mode 100644 dockerfiles/aliases.sh create mode 100644 dockerfiles/aliases_print.sh diff --git a/dockerfiles/alias_print.sh b/dockerfiles/alias_print.sh deleted file mode 100755 index cb51484cea..0000000000 --- a/dockerfiles/alias_print.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -# Source .bashrc to load aliases -if [ -f ~/.bashrc ]; then - source ~/.bashrc -fi - -# Print the aliases -echo "Alias List with Descriptions:" -echo "" -echo "1. rmge ='micromamba activate rmg_env'" -echo " - Activates the 'rmg_env' environment using Conda." -echo "" -echo "2. arce ='micromamba activate arc_env'" -echo " - Activates the 'arc_env' environment using Conda." -echo "" -echo "3. rmg ='python-jl /home/rmguser/Code/RMG-Py/rmg.py input.py'" -echo " - Runs the RMG program with 'input.py' using Python in the Julia environment." -echo "" -echo "4. deact ='micromamba deactivate'" -echo " - Deactivates the current Micromamba environment." -echo "" -echo "5. rmgcode='cd /home/rmguser/Code/RMG-Py/'" -echo " - Changes the current directory to the RMG-Py code directory." -echo "" -echo "6. rmgdb ='cd /home/rmguser/Code/RMG-database/" -echo " - Changes the current directory to the RMG database directory." -echo "" -echo "7. arcode ='cd /home/rmguser/Code/ARC'" -echo " - Changes the current directory to the ARC code directory." -echo "" - -# Execute the command specified as CMD in Dockerfile, or the command passed to docker run -exec "$@" diff --git a/dockerfiles/aliases.sh b/dockerfiles/aliases.sh new file mode 100644 index 0000000000..7e56be1fb7 --- /dev/null +++ b/dockerfiles/aliases.sh @@ -0,0 +1,28 @@ +# personalised shortcuts – sourced for every login shell +alias rc='source ~/.bashrc' +alias rce='nano ~/.bashrc' +alias erc='nano ~/.bashrc' + +# micromamba drop-ins +alias mamba='micromamba' +alias conda='micromamba' +alias deact='micromamba deactivate' + +# env activators +alias rmge='micromamba activate rmg_env' +alias arce='micromamba activate arc_env' + +# code roots (set once at image build) +export rmgpy_path=/home/rmguser/Code/RMG-Py +export rmgdb_path=/home/rmguser/Code/RMG-database +export arc_path=/home/rmguser/Code/ARC + +alias rmgcode='cd "$rmgpy_path"' +alias dbcode='cd "$rmgdb_path"' +alias arcode='cd "$arc_path"' + +# job wrappers +alias rmg='python-jl "$rmgpy_path/rmg.py" input.py > >(tee -a stdout.log) 2> >(tee -a stderr.log >&2)' +alias arkane='python "$rmgpy_path/Arkane.py" input.py > >(tee -a stdout.log) 2> >(tee -a stderr.log >&2)' +alias arc='python "$arc_path/ARC.py" input.yml > >(tee -a stdout.log) 2> >(tee -a stderr.log >&2)' +alias arcrestart='python "$arc_path/ARC.py" restart.yml > >(tee -a stdout.log) 2> >(tee -a stderr.log >&2)' diff --git a/dockerfiles/aliases_print.sh b/dockerfiles/aliases_print.sh new file mode 100644 index 0000000000..44ed7eb6aa --- /dev/null +++ b/dockerfiles/aliases_print.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ensure the aliases defined in /etc/profile.d/ are loaded +# (interactive login shells source them automatically, but +# non-login shells like `docker exec … bash` do not) +shopt -q login_shell || source /etc/profile + +cat <<'EOF' +╭─────────────────────────────────────────────────────────────╮ +│ Built-in aliases & helpers │ +╰─────────────────────────────────────────────────────────────╯ + +‣ Environment switches + rmge → micromamba activate rmg_env + arce → micromamba activate arc_env + deact → micromamba deactivate + +‣ Jump to source trees + rmgcode → cd $rmgpy_path + dbcode → cd $rmgdb_path + arcode → cd $arc_path + +‣ One-liner runners + rmg → python-jl $rmgpy_path/rmg.py input.py + arkane → python $rmgpy_path/Arkane.py input.py + arc → python $arc_path/ARC.py input.yml + arcrestart → python $arc_path/ARC.py restart.yml + +Type aliases again at any time to reopen this cheat-sheet. +EOF diff --git a/dockerfiles/entrywrapper.sh b/dockerfiles/entrywrapper.sh index ce947f7158..388315afc2 100644 --- a/dockerfiles/entrywrapper.sh +++ b/dockerfiles/entrywrapper.sh @@ -1,61 +1,34 @@ -#!/bin/bash +#!/usr/bin/env bash +set -euo pipefail -# Function to initialize Micromamba -initialize_micromamba() { - eval "$(micromamba shell hook --shell bash)" -} +# 1) Micromamba setup (for both interactive & non-interactive) +eval "$(micromamba shell hook --shell bash)" -# Function to check and adjust permissions for mounted directories -# Excludes directories within /opt -check_and_adjust_permissions() { - for dir in /home/rmguser/*; do - # Check if it's a directory and ensure it's not within /opt - if [ -d "$dir" ] && [[ $dir != /opt/* ]]; then - # Adjust permissions for the directory - chmod -R 777 "$dir" - fi - done -} - -# Function to run for interactive mode -run_interactive() { - check_and_adjust_permissions - echo "Container started in interactive mode" - - /home/rmguser/alias_print.sh - - /bin/bash -} - -# Function to run for non-interactive mode -run_non_interactive() { - check_and_adjust_permissions - echo "Container started in non-interactive mode" - initialize_micromamba - - case "$1" in - rmg) - # Activate rmg_env and run RMG with python-jl - echo "Running with RMG environment..." - micromamba activate rmg_env - python-jl /home/rmguser/Code/RMG-Py/rmg.py "$2" - ;; - arc) - # Activate arc_env and run ARC - echo "Running with ARC environment..." - micromamba activate arc_env - python /home/rmguser/Code/ARC/ARC.py "$2" - ;; - *) - echo "Invalid option. Please specify 'rmg' or 'arc' as the first argument." - exit 1 - ;; - esac -} +# 1.1) Ensure the Code directory is owned by rmguser +if [ ! -O /home/rmguser/Code ]; then + chown -R rmguser:rmguser /home/rmguser/Code +fi -# Determine the run mode based on CONTAINER_MODE environment variable or terminal attachment -if [ "$CONTAINER_MODE" = "interactive" ] || ([ -z "$CONTAINER_MODE" ] && [ -t 0 ] && [ -t 1 ]); then - run_interactive -else - run_non_interactive "$@" +# 2) Interactive mode: login shells or docker exec … bash +if [ -t 0 ] && [ -t 1 ]; then +# # Show the aliases cheat-sheet +# /home/rmguser/.aliases_print.sh + # Drop into an interactive Bash + exec /bin/bash -l fi + +# 3) Non-interactive command mode: rmg or arc +case "${1:-}" in + rmg) + micromamba activate rmg_env + exec python-jl /home/rmguser/Code/RMG-Py/rmg.py "${2:-input.py}" + ;; + arc) + micromamba activate arc_env + exec python /home/rmguser/Code/ARC/ARC.py "${2:-input.yml}" + ;; + *) + echo "Usage: [input-file]" >&2 + exit 1 + ;; +esac From 7e1273359f0cb8c0ee214aded02a83e0c0ce0030 Mon Sep 17 00:00:00 2001 From: Calvin Pieters Date: Mon, 28 Jul 2025 13:20:29 +0300 Subject: [PATCH 3/5] Updating environment.yml Better separate packages for readability. Tighten versions of packages. Removed Anaconda from channel list --- environment.yml | 103 +++++++++++++++++++++++++++--------------------- 1 file changed, 57 insertions(+), 46 deletions(-) diff --git a/environment.yml b/environment.yml index cdad14ac3a..d9377c0899 100644 --- a/environment.yml +++ b/environment.yml @@ -1,61 +1,72 @@ name: arc_env channels: - - defaults - - rmg - - conda-forge - - cantera - - anaconda + - rmg # RMG-specific wheels + - conda-forge # modern science stack + - cantera # cantera 2.6 build + - defaults # only for legacy libs like libboost 1.67 + dependencies: - - cairo - - cairocffi - - cantera::cantera=2.6 - - conda-forge::cclib >=1.7.0 + # ——— hard pins ———————————————————————————————— + - python=3.7.* + - numpy=1.20.1 + + # ——— chemistry / RMG stack ———————————————— - rmg::chemprop - - coolprop - - coverage - - cython >=0.25.2 - - ffmpeg - - rmg::gprof2dot - - graphviz - - h5py - - jinja2 - - jupyter + - rmg::pydas>=1.0.2 + - rmg::pydqed>=1.0.1 + - rmg::pyrdl - rmg::lpsolve55 - - markupsafe - - matplotlib >=1.5 - - conda-forge::mopac - - mpmath - - rmg::muq2 - - networkx - rmg::numdifftools - - numpy==1.20.1 - - conda-forge::openbabel >= 3 + - rmg::quantities + - rmg::muq2 + + # rdkit from conda-forge (links to boost-cpp ≥1.70, no missing libboost) + - conda-forge::rdkit>=2020.03.3.0 + + # ——— core numerics / plotting —————————————— + - scipy - pandas + - scikit-learn + - matplotlib>=1.5 + - h5py + - networkx + - cython>=0.25.2 - psutil - - rmg::pydas >=1.0.2 + - mpmath - pydot - - rmg::pydqed >=1.0.1 - - pymongo - pyparsing - - rmg::pyrdl - - python >=3.7 - pyyaml - - rmg::quantities - - rmg::rdkit >=2020.03.3.0 - - scikit-learn - - scipy - - connie::symmetry + + # ——— thermo / quantum extras —————————————— + - cantera::cantera=2.6 + - conda-forge::openbabel=3.* + - conda-forge::cclib>=1.7.0 + - conda-forge::mopac + - conda-forge::qcelemental + - conda-forge::ase=3.22.1 + - coolprop + + # ——— utilities ———————————————————————————————— + - cairo + - cairocffi + - ffmpeg + - graphviz - xlrd - xlwt - - anaconda::sphinx_rtd_theme - - anaconda::paramiko >=2.6.0 - - conda-forge::py3dmol >= 0.8.0 - - conda-forge::ase==3.22.1 - - anaconda::ipython - - anaconda::sphinx - - anaconda::sphinxcontrib-jsmath - - conda-forge::qcelemental - - mako + - jinja2 + - markupsafe + - paramiko=2.6.* + - connie::symmetry - pytables - - anaconda::pytest + + # ——— docs / notebooks / tests (optional) ——————— + - jupyter=1.0.* + - sphinx + - sphinx_rtd_theme + - sphinxcontrib-jsmath + - mako=1.2.3 + - pytest - conda-forge::pytest-cov + - conda-forge::py3dmol>=0.8.0 + - coverage + - ipython From 70d66517ccc33bd395ef76799a6610e6c457c08f Mon Sep 17 00:00:00 2001 From: Calvin Pieters Date: Mon, 28 Jul 2025 13:20:37 +0300 Subject: [PATCH 4/5] Adds libgfortran dependency Adds libgfortran as a dependency to ensure compatibility and prevent potential issues related to Fortran libraries when using the xtb package. --- devtools/xtb_environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devtools/xtb_environment.yml b/devtools/xtb_environment.yml index 55f712a8ce..dc74c2c433 100644 --- a/devtools/xtb_environment.yml +++ b/devtools/xtb_environment.yml @@ -5,4 +5,4 @@ dependencies: - python=3.7 - xtb=6.3.3 - pyyaml - \ No newline at end of file + - libgfortran=14.2.0 From 35d891b6611f1277d2b3357f33fa35d9b83e681a Mon Sep 17 00:00:00 2001 From: Calvin Pieters Date: Mon, 28 Jul 2025 13:20:42 +0300 Subject: [PATCH 5/5] Enhances CI/CD caching and RMG-Py checkout Improves caching efficiency in CI/CD workflows by updating cache keys and using specific commit SHA for RMG-Py. This ensures that the correct version of RMG-Py is used and avoids unnecessary checkouts, optimizing build times. It also updates conda setup and caching. Also switches to actions/checkout@v4 for improved performance. --- .github/workflows/cont_int.yml | 33 ++++-- .github/workflows/update-cache.yml | 176 +++++++++++++++-------------- 2 files changed, 113 insertions(+), 96 deletions(-) diff --git a/.github/workflows/cont_int.yml b/.github/workflows/cont_int.yml index 98a0ac5263..49f57e2aaa 100644 --- a/.github/workflows/cont_int.yml +++ b/.github/workflows/cont_int.yml @@ -11,6 +11,8 @@ on: schedule: - cron: '0 0 * * *' +env: + RMG_PY_COMMIT: 55464c54d1fa61b531e865682df598d33718597d jobs: build: @@ -37,22 +39,28 @@ jobs: - name: Cache RMG-Py id: cache-rmg-py uses: actions/cache@v3 - env: - CACHE_NUMBER: 2 with: path: RMG-Py - key: ${{ runner.os }}-rmg-main + key: ${{ runner.os }}-rmg-${{ env.RMG_PY_COMMIT }} restore-keys: | ${{ runner.os }}-rmg- - - name: Checkout RMG-py + # - name: Checkout RMG-Py at specific commit + # if: steps.cache-rmg-py.outputs.cache-hit != 'true' + # uses: actions/checkout@v3 + # with: + # repository: ReactionMechanismGenerator/RMG-Py + # path: RMG-Py + # ref: ${{ env.RMG_PY_COMMIT }}^1 + # fetch-depth: 0 + + - name: Clone & checkout RMG-Py at specific SHA if: steps.cache-rmg-py.outputs.cache-hit != 'true' - uses: actions/checkout@v3 - with: - repository: ReactionMechanismGenerator/RMG-Py - path: RMG-Py - ref: main - fetch-depth: 1 + run: | + git clone --filter=blob:none --no-checkout https://github.com/ReactionMechanismGenerator/RMG-Py.git RMG-Py + cd RMG-Py + git fetch --no-tags --prune origin ${RMG_PY_COMMIT} + git checkout --detach ${RMG_PY_COMMIT} - name: Cache RMG-database id: cache-rmg-db @@ -64,6 +72,7 @@ jobs: key: ${{ runner.os }}-rmgdb-main restore-keys: | ${{ runner.os }}-rmgdb- + - name: Checkout RMG-database if: steps.cache-rmg-db.outputs.cache-hit != 'true' uses: actions/checkout@v3 @@ -71,7 +80,7 @@ jobs: repository: ReactionMechanismGenerator/RMG-database path: RMG-database ref: main - fetch-depth: 1 + fetch-depth: 0 - name: Cache AutoTST id: cache-autotst @@ -135,7 +144,7 @@ jobs: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('environment.yml') }} - name: Set up miniconda - uses: conda-incubator/setup-miniconda@v3 + uses: conda-incubator/setup-miniconda@v3.2.0 with: auto-update-conda: true environment-file: environment.yml diff --git a/.github/workflows/update-cache.yml b/.github/workflows/update-cache.yml index ba9f15e8f4..dfce1d1114 100644 --- a/.github/workflows/update-cache.yml +++ b/.github/workflows/update-cache.yml @@ -2,128 +2,136 @@ name: Update cache on: schedule: - - cron: "0 0 */7 * *" + - cron: '0 0 */7 * *' push: - branches: - - main + branches: [ main ] jobs: update_cache: runs-on: ubuntu-latest defaults: run: - shell: bash -el {0} + shell: bash -el {0} + steps: + # ────────── source repo ────────── - name: Checkout ARC - uses: actions/checkout@v3 - + uses: actions/checkout@v4 + + # ────────── RMG‑Py ────────── - name: Cache RMG id: cache-rmg - uses: actions/cache@v2 + uses: actions/cache@v4 with: - path: RMG-Py - key: ${{ runner.os }}-rmg-main - - - name: Checkout RMG - uses: actions/checkout@v3 + path: RMG-Py + key: ${{ runner.os }}-rmg-main + restore-keys: | + ${{ runner.os }}-rmg- + - name: Checkout RMG-Py + if: steps.cache-rmg.outputs.cache-hit != 'true' + uses: actions/checkout@v4 with: - repository: ReactionMechanismGenerator/RMG-Py - path: RMG-Py - ref: main - fetch-depth: 1 + repository: ReactionMechanismGenerator/RMG-Py + path: RMG-Py + ref: 55464c54d1fa61b531e865682df598d33718597d + fetch-depth: 1 + # ────────── RMG‑database ────────── - name: Cache RMG-database id: cache-rmg-db - uses: actions/cache@v2 + uses: actions/cache@v4 with: - path: RMG-database - key: ${{ runner.os }}-rmgdb-main - + path: RMG-database + key: ${{ runner.os }}-rmgdb-main + restore-keys: | + ${{ runner.os }}-rmgdb- - name: Checkout RMG-database - uses: actions/checkout@v3 + if: steps.cache-rmg-db.outputs.cache-hit != 'true' + uses: actions/checkout@v4 with: - repository: ReactionMechanismGenerator/RMG-database - path: RMG-database - ref: main - fetch-depth: 1 - + repository: ReactionMechanismGenerator/RMG-database + path: RMG-database + ref: main + fetch-depth: 1 + + # ────────── AutoTST ────────── - name: Cache AutoTST id: cache-autotst - uses: actions/cache@v2 + uses: actions/cache@v4 with: - path: AutoTST - key: ${{ runner.os }}-autotst-main - restore-keys: | - ${{ runner.os }}-autotst- + path: AutoTST + key: ${{ runner.os }}-autotst-main + restore-keys: | + ${{ runner.os }}-autotst- - name: Checkout AutoTST if: steps.cache-autotst.outputs.cache-hit != 'true' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: - repository: ReactionMechanismGenerator/AutoTST - path: AutoTST - ref: main - fetch-depth: 1 - + repository: ReactionMechanismGenerator/AutoTST + path: AutoTST + ref: main + fetch-depth: 1 + + # ────────── TS‑GCN ────────── - name: Cache TS-GCN id: cache-tsgcn - uses: actions/cache@v2 + uses: actions/cache@v4 with: - path: TS-GCN - key: ${{ runner.os }}-tsgcn-main - restore-keys: | - ${{ runner.os }}-tsgcn- + path: TS-GCN + key: ${{ runner.os }}-tsgcn-main + restore-keys: | + ${{ runner.os }}-tsgcn- - name: Checkout TS-GCN if: steps.cache-tsgcn.outputs.cache-hit != 'true' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: - repository: ReactionMechanismGenerator/TS-GCN - path: TS-GCN - ref: main - fetch-depth: 1 - + repository: ReactionMechanismGenerator/TS-GCN + path: TS-GCN + ref: main + fetch-depth: 1 + + # ────────── KinBot ────────── - name: Cache KinBot id: cache-kinbot - uses: actions/cache@v2 + uses: actions/cache@v4 with: - path: KinBot - key: ${{ runner.os }}-kinbot-2.0.6 - restore-keys: | - ${{ runner.os }}-kinbot- - - - name: Checkout Kinbot 2.0.6 + path: KinBot + key: ${{ runner.os }}-kinbot-2.0.6 + restore-keys: | + ${{ runner.os }}-kinbot- + - name: Checkout KinBot 2.0.6 if: steps.cache-kinbot.outputs.cache-hit != 'true' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: - repository: zadorlab/KinBot - path: KinBot - ref: v2.0.6 - fetch-depth: 1 - - - name: Cache Packages Packages - uses: actions/cache@v2 - env: - CACHE_NUMBER: 0 + repository: zadorlab/KinBot + path: KinBot + ref: v2.0.6 + fetch-depth: 1 + + # ────────── Conda package cache ────────── + - name: Cache Conda packages + id: cache-conda-pkgs + uses: actions/cache@v4 with: - path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('environment.yml') }} + path: ~/conda_pkgs_dir + key: ${{ runner.os }}-conda-0-${{ hashFiles('environment.yml') }} + + # ────────── Create / restore env ────────── - name: Setup ARC Env - uses: conda-incubator/setup-miniconda@v3 + uses: conda-incubator/setup-miniconda@v3.2.0 with: - auto-update-conda: true - environment-file: environment.yml - activate-environment: arc_env - miniconda-version: latest - conda-solver: libmamba - + auto-update-conda: true + environment-file: environment.yml + activate-environment: arc_env + miniconda-version: latest + conda-solver: libmamba + + # ────────── Update env & save to cache ────────── + - name: Update environment + run: mamba env update -n arc_env -f environment.yml + - name: Cache ARC env - uses: actions/cache@v2 + uses: actions/cache@v4 with: - path: ${{ env.CONDA }}/envs - key: conda-${{ runner.os }}--${{ runner.arch }}--${{ env.CACHE_NUMBER}} - env: - # Increase this value to reset cache if etc/example-environment.yml has not changed - CACHE_NUMBER: 0 - id: cache-arc-env - - name: Update environment - run: conda env update -n arc_env -f environment.yml - if: steps.cache-arc-env.outputs.cache-hit != 'true' + path: ${{ env.CONDA }}/envs + key: conda-${{ runner.os }}-${{ runner.arch }}-0