Add python libraries in a separate conda env

Author: pantherman594

features/src/workbench-tools/devcontainer-feature.json

@@ -4,6 +4,16 @@
   "name": "Workbench Tools",
   "description": "Installs common tools for Workbench Apps. Currently it only supports Debian-based systems (e.g. Ubuntu) on x86_64.",
   "options": {
+      "libEnv": {
+          "type": "string",
+          "default": "/opt/conda/envs/workbench-ds",
+          "description": "The path to the conda environment where Python libraries (e.g. numpy, plotly, scipy) should be installed. This could be a pre-existing environment or a new one."
+      },
+      "libPythonVersion": {
+          "type": "string",
+          "default": "3.10",
+          "description": "The Python version to use for the installed libraries."
+      },
       "cloud": {
           "type": "string",
           "default": "",

features/src/workbench-tools/install-conda.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
-# install.sh installs common workbench tools in the devcontainer. Currently it
-# only supports Debian-based systems (e.g. Ubuntu) on x86_64.
+# install.sh installs common workbench tools in the devcontainer.
+# Designed as an isolated feature that does not interfere with the host image.
 
 set -o errexit
 set -o nounset
@@ -10,6 +10,8 @@ set -o xtrace
 
 readonly CLOUD="${CLOUD:-""}"
 readonly USERNAME="${USERNAME:-"root"}"
+readonly LIB_ENV="${LIBENV:-"/opt/conda/envs/workbench-ds"}"
+readonly LIB_PYTHON_VERSION="${LIBPYTHONVERSION:-"3.10"}"
 USER_HOME_DIR="${USERHOMEDIR:-"/home/${USERNAME}"}"
 if [[ "${USER_HOME_DIR}" == "/home/root" ]]; then
     USER_HOME_DIR="/root"
@@ -23,10 +25,15 @@ WORKDIR="$(mktemp -d)"
 readonly WORKDIR
 
 readonly WORKBENCH_TOOLS_DIR="/opt/workbench-tools"
+export MAMBA_ROOT_PREFIX="${WORKBENCH_TOOLS_DIR}/conda-root"
 
 function cleanup() {
     rm -rf "${WORKDIR:?}"
     rm -rf /var/lib/apt/lists/*
+    # Clean up micromamba caches to save image space
+    if command -v micromamba &> /dev/null; then
+        micromamba clean --all --yes
+    fi
 }
 
 trap 'cleanup' EXIT
@@ -61,23 +68,27 @@ check_packages \
     git \
     sed \
     sudo \
-    tar
+    tar \
+    bzip2
 
-if ! mamba --version &>/dev/null; then
-    source ./install-conda.sh
+# Install Micromamba
+if ! type micromamba &>/dev/null; then
+    echo "Installing Micromamba..."
+    curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -C /usr/local -xvj bin/micromamba
 fi
 
 # Install the samtools family of tools in a separate environment since some of
-# the other tools depend on old versions of these. This will take priority in
-# the PATH.
+# the other tools depend on old versions of these.
 CONDA_PACKAGES_1=(
     "bcftools"
     "htslib" # includes bgzip and tabix
     "samtools"
 )
 
+# Environment 2 contains the genomics CLI tools. They will be added to the
+# PATH but will not be usable as Python libraries.
 CONDA_PACKAGES_2=(
-    "python=3.9"
+    "python=3.10"
     "pip"
     "perl==5.32.1"
     "bedtools"
@@ -91,13 +102,53 @@ CONDA_PACKAGES_2=(
     "vcftools"
 )
 
+# Environment 3 contains data science Python libraries. These should be
+# accessible from the user's default Python environment, which is why we install
+# them separately and give the user control over whether to inject them into an
+# existing environment or create a new one.
+CONDA_PACKAGES_3=(
+    "python=${LIB_PYTHON_VERSION}"
+    "google-cloud-storage"
+    "ipykernel"
+    "ipywidgets"
+    "jupyter"
+    "openai"
+    "matplotlib"
+    "numpy"
+    "plotly"
+    "pandas"
+    "seaborn"
+    "scikit-learn"
+    "scipy"
+    "tqdm"
+)
+
+# Build isolated environments
 mkdir -p "${WORKBENCH_TOOLS_DIR}"
-mamba create --prefix "${WORKBENCH_TOOLS_DIR}/1" -c bioconda -y "${CONDA_PACKAGES_1[@]}"
-mamba create --prefix "${WORKBENCH_TOOLS_DIR}/2" -c bioconda -y "${CONDA_PACKAGES_2[@]}"
+echo "Building Environment 1 (Samtools family)..."
+micromamba create --prefix "${WORKBENCH_TOOLS_DIR}/1" -c bioconda -c conda-forge -y "${CONDA_PACKAGES_1[@]}"
+
+echo "Building Environment 2 (Genomics CLI Tools)..."
+micromamba create --prefix "${WORKBENCH_TOOLS_DIR}/2" -c bioconda -c conda-forge -y "${CONDA_PACKAGES_2[@]}"
+
+echo "Building Environment 3 (Python Libraries)..."
+LIB_ENV_EXISTS=0
+
+if [ -d "${LIB_ENV}" ]; then
+    # SCENARIO A: Target environment already exists on host. Inject packages into it.
+    LIB_ENV_EXISTS=1
+    echo "Host environment detected at ${LIB_ENV}. Injecting data science packages..."
+    micromamba install --prefix "${LIB_ENV}" -c conda-forge -y "${CONDA_PACKAGES_3[@]}"
+else
+    # SCENARIO B: Target environment does not exist. Create it from scratch.
+    echo "No host environment found. Creating standalone environment at ${LIB_ENV}..."
+    mkdir -p "$(dirname "${LIB_ENV}")"
+    micromamba create --prefix "${LIB_ENV}" -c conda-forge -y "${CONDA_PACKAGES_3[@]}"
+fi
 
 # Install dsub via pip if on GCP. The conda version is outdated.
 if [[ "${CLOUD}" == "gcp" ]]; then
-    "${WORKBENCH_TOOLS_DIR}/2/bin/pip" install dsub
+    "${WORKBENCH_TOOLS_DIR}/2/bin/pip" install --no-cache-dir dsub
 fi
 
 # Force the perl and python scripts to use the correct perl/python
@@ -106,28 +157,31 @@ find -L "${WORKBENCH_TOOLS_DIR}/2/bin" -type f -executable -exec \
         -e "1s|^#\!/usr/bin/env perl\\r\?$|#\!${WORKBENCH_TOOLS_DIR}/2/bin/perl|" \
         -e "1s|^#\!/usr/bin/env python\\r\?$|#\!${WORKBENCH_TOOLS_DIR}/2/bin/python|" {} \;
 
-# Make the login user the owner of the conda environment
+# Make the login user the owner of the conda environments
 chown -R "${USERNAME}:" "${WORKBENCH_TOOLS_DIR}"
+chown -R "${USERNAME}:" "${LIB_ENV}"
 
 {
+    echo "# Workbench Tools Configuration"
+    
+    # If we created a standalone Python libraries environment from scratch, make it the default terminal Python.
+    # If it already existed (LIB_ENV_EXISTS=1), we leave the host image's PATH untouched to prevent shadowing.
+    if [[ "${LIB_ENV_EXISTS}" == "0" ]]; then
+        printf 'export PATH="%s/bin:$PATH"\n' "${LIB_ENV}"
+    fi
+    
     # Set PATH to include workbench-tools binaries
     # shellcheck disable=SC2016 # we want $PATH to be evaluated at runtime
-    printf 'export PATH="%s:$PATH"\n' "${WORKBENCH_TOOLS_DIR}/1/bin:${WORKBENCH_TOOLS_DIR}/2/bin"
-
-    # Set CROMWELL_JAR environment variable
-    printf 'export CROMWELL_JAR="%s"\n' "${WORKBENCH_TOOLS_DIR}/2/share/cromwell/cromwell.jar"
-
-    # Make dsub a function that includes the correct PYTHONPATH. NeMo sets
-    # PYTHONPATH so we need to override it here. We use a function instead of an
-    # alias because aliases are not expanded in non-interactive shells.
-    # shellcheck disable=SC2016 # we want $PYTHONPATH to be evaluated at runtime
-    printf 'function dsub() (PYTHONPATH="%s/2/lib/python3.9/site-packages:${PYTHONPATH:-}" "%s/2/bin/dsub" "$@")\n' "${WORKBENCH_TOOLS_DIR}" "${WORKBENCH_TOOLS_DIR}" 
+    printf 'export PATH="$PATH:%s/1/bin:%s/2/bin"\n' "${WORKBENCH_TOOLS_DIR}" "${WORKBENCH_TOOLS_DIR}"
+    
+    # Set Cromwell JAR
+    printf 'export CROMWELL_JAR="%s/2/share/cromwell/cromwell.jar"\n' "${WORKBENCH_TOOLS_DIR}"
 } >> "${USER_HOME_DIR}/.bashrc"
 
 # Allow .bashrc to be sourced in non-interactive shells
 sed -i '/^# If not running interactively/,/esac/d' "${USER_HOME_DIR}/.bashrc" || true
 
 # Make sure the login user is the owner of their .bashrc
-chown -R "${USERNAME}:" "${USER_HOME_DIR}/.bashrc"
+chown "${USERNAME}:" "${USER_HOME_DIR}/.bashrc"
 
-echo "Done!"
+echo "Workbench tools installation complete!"

features/src/workbench-tools/install.sh

@@ -22,6 +22,8 @@
   ],
   "features": {
     "./.devcontainer/features/workbench-tools": {
+      "libEnv": "/opt/conda/envs/jupyter", // Use the jupyter conda environment
+      "libPythonVersion": "3.10", // This needs to match the Python version in the Dockerfile
       "cloud": "${templateOption:cloud}",
       "username": "jupyter",
       "userHomeDir": "/home/jupyter"

src/custom-workbench-jupyter-template/.devcontainer.json

@@ -11,6 +11,8 @@
   "postStartCommand": "./startupscript/remount-on-restart.sh jupyter /home/jupyter \"${templateOption:cloud}\" \"${templateOption:login}\"",
   "features": {
     "./.devcontainer/features/workbench-tools": {
+      "libEnv": "/opt/conda/envs/jupyter", // Use the jupyter conda environment
+      "libPythonVersion": "3.10", // This needs to match the Python version in the Dockerfile
       "cloud": "${templateOption:cloud}",
       "username": "jupyter",
       "userHomeDir": "/home/jupyter"

src/jupyter-aou/.devcontainer.json

@@ -44,6 +44,21 @@ if [[ "$HAS_WORKBENCH_TOOLS" == "true" ]]; then
     check "vep: filter_vep" "filter_vep --help > /dev/null"
     check "vep: variant_recoder" "variant_recoder --help | head -n10"
     check "vep: haplo" "haplo --help | head -n10"
+
+    # Python packages
+    check "python: google-cloud-storage" 'python3 -c "import google.cloud.storage"'
+    check "python: ipykernel" 'python3 -c "import ipykernel"'
+    check "python: ipywidgets" 'python3 -c "import ipywidgets"'
+    check "python: jupyter" 'python3 -c "import jupyter"'
+    check "python: openai" 'python3 -c "import openai"'
+    check "python: matplotlib" 'python3 -c "import matplotlib"'
+    check "python: numpy" 'python3 -c "import numpy"'
+    check "python: plotly" 'python3 -c "import plotly"'
+    check "python: pandas" 'python3 -c "import pandas"'
+    check "python: seaborn" 'python3 -c "import seaborn"'
+    check "python: scikit-learn" 'python3 -c "import sklearn"'
+    check "python: scipy" 'python3 -c "import scipy"'
+    check "python: tqdm" 'python3 -c "import tqdm"'
 fi
 
 # The postgres-client feature should install these