add new tools (#309)

* add new tools

* use workbench python

* bump python version

* undo python version bump

---------

Co-authored-by: Yu Hu <yuhuyoyo@verily.health>

Author: yuhuyoyo

features/src/workbench-tools/install.sh

@@ -89,6 +89,19 @@ CONDA_PACKAGES_2=(
     "plink2"
     "regenie"
     "vcftools"
+    "conda-forge::google-cloud-storage"
+    "conda-forge::ipykernel"
+    "conda-forge::ipywidgets"
+    "conda-forge::jupyter"
+    "conda-forge::openai"
+    "conda-forge::matplotlib"
+    "conda-forge::numpy"
+    "conda-forge::plotly"
+    "conda-forge::pandas"
+    "conda-forge::seaborn"
+    "conda-forge::scikit-learn"
+    "conda-forge::scipy"
+    "conda-forge::tqdm"
 )
 
 mkdir -p "${WORKBENCH_TOOLS_DIR}"
@@ -117,6 +130,12 @@ chown -R "${USERNAME}:" "${WORKBENCH_TOOLS_DIR}"
     # Set CROMWELL_JAR environment variable
     printf 'export CROMWELL_JAR="%s"\n' "${WORKBENCH_TOOLS_DIR}/2/share/cromwell/cromwell.jar"
 
+    # Prepend workbench-tools Python site-packages to PYTHONPATH to prevent conflicts
+    # with base image Python environments (e.g., NeMo's Python 3.12). This ensures
+    # workbench-tools packages and their dependencies are found first.
+    # shellcheck disable=SC2016 # we want $PYTHONPATH to be evaluated at runtime
+    printf 'export PYTHONPATH="%s/2/lib/python3.9/site-packages:${PYTHONPATH:-}"\n' "${WORKBENCH_TOOLS_DIR}"
+
     # Make dsub a function that includes the correct PYTHONPATH. NeMo sets
     # PYTHONPATH so we need to override it here. We use a function instead of an
     # alias because aliases are not expanded in non-interactive shells.

test/test.sh

@@ -44,6 +44,20 @@ if [[ "$HAS_WORKBENCH_TOOLS" == "true" ]]; then
     check "vep: filter_vep" "filter_vep --help > /dev/null"
     check "vep: variant_recoder" "variant_recoder --help | head -n10"
     check "vep: haplo" "haplo --help | head -n10"
+    # Python packages (use conda python directly)
+    check "python: google-cloud-storage" '/opt/workbench-tools/2/bin/python3 -c "import google.cloud.storage"'
+    check "python: ipykernel" '/opt/workbench-tools/2/bin/python3 -c "import ipykernel"'
+    check "python: ipywidgets" '/opt/workbench-tools/2/bin/python3 -c "import ipywidgets"'
+    check "python: jupyter" '/opt/workbench-tools/2/bin/python3 -c "import jupyter"'
+    check "python: openai" '/opt/workbench-tools/2/bin/python3 -c "import openai"'
+    check "python: matplotlib" '/opt/workbench-tools/2/bin/python3 -c "import matplotlib"'
+    check "python: numpy" '/opt/workbench-tools/2/bin/python3 -c "import numpy"'
+    check "python: plotly" '/opt/workbench-tools/2/bin/python3 -c "import plotly"'
+    check "python: pandas" '/opt/workbench-tools/2/bin/python3 -c "import pandas"'
+    check "python: seaborn" '/opt/workbench-tools/2/bin/python3 -c "import seaborn"'
+    check "python: scikit-learn" '/opt/workbench-tools/2/bin/python3 -c "import sklearn"'
+    check "python: scipy" '/opt/workbench-tools/2/bin/python3 -c "import scipy"'
+    check "python: tqdm" '/opt/workbench-tools/2/bin/python3 -c "import tqdm"'
 fi
 
 # The postgres-client feature should install these