From 4da809ef79261db319df510c5190dc0424ba910e Mon Sep 17 00:00:00 2001 From: Sajid Ali Date: Mon, 15 Sep 2025 13:42:33 -0400 Subject: [PATCH] tools & software rework new file: .helix/config.toml modified: docs/cloud/04_dataproc/03_computation.md modified: docs/hpc/03_storage/06_best_practices.md modified: docs/hpc/03_storage/07_large_number_of_small_files.md new file: docs/hpc/06_tools_and_software/01_intro.md renamed: docs/hpc/06_tools_and_software/01_software_on_greene.md -> docs/hpc/06_tools_and_software/02_licensed_software.md new file: docs/hpc/06_tools_and_software/03_modules.md renamed: docs/hpc/06_tools_and_software/02_python_packages_with_virtual_environments.mdx -> docs/hpc/06_tools_and_software/04_python_packages_with_virtual_environments.mdx renamed: docs/hpc/06_tools_and_software/03_r_packages_with_renv.mdx -> docs/hpc/06_tools_and_software/05_r_packages_with_renv.mdx renamed: docs/hpc/06_tools_and_software/04_conda_environments.mdx -> docs/hpc/06_tools_and_software/06_conda_environments.mdx renamed: docs/hpc/06_tools_and_software/05_sqlite_handling_large_structured_data.md -> docs/hpc/06_tools_and_software/07_sqlite_handling_large_structured_data.md modified: docs/hpc/06_tools_and_software/_category_.json modified: docs/hpc/07_containers/03_singularity_with_conda.md modified: docusaurus.config.ts --- .helix/config.toml | 2 + docs/cloud/04_dataproc/03_computation.md | 2 +- docs/hpc/03_storage/06_best_practices.md | 2 +- .../07_large_number_of_small_files.md | 2 +- docs/hpc/06_tools_and_software/01_intro.md | 28 ++++++ ...e_on_greene.md => 02_licensed_software.md} | 92 ++++--------------- docs/hpc/06_tools_and_software/03_modules.md | 19 ++++ ...on_packages_with_virtual_environments.mdx} | 62 +++++++------ ...h_renv.mdx => 05_r_packages_with_renv.mdx} | 24 +++-- ...ronments.mdx => 06_conda_environments.mdx} | 30 +++--- ..._sqlite_handling_large_structured_data.md} | 2 +- .../hpc/06_tools_and_software/_category_.json | 2 +- .../03_singularity_with_conda.md | 2 +- docusaurus.config.ts | 12 ++- 14 files changed, 145 insertions(+), 136 deletions(-) create mode 100644 .helix/config.toml create mode 100644 docs/hpc/06_tools_and_software/01_intro.md rename docs/hpc/06_tools_and_software/{01_software_on_greene.md => 02_licensed_software.md} (73%) create mode 100644 docs/hpc/06_tools_and_software/03_modules.md rename docs/hpc/06_tools_and_software/{02_python_packages_with_virtual_environments.mdx => 04_python_packages_with_virtual_environments.mdx} (59%) rename docs/hpc/06_tools_and_software/{03_r_packages_with_renv.mdx => 05_r_packages_with_renv.mdx} (87%) rename docs/hpc/06_tools_and_software/{04_conda_environments.mdx => 06_conda_environments.mdx} (91%) rename docs/hpc/06_tools_and_software/{05_sqlite_handling_large_structured_data.md => 07_sqlite_handling_large_structured_data.md} (99%) diff --git a/.helix/config.toml b/.helix/config.toml new file mode 100644 index 0000000000..ecf8010bbb --- /dev/null +++ b/.helix/config.toml @@ -0,0 +1,2 @@ +[editor] +soft-wrap.enable = true diff --git a/docs/cloud/04_dataproc/03_computation.md b/docs/cloud/04_dataproc/03_computation.md index 4ae7c07b3e..ab4f867a68 100644 --- a/docs/cloud/04_dataproc/03_computation.md +++ b/docs/cloud/04_dataproc/03_computation.md @@ -120,7 +120,7 @@ trino --catalog=hive ## Using Conda -NYU Dataproc comes with `miniconda3` by default. This can be used to manage Python packages within your filesystem home directory. See [here](https://docs.conda.io/projects/conda/en/latest/user-guide/getting-started.html) or [here](../../hpc/06_tools_and_software/04_conda_environments.mdx) for more information on the conda command. +NYU Dataproc comes with `miniconda3` by default. This can be used to manage Python packages within your filesystem home directory. See [here](https://docs.conda.io/projects/conda/en/latest/user-guide/getting-started.html) or [here](../../hpc/06_tools_and_software/06_conda_environments.mdx) for more information on the conda command. ## Using Jupyter Notebooks diff --git a/docs/hpc/03_storage/06_best_practices.md b/docs/hpc/03_storage/06_best_practices.md index 9a0baa091b..15a0c5f0da 100644 --- a/docs/hpc/03_storage/06_best_practices.md +++ b/docs/hpc/03_storage/06_best_practices.md @@ -48,4 +48,4 @@ Your home directory has a relatively small number of inodes. If you create a conda or python environment in you home directory, this can eat up all the inodes. ::: -Please review the [Package Management section](../06_tools_and_software/01_software_on_greene.md#package-management-for-r-python--julia-and-conda-in-general) of the [Greene Software Page](../06_tools_and_software/01_software_on_greene.md). +Please review the [Package Management section](../06_tools_and_software/01_intro.md#package-management-for-r-python--julia-and-conda-in-general) of the [Greene Software Page](../06_tools_and_software/01_intro.md). diff --git a/docs/hpc/03_storage/07_large_number_of_small_files.md b/docs/hpc/03_storage/07_large_number_of_small_files.md index 49aa009581..3fc6f781c1 100644 --- a/docs/hpc/03_storage/07_large_number_of_small_files.md +++ b/docs/hpc/03_storage/07_large_number_of_small_files.md @@ -45,7 +45,7 @@ This option may require a lot of RAM - thus you may have to wait a long time for Special containers, which allow to either load data fast fully or access chosen elements without loading the whole dataset into RAM. ### SQLite -If you have structured data, a good option would be to use SQLite. Please see [SQLite: Handling Large Structured Data](../06_tools_and_software/05_sqlite_handling_large_structured_data.md) for more information. +If you have structured data, a good option would be to use SQLite. Please see [SQLite: Handling Large Structured Data](../06_tools_and_software/07_sqlite_handling_large_structured_data.md) for more information. ### HDF5 One can think about HDF5 file as a "container file" (database of a sort), which holds a lot of objects inside. diff --git a/docs/hpc/06_tools_and_software/01_intro.md b/docs/hpc/06_tools_and_software/01_intro.md new file mode 100644 index 0000000000..d430e7a831 --- /dev/null +++ b/docs/hpc/06_tools_and_software/01_intro.md @@ -0,0 +1,28 @@ +# Start here! + +We encourage you to setup your own computational environment on Torch and to assist you in doing so, we allow you to run [Apptainer](../07_containers/01_intro.md) (formerly known as Singularity) containers, we manage licensed software suites and offer extensive documentation, training and support. + +:::tip +We stongly advise that you setup your own computational enviromments via Apptainer containers and overlay files. Detailed documentation is available in the [containers section](../07_containers/01_intro.md). +::: + +## Package Management for R, Python, & Julia, and Conda in general +- [Conda environments (Python, R)](../06_tools_and_software/06_conda_environments.mdx) +- [Using virtual environments for Python](../06_tools_and_software/04_python_packages_with_virtual_environments.mdx) +- [Managing R packages with renv](../06_tools_and_software/05_r_packages_with_renv.mdx) + +## Examples of software usage on Greene +Examples can be found under `/scratch/work/public/examples/` and include the following + +| | | | +|-------------------|-----------------------|-----------------------| +| alphafold | knitro | Singularity | +| amd GPUs | lammps | slurm | +| comsol | matlab | spark | +| c-sharp | mathematica | stata | +| crystal17 | namd | squashfs | +| fluent | orca | trinity | +| gaussian | quantum-espresso | vnc | +| hadoop-streaming | R | vscode | +| julia | sas | xvfb | +| jupyter notebooks | schrodinger | | diff --git a/docs/hpc/06_tools_and_software/01_software_on_greene.md b/docs/hpc/06_tools_and_software/02_licensed_software.md similarity index 73% rename from docs/hpc/06_tools_and_software/01_software_on_greene.md rename to docs/hpc/06_tools_and_software/02_licensed_software.md index d63c3da192..ef460d9d61 100644 --- a/docs/hpc/06_tools_and_software/01_software_on_greene.md +++ b/docs/hpc/06_tools_and_software/02_licensed_software.md @@ -1,75 +1,19 @@ -# Software on Greene - -## Software Overview -There are different types of software packages available - -- Use `module avail` command to see preinstalled software. - - This includes the licensed software listed below -- Singularity Containers - - You can find those already built and ready to use, at location `/scratch/work/public/singularity/` - - For more information on running software with Singularity, See our [Containers Intro](../07_containers/01_intro.md). -- Python/R/Julia packages can be installed by a user - -If you need another linux program installed, please contact us at [hpc@nyu.edu](mailto:hpc@nyu.edu) - -## Software and Environment Modules -Lmod, an Environment Module system, is a tool for managing multiple versions and configurations of software packages and is used by many HPC centers around the world. With Environment Modules, software packages are installed away from the base system directories, and for each package, an associated modulefile describes what must be altered in a user's shell environment - such as the $PATH environment variable - in order to use the software package. The modulefile also describes dependencies and conflicts between this software package and other packages and versions. - -To use a given software package, you load the corresponding module. Unloading the module afterwards cleanly undoes the changes that loading the module made to your environment, thus freeing you to use other software packages that might have conflicted with the first one. - -Below is a list of modules and their associated functions: - -| Command | Function | -|-----------------------------------|-----------------------------------------------------------------------| -| module unload `` | unload a module | -| module show `` | see exactly what effect loading the module will have | -| module purge | remove all loaded modules from your environment | -| module load `` | load a module | -| module whatis `` | find out more about a software package | -| module list | check which modules are currently loaded in your environment | -| module avail | check what software packages are available | -| module help `` | A module file may include more detailed help for the software package | - - -## Package Management for R, Python, & Julia, and Conda in general -- [Conda environments (Python, R)](../06_tools_and_software/04_conda_environments.mdx) -- [Using virtual environments for Python](../06_tools_and_software/02_python_packages_with_virtual_environments.mdx) -- [Managing R packages with renv](../06_tools_and_software/03_r_packages_with_renv.mdx) - -## Examples of software usage on Greene -Examples can be found under `/scratch/work/public/examples/` and include the following - -| | | | -|-------------------|-----------------------|-----------------------| -| alphafold | knitro | Singularity | -| amd GPUs | lammps | slurm | -| comsol | matlab | spark | -| c-sharp | mathematica | stata | -| crystal17 | namd | squashfs | -| fluent | orca | trinity | -| gaussian | quantum-espresso | vnc | -| hadoop-streaming | R | vscode | -| julia | sas | xvfb | -| jupyter notebooks | schrodinger | | - -## Accessing Datasets with Singularity -- [Singularity for Datasets](../07_containers/04_squash_file_system_and_singularity.md) - -## Licensed Software -### SCHRODINGER +# Licensed Software + +## SCHRODINGER Schrödinger provides a complete suite of software solutions with the latest advances in pharmaceutical research and computational chemistry. The NYU New York campus has a limited number of licenses for the Biologics Suite (ConfGen, Epik, Jaguar, Jaguar pKa, MacroModel, Prime, QSite, SiteMap), BioLuminate and the Basic Docking Suite. :::note Schrödinger can be used for non-commercial, academic purposes ONLY. ::: -#### Using SCHRODINGER on HPC Cluster +### Using SCHRODINGER on HPC Cluster To load Schrodinger module execute ```sh $ module load schrodinger/2024.4 ``` -#### Using SCHRODINGER on NYU Lab Computers +### Using SCHRODINGER on NYU Lab Computers 1. Request your account at: [https://www.schrodinger.com/request-account](https://www.schrodinger.com/request-account) 2. Download the software at: [https://www.schrodinger.com/downloads/releases](https://www.schrodinger.com/downloads/releases) @@ -139,13 +83,13 @@ QSITE 5 SITEMAP 10 ``` -#### Schrodinger Example Files +### Schrodinger Example Files Example SBATCH jobs and outputs are available to review here: ```sh /scratch/work/public/examples/schrodinger/ ``` -### COMSOL +## COMSOL COMSOL is a problem-solving simulation environment, enforcing compatibility guarantees consistent multiphysics models. COMSOL Multiphysics is a general-purpose software platform, based on advanced numerical methods, for modeling and simulating physics-based problems. The package is cross-platform (Windows, Mac, Linux). The COMSOL Desktop helps you organize your simulation by presenting a clear overview of your model at any point. It uses functional form, structure, and aesthetics as the means to achieve simplicity for modeling complex realities. :::note @@ -176,7 +120,7 @@ cp /scratch/work/public/examples/comsol/test-input.mph /scratch//example sbatch run-comsol.sbatch ``` -### MATHEMATICA +## MATHEMATICA Mathematica is a general computing environment with organizing algorithmic, visualization, and user interface capabilities. The many mathematical algorithms included in Mathematica make computation easy and fast. To run Mathematica on the Greene HPC cluster, please load the relevant module in your batch job submission script: @@ -204,7 +148,7 @@ cp /scratch/work/public/examples/mathematica/basic/run-mathematica.sbatch /scrat sbatch run-mathematica.sbatch ``` -### SAS +## SAS SAS is a software package which enables programmers to perform many tasks, including: - Information retrieval - Data management @@ -220,7 +164,7 @@ SAS is a software package which enables programmers to perform many tasks, inclu There are licenses for 2 CPUs on the HPC Cluster. -#### Running a parallel SAS job on HPC cluster (Greene): +### Running a parallel SAS job on HPC cluster (Greene): To submit a SAS job for running on multiple processing elements, follow below steps: @@ -240,10 +184,10 @@ cp /scratch/work/public/examples/sas/run-sas.sbatch /scratch//example/ sbatch run-sas.sbatch ``` -### MATLAB +## MATLAB [MATLAB](https://www.mathworks.com/products/matlab.html) is a technical computing environment for high performance numeric computation and visualization. MATLAB integrates numerical analysis, matrix computation, signal processing, and graphics in an easy to use environment without using traditional programming. -#### MATLAB on personal computers and laptops +### MATLAB on personal computers and laptops NYU has a Total Academic Headcount (TAH) license which provides campus-wide access to MATLAB, Simulink, and a variety of add-on products. All faculty, researchers, and students (on any NYU campus) can use MATLAB on their personal computers and laptops and may go to the following site to download the NYU site license software free of charge. @@ -259,10 +203,10 @@ In order to run MATLAB interactively on the cluster, [start an interactive slurm Mathworks has provided a [Greene Matlab User Guide](https://drive.google.com/file/d/1lNNzf4lsFuH9a4bbsO18roCGhT3DwUq2/view) that presents useful tips and practices for using Matlab on the cluster. -### STATA +## STATA Stata is a command and menu-driven software package for statistical analysis. It is available for Windows, Mac, and Linux operating systems. Most of its users work in research. Stata's capabilities include data management, statistical analysis, graphics, simulations, regression and custom programming. -#### Running a parallel STATA job on HPC cluster (Greene): +### Running a parallel STATA job on HPC cluster (Greene): To submit a STATA job for running on multiple processing elements, follow below steps. @@ -281,12 +225,12 @@ cp /scratch/work/public/examples/stata/stata-test.do /scratch//example/ sbatch run-stata.sbatch ``` -### GAUSSIAN +## GAUSSIAN Gaussian uses basic quantum mechanic electronic structure programs. This software is capable of handling proteins and large molecules using semi-empirical, ab initio molecular orbital (MO), density functional, and molecular mechanics calculations. The NYU Gaussian license only covers PIs at the Washington Square Park campus. We will grant access to you after verifying your WSP affiliation. For access, please email [hpc@nyu.edu](mailto:hpc.nyu.edu). -#### Running a parallel Gaussian job on HPC cluster (Greene): +### Running a parallel Gaussian job on HPC cluster (Greene): To submit a Gaussian job for running on multiple processing elements, follow below steps. @@ -302,10 +246,10 @@ cp /scratch/work/public/examples/gaussian/basic/run-gaussian.sbatch /scratch/` | unload a module | +| module show `` | see exactly what effect loading the module will have | +| module purge | remove all loaded modules from your environment | +| module load `` | load a module | +| module whatis `` | find out more about a software package | +| module list | check which modules are currently loaded in your environment | +| module avail | check what software packages are available | +| module help `` | A module file may include more detailed help for the software package | + diff --git a/docs/hpc/06_tools_and_software/02_python_packages_with_virtual_environments.mdx b/docs/hpc/06_tools_and_software/04_python_packages_with_virtual_environments.mdx similarity index 59% rename from docs/hpc/06_tools_and_software/02_python_packages_with_virtual_environments.mdx rename to docs/hpc/06_tools_and_software/04_python_packages_with_virtual_environments.mdx index ca6dcaada6..aa29be619c 100644 --- a/docs/hpc/06_tools_and_software/02_python_packages_with_virtual_environments.mdx +++ b/docs/hpc/06_tools_and_software/04_python_packages_with_virtual_environments.mdx @@ -1,8 +1,12 @@ # Python Packages with Virtual Environments -In order to be able to install new Python packages and make your work reproducible, please use virtual environments. -There is more than one way to create a private environment in Python. +:::tip +This page describes how you can create virtual environments without containers in `$SCRATCH`/`/home/$USER`. We strongly encourage you to create your computational environments within Overlay files attached to Apptainer containers [as described in this section](../07_containers/03_singularity_with_conda.md). Please note that creating both kinds of conda environments is strongly discouraged as it leads to packages from one environment being accidentally used in another. +::: + + +In order to be able to install new Python packages and make your work reproducible, please use virtual environments. There is more than one way to create a private environment in Python. ## Create project directory and load Python module ```sh @@ -17,60 +21,58 @@ module load python/intel/3.8.6 ## Automatic deletion of your files This page describes the installation of packages on /scratch. One has to remember, though, that files stored in the HPC scratch file system are subject to the HPC Scratch old file purging policy:
-**Files on the /scratch file system that have not been accessed for 60 or more days will be purged** (see [HPC Storage](../03_storage/01_intro_and_data_management.mdx) for details). -Thus you can consider the following options +
+ +**Automatic deletion of your files** +Files on the `/scratch` file system that have not been accessed for 60 or more days will be purged (see [HPC Storage](../03_storage/01_intro_and_data_management.mdx) for details). Click to see how you can work around this limitation. + +:::tip +Thus you can consider the following options: - Reinstall your packages if some of the files get deleted - You can do this manually - - You can also do this automatically. For example, within a workflow of pipeline software like [Nextflow](https://www.nextflow.io/) -- Pay for "Research Project Space" - see details on the [Research Project Space](../03_storage/05_research_project_space.mdx) page. -- Use Singularity and install packages within a corresponding overlay file - See instructions on our [Singularity with Conda](../07_containers/03_singularity_with_conda.md) page. + - You can do this automatically. For example, within a workflow of a pipeline software like [Nextflow](https://www.nextflow.io/) +- Pay for "Research Project Space" - for details see [Research Project Space](../03_storage/05_research_project_space.mdx) +::: +
## Create virtual environment -It is advisable to create private environment inside the project directory. This boosts reproducibility and does not use space in `/home/$USER` - -### virtualenv -[virtualenv](https://virtualenv.pypa.io/en/latest/) is a tool to create isolated Python environments - -Since Python 3.3, a subset of it has been integrated into the standard library under the venv module. - -You can create new virtual environment in two ways: - -- empty -- inherit all packages from those installed on HPC already (and available in PATH after you load python module) - +It is advisable to create private environment inside the project directory. This boosts reproducibility and does not use space in `/home/$USER`: ```sh ## created directory for your project and cd there mkdir /scratch/$USER/my_project cd /scratch/$USER/my_project +``` +### `virtualenv` +[`virtualenv`](https://virtualenv.pypa.io/en/latest/) is a tool to create isolated Python environments. Since Python 3.3, a subset of it has been integrated into the standard library under the venv module. You can create new virtual environment in two ways: +- empty +```sh ## Create an EMPTY virtual environment virtualenv venv - +``` +- inherit all packages from those installed on HPC already (and available in PATH after you load python module) +```sh ## Create an virtual environment that inherits system packages virtualenv venv --system-site-packages ``` -### venv -[venv](https://docs.python.org/3/library/venv.html) is package shipped with Python. It provides subset of options available in virtualenv tool ([link](https://virtualenv.pypa.io/en/latest/)). +### `venv` +[`venv`](https://docs.python.org/3/library/venv.html) is package shipped with Python. It provides subset of options available in virtualenv tool ([link](https://virtualenv.pypa.io/en/latest/)). ```sh python -m venv venv ``` -Create new virtual environment in current directory - +Create new virtual environment in current directory: - empty -- inherit all packages from those installed on HPC already (and available in PATH after you load python module) ```sh -## created directory for your project and cd there -mkdir /scratch/$USER/my_project -cd /scratch/$USER/my_project -##EMPTY ## (use venv command to create environment called "venv") python -m venv venv - +``` +- inherit all packages from those installed on HPC already (and available in PATH after you load python module) +```sh ## Inhering all packages python -m venv venv --system-site-packages ``` diff --git a/docs/hpc/06_tools_and_software/03_r_packages_with_renv.mdx b/docs/hpc/06_tools_and_software/05_r_packages_with_renv.mdx similarity index 87% rename from docs/hpc/06_tools_and_software/03_r_packages_with_renv.mdx rename to docs/hpc/06_tools_and_software/05_r_packages_with_renv.mdx index 461b608aeb..97d2582be4 100644 --- a/docs/hpc/06_tools_and_software/03_r_packages_with_renv.mdx +++ b/docs/hpc/06_tools_and_software/05_r_packages_with_renv.mdx @@ -26,17 +26,21 @@ module load r/gcc/4.5.0 R ``` -### Automatic deletion of your files -This page describes the installation of packages on /scratch. One has to remember, though, that files stored in the HPC scratch file system are subject to the HPC Scratch old file purging policy:
-**Files on the `/scratch` file system that have not been accessed for 60 or more days will be purged.** Please see details at [HPC Storage](../03_storage/01_intro_and_data_management.mdx). - +
+ +**Automatic deletion of your files** +Files on the `/scratch` file system that have not been accessed for 60 or more days will be purged (see [HPC Storage](../03_storage/01_intro_and_data_management.mdx) for details). Click to see how you can work around this limitation. + +:::tip Thus you can consider the following options: - - Reinstall your packages if some of the files get deleted - You can do this manually - You can do this automatically. For example, within a workflow of a pipeline software like [Nextflow](https://www.nextflow.io/) -- Pay for "Research Project Space" - Details available at [Research Project Space](../03_storage/05_research_project_space.mdx) +- Pay for "Research Project Space" - for details see [Research Project Space](../03_storage/05_research_project_space.mdx) - Use Singularity and install packages within a corresponding overlay file - Details available at [Squash File System and Singularity](../07_containers/04_squash_file_system_and_singularity.md) +::: +
+ ### Cache directory setup By default, renv will cache package installation files to your home directory (most likely either in `~/.local/share/renv` or `~/.cache/R/renv/` or something similar). @@ -177,7 +181,7 @@ renv::snapshot() 2. Take file `renv.lock` and copy it to a new location for the project 3. At the new location - restore environment: go to directory of the project and execute. :::warning -Make sure version of R is the same +Make sure version of `R` is the same ::: ```R ## Reproduce environment @@ -187,7 +191,7 @@ R renv::restore() renv::init() ``` -`renv` will install/compile what is needed on any system (Linux, Windows, etc). You can share your code with other researchers no matter what system they use. However, you should be careful that the same version of R is used between systems, as mentioned above. +`renv` will install/compile what is needed on any system (Linux, Windows, etc). You can share your code with other researchers no matter what system they use. However, you should be careful that the same version of `R` is used between systems, as mentioned above. ### What to save/publish/commit with Git In order to have your work reproducible by you and/or others: @@ -195,8 +199,8 @@ In order to have your work reproducible by you and/or others: - be sure to include `renv.lock` (which lists all packages and versions that you use including the version of R) ### Migrating from Packrat -The `renv` package has replaced the now deprecated Packrat package. The `renv::migrate()` function makes it possible to migrate projects from Packrat to `renv`. See the ?migrate documentation for more details. In essence, calling `renv::migrate("")` will be enough to migrate the Packrat library and lockfile such that they can then be used by renv. +The `renv` package has replaced the now deprecated `Packrat` package. The `renv::migrate()` function makes it possible to migrate projects from `Packrat` to `renv`. See the ?migrate documentation for more details. In essence, calling `renv::migrate("")` will be enough to migrate the `Packrat` library and lockfile such that they can then be used by `renv`. ## Useful links -- [https://rstudio.github.io/renv/articles/renv.html](https://rstudio.github.io/renv/articles/renv.html) +- [Introduction to `renv`](https://rstudio.github.io/renv/articles/renv.html) diff --git a/docs/hpc/06_tools_and_software/04_conda_environments.mdx b/docs/hpc/06_tools_and_software/06_conda_environments.mdx similarity index 91% rename from docs/hpc/06_tools_and_software/04_conda_environments.mdx rename to docs/hpc/06_tools_and_software/06_conda_environments.mdx index 72b3a970af..589e56d60f 100644 --- a/docs/hpc/06_tools_and_software/04_conda_environments.mdx +++ b/docs/hpc/06_tools_and_software/06_conda_environments.mdx @@ -1,11 +1,14 @@ # Conda Environments (Python, R) -## What is Conda? -Package, dependency and environment management for any language—Python, R, Ruby, Lua, Scala, Java, JavaScript, C/ C++, FORTRAN, and more. +:::tip +This page describes how you can create conda environments without containers in `$SCRATCH`/`/home/$USER`. We strongly encourage you to create conda environments within Overlay files attached to Apptainer containers [as described in this section](../07_containers/03_singularity_with_conda.md). Please note that creating both kinds of conda environments is strongly discouraged as it leads to packages from one environment being accidentally used in another. +::: + -Please find more information at [the official documentation page](https://docs.conda.io/en/latest/) +## What is Conda? +Package, dependency and environment management for any language—Python, R, Ruby, Lua, Scala, Java, JavaScript, C/ C++, FORTRAN, and more. Please find more information at [the official documentation page](https://docs.conda.io/en/latest/) -Conda provides a great way to install packages that are already compiled, so you don't need to go over the long compilation process. If a package you need is not available, you can install it (and compile it when needed) using pip (Python) or install.packages (R). +Conda provides a great way to install packages that are already compiled, so you don't need to go over the long compilation process. If a package you need is not available, you can install it (and compile it when needed) using `pip` (Python) or `install.packages` (R). :::note **Reproducibility**: @@ -24,24 +27,25 @@ However, Conda has a significantly larger number of pre-compiled packages. ### Disadvantages -- Conda does not take advantage of packages already installed in the system (while [virtualenv and venv](./02_python_packages_with_virtual_environments.mdx) do) +- Conda does not take advantage of packages already installed in the system (while [virtualenv and venv](./04_python_packages_with_virtual_environments.mdx) do) - As you will see below, you may need to do additional steps to keep track of all installed packages (including those installed by pip and/or install.packages) -::::warning[Automatic deletion of your files] -This page describes the installation of packages on `/scratch`. One has to remember, though, that files stored in the HPC scratch file system are subject to the HPC Scratch old file purging policy:
-Files on the `/scratch` file system that have not been accessed for 60 or more days will be purged.
-You can read more this in [Data Management](../03_storage/01_intro_and_data_management.mdx). +
+ +**Automatic deletion of your files** +Files on the `/scratch` file system that have not been accessed for 60 or more days will be purged (see [HPC Storage](../03_storage/01_intro_and_data_management.mdx) for details). Click to see how you can work around this limitation. + :::tip Thus you can consider the following options: - - Reinstall your packages if some of the files get deleted - You can do this manually - You can do this automatically. For example, within a workflow of a pipeline software like [Nextflow](https://www.nextflow.io/) - Pay for "Research Project Space" - for details see [Research Project Space](../03_storage/05_research_project_space.mdx) -- Use Singularity and install packages within a corresponding overlay file - for details see [Singularity with Conda](../07_containers/03_singularity_with_conda.md) +- Use Singularity and install packages within a corresponding overlay file - Details available at [Squash File System and Singularity](../07_containers/04_squash_file_system_and_singularity.md) ::: -:::: +
+ ## Initializing Conda Load anaconda module @@ -241,7 +245,7 @@ The command `conda list --export` will not include packages installed by `instal #### `renv` -Please see details of using `renv` with conda for reproducibilty on [R packages with `renv`](./03_r_packages_with_renv.mdx). +Please see details of using `renv` with conda for reproducibilty on [R packages with `renv`](./05_r_packages_with_renv.mdx). ## Use conda env in a batch script The part of the batch script that will call the command should look like: diff --git a/docs/hpc/06_tools_and_software/05_sqlite_handling_large_structured_data.md b/docs/hpc/06_tools_and_software/07_sqlite_handling_large_structured_data.md similarity index 99% rename from docs/hpc/06_tools_and_software/05_sqlite_handling_large_structured_data.md rename to docs/hpc/06_tools_and_software/07_sqlite_handling_large_structured_data.md index cb932bf643..6f50a45e40 100644 --- a/docs/hpc/06_tools_and_software/05_sqlite_handling_large_structured_data.md +++ b/docs/hpc/06_tools_and_software/07_sqlite_handling_large_structured_data.md @@ -79,7 +79,7 @@ conda will install pre-compiled packages. Which is good (faster) and bad (not fu ::: :::tip -Alternative: install packages to a local directory or use renv as described in [R Packages with renv](./03_r_packages_with_renv.mdx) +Alternative: install packages to a local directory or use renv as described in [R Packages with renv](./05_r_packages_with_renv.mdx) ```sh mkdir /scratch/$USER/projects/myTempProject cd /scratch/$USER/projects/myTempProject diff --git a/docs/hpc/06_tools_and_software/_category_.json b/docs/hpc/06_tools_and_software/_category_.json index 401ad9049b..20da425159 100644 --- a/docs/hpc/06_tools_and_software/_category_.json +++ b/docs/hpc/06_tools_and_software/_category_.json @@ -1,3 +1,3 @@ { - "label": "Tools & Software available" + "label": "Tools & Software" } diff --git a/docs/hpc/07_containers/03_singularity_with_conda.md b/docs/hpc/07_containers/03_singularity_with_conda.md index bd739d56bb..0589806288 100644 --- a/docs/hpc/07_containers/03_singularity_with_conda.md +++ b/docs/hpc/07_containers/03_singularity_with_conda.md @@ -286,7 +286,7 @@ Singularity> pip install tensorboard ``` :::note -Please see [Conda Environments](../06_tools_and_software/04_conda_environments.mdx) for information on how to configure your conda environment. +Please see [Conda Environments](../06_tools_and_software/06_conda_environments.mdx) for information on how to configure your conda environment. ::: :::tip Please also keep in mind that once the overlay image is opened in default read-write mode, the file will be locked. You will not be able to open it from a new process. Once the overlay is opened either in read-write or read-only mode, it cannot be opened in RW mode from other processes either. For production jobs to run, the overlay image should be open in read-only mode. You can run many jobs at the same time as long as they are run in read-only mode. In this ways, it will protect the computation software environment, software packages are not allowed to change when there are jobs running. diff --git a/docusaurus.config.ts b/docusaurus.config.ts index 5c3923f9f5..1bc27a566f 100644 --- a/docusaurus.config.ts +++ b/docusaurus.config.ts @@ -163,8 +163,7 @@ const config: Config = { items: [ { label: "Feedback", - to: - "https://docs.google.com/forms/d/e/1FAIpQLSeHnmkPdR_IvWnT6a7U_V3RpfmQrpS8hjxI11FNnsZMlrBa4g/viewform", + to: "https://docs.google.com/forms/d/e/1FAIpQLSeHnmkPdR_IvWnT6a7U_V3RpfmQrpS8hjxI11FNnsZMlrBa4g/viewform", }, { label: "Announcements", @@ -182,7 +181,14 @@ const config: Config = { prism: { theme: prismThemes.oneLight, darkTheme: prismThemes.palenight, - additionalLanguages: ["bash", "shell-session", "lua", "sql", "julia"], + additionalLanguages: [ + "bash", + "shell-session", + "lua", + "sql", + "julia", + "r", + ], }, } satisfies Preset.ThemeConfig,