From 12cc64e4061a09473fbbb00e962d59873115a8f2 Mon Sep 17 00:00:00 2001 From: Sajid Ali Date: Thu, 12 Jun 2025 19:33:00 -0400 Subject: [PATCH] reorganize modified: docs/hpc/01_getting_started/03_walkthrough_request_hpc_account.mdx modified: docs/hpc/01_getting_started/04_walkthrough_approve_hpc_account_request.md modified: docs/hpc/01_getting_started/05_walkthrough_renew_hpc_account.md renamed: docs/hpc/02_connecting_to_hpc/01_connecting_to_hpc.md -> docs/hpc/02_connecting_to_hpc/01_connecting_to_hpc.mdx modified: docs/hpc/03_storage/06_best_practices.md modified: docs/hpc/03_storage/07_large_number_of_small_files.md modified: docs/hpc/03_storage/08_transferring_cloud_storage_data_with_rclone.md modified: docs/hpc/05_submitting_jobs/01_slurm_submitting_jobs.md deleted: docs/hpc/06_tools_and_software/01_intro.md renamed: docs/hpc/06_tools_and_software/05_software_on_greene.md -> docs/hpc/06_tools_and_software/01_software_on_greene.md modified: docs/hpc/06_tools_and_software/02_conda_environments.md renamed: docs/hpc/06_tools_and_software/07_sqlite_handling_large_structured_data.md -> docs/hpc/06_tools_and_software/05_sqlite_handling_large_structured_data.md deleted: docs/hpc/06_tools_and_software/06_singularity_run_custom_applications_with_containers.md modified: docs/hpc/07_containers/01_intro.md renamed: docs/hpc/06_tools_and_software/static/singularity1.png -> docs/hpc/07_containers/static/singularity1.png renamed: docs/hpc/06_tools_and_software/static/singularity2.png -> docs/hpc/07_containers/static/singularity2.png renamed: docs/hpc/06_tools_and_software/static/singularity3.png -> docs/hpc/07_containers/static/singularity3.png renamed: docs/hpc/06_tools_and_software/static/singularity4.png -> docs/hpc/07_containers/static/singularity4.png renamed: docs/hpc/06_tools_and_software/static/singularity5.png -> docs/hpc/07_containers/static/singularity5.png modified: docs/hpc/09_ood/01_open_on_demand.md --- .../03_walkthrough_request_hpc_account.mdx | 2 +- ...walkthrough_approve_hpc_account_request.md | 4 +- .../05_walkthrough_renew_hpc_account.md | 4 +- ...ing_to_hpc.md => 01_connecting_to_hpc.mdx} | 24 +++-- docs/hpc/03_storage/06_best_practices.md | 2 +- .../07_large_number_of_small_files.md | 2 +- ...sferring_cloud_storage_data_with_rclone.md | 2 +- .../01_slurm_submitting_jobs.md | 5 +- docs/hpc/06_tools_and_software/01_intro.md | 1 - ..._on_greene.md => 01_software_on_greene.md} | 2 +- .../02_conda_environments.md | 2 +- ..._sqlite_handling_large_structured_data.md} | 0 ...run_custom_applications_with_containers.md | 89 ----------------- docs/hpc/07_containers/01_intro.md | 90 +++++++++++++++++- .../static/singularity1.png | Bin .../static/singularity2.png | Bin .../static/singularity3.png | Bin .../static/singularity4.png | Bin .../static/singularity5.png | Bin docs/hpc/09_ood/01_open_on_demand.md | 2 +- 20 files changed, 120 insertions(+), 111 deletions(-) rename docs/hpc/02_connecting_to_hpc/{01_connecting_to_hpc.md => 01_connecting_to_hpc.mdx} (88%) delete mode 100644 docs/hpc/06_tools_and_software/01_intro.md rename docs/hpc/06_tools_and_software/{05_software_on_greene.md => 01_software_on_greene.md} (99%) rename docs/hpc/06_tools_and_software/{07_sqlite_handling_large_structured_data.md => 05_sqlite_handling_large_structured_data.md} (100%) delete mode 100644 docs/hpc/06_tools_and_software/06_singularity_run_custom_applications_with_containers.md rename docs/hpc/{06_tools_and_software => 07_containers}/static/singularity1.png (100%) rename docs/hpc/{06_tools_and_software => 07_containers}/static/singularity2.png (100%) rename docs/hpc/{06_tools_and_software => 07_containers}/static/singularity3.png (100%) rename docs/hpc/{06_tools_and_software => 07_containers}/static/singularity4.png (100%) rename docs/hpc/{06_tools_and_software => 07_containers}/static/singularity5.png (100%) diff --git a/docs/hpc/01_getting_started/03_walkthrough_request_hpc_account.mdx b/docs/hpc/01_getting_started/03_walkthrough_request_hpc_account.mdx index 59eb15b0d8..ff8c2456d3 100644 --- a/docs/hpc/01_getting_started/03_walkthrough_request_hpc_account.mdx +++ b/docs/hpc/01_getting_started/03_walkthrough_request_hpc_account.mdx @@ -3,7 +3,7 @@ import TabItem from '@theme/TabItem'; # How to request an HPC account -First, check if you already have an account. You can check this by attempting to log in to the cluster, according to the instructions at [Connecting to the HPC Cluster](../02_connecting_to_hpc/01_connecting_to_hpc.md) +First, check if you already have an account. You can check this by attempting to log in to the cluster, according to the instructions at [Connecting to the HPC Cluster](../02_connecting_to_hpc/01_connecting_to_hpc.mdx) :::info VPN Needed You need to be on the NYU VPN to perform this task! diff --git a/docs/hpc/01_getting_started/04_walkthrough_approve_hpc_account_request.md b/docs/hpc/01_getting_started/04_walkthrough_approve_hpc_account_request.md index 0cd1ddefa5..eae00440ef 100644 --- a/docs/hpc/01_getting_started/04_walkthrough_approve_hpc_account_request.md +++ b/docs/hpc/01_getting_started/04_walkthrough_approve_hpc_account_request.md @@ -1,10 +1,12 @@ # How to approve an HPC Account Request +When someone nominates you as their HPC sponsor, you should be notified by email. + :::info VPN Needed You need to be on the NYU VPN to perform this task! ::: -When someone nominates you as their HPC sponsor, you should be notified by email. You can also [log into IIQ at any time](https://iiq.nyu.edu/identityiq), and if you have a request awaiting your approval, it will appear in your "Actions Items" box, as per the following screenshot: +You can also [log into IIQ at any time](https://iiq.nyu.edu/identityiq), and if you have a request awaiting your approval, it will appear in your "Actions Items" box, as per the following screenshot: !["Actions Items" box](./static/work_item.png) Another way to get to pending approvals is to click on the line item in the “Latest Approvals” section which will lead directly to the approval page. For new HPC Account Requests, the page will look like this: diff --git a/docs/hpc/01_getting_started/05_walkthrough_renew_hpc_account.md b/docs/hpc/01_getting_started/05_walkthrough_renew_hpc_account.md index 27b245488f..1beeda00af 100644 --- a/docs/hpc/01_getting_started/05_walkthrough_renew_hpc_account.md +++ b/docs/hpc/01_getting_started/05_walkthrough_renew_hpc_account.md @@ -1,11 +1,11 @@ # Renewing your HPC Account -Login to the URL given below, using your NetID/password, to create or manage HPC Account Requests: - :::info VPN Needed You need to be on the NYU VPN to perform this task! ::: +Login to the URL given below, using your NetID/password, to create or manage HPC Account Requests: + https://identity.it.nyu.edu/ Upon logging in, an end user’s landing page will look like this diff --git a/docs/hpc/02_connecting_to_hpc/01_connecting_to_hpc.md b/docs/hpc/02_connecting_to_hpc/01_connecting_to_hpc.mdx similarity index 88% rename from docs/hpc/02_connecting_to_hpc/01_connecting_to_hpc.md rename to docs/hpc/02_connecting_to_hpc/01_connecting_to_hpc.mdx index f9c23939ad..a60b56401f 100644 --- a/docs/hpc/02_connecting_to_hpc/01_connecting_to_hpc.md +++ b/docs/hpc/02_connecting_to_hpc/01_connecting_to_hpc.mdx @@ -1,22 +1,32 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + # Connecting to the HPC Cluster There are several ways to interact with the Greene HPC cluster. Similar to other Linux clusters, the most common method of connection is via a Command Line Interface (CLI). A CLI is a program that allows you to create and delete files, run programs, and navigate through directories and files via a typed prompt. On Mac, the built-in CLI application is called Terminal. While Windows 11 machines support a Linux Subsystem, which allows for similar functionality, a popular tool used to connect to a Linux server is a free application called [PuTTY][putty link]. +:::tip If you'd prefer to access the HPC cluster via a web gateway instead, please refer to [the section describing OpenOnDemand](./03_ood.md). +::: The following sections will outline basic ways to connect to the Greene cluster. Access to the clusters is primarily handled via the Secure Shell Protocol, or ssh. Below we outline ways to connect via ssh on Mac, Linux, and Windows machines. -## Remote Connections with the NYU VPN & HPC Gateway Server +:::info Connecting to Torch from outside NYU Network -If you are connecting from a remote location that is not on the NYU network (your home for example), you have two options: -1. **VPN Option:** [Set up your computer to use the NYU VPN][nyu vpn link]. Once you've created a VPN connection, you can proceed as if you were connected to the NYU net - -2. **Gateway Option:** Go through our gateway servers (example below). Gateways are designed to support only a very minimal set of commands and their only purpose is to let users connect HPC systems without needing to first connect to the VPN + + + [Set up your computer to use the NYU VPN][nyu vpn link]. Once you've created a VPN connection, you can proceed as if you were connected to the NYU net + + + Go through our gateway servers (example below). Gateways are designed to support only a very minimal set of commands and their only purpose is to let users connect HPC systems without needing to first connect to the VPN + + +::: You do not need to use the NYU VPN or gateways if you are connected to the NYU network (wired connection in your office or WiFi) or if you have VPN connection initiated. In this case you can ssh directly to the clusters. -## Command Line Interface (Use Terminal) +## Command Line Interface (with a Terminal) ### Mac & Linux Access @@ -48,7 +58,7 @@ Instructions on WSL installation can be found here: [https://docs.microsoft.com/ - If you are using WSL 2 (Windows subsystem for Linux), you may not be able to access internet when Cisco AnyConnect VPN, installed from exe file, is activated. A potential solution: uninstall Cisco AnyConnect and install AnyConnect using Microsoft Store, and then setup new VPN connection using settings described on [IT webpage][install vpn on windows link]. ::: -### Setting up SSH Keys +## Setting up SSH Keys Instead of typing your password every time you need to log in, you can also specify an ssh key. diff --git a/docs/hpc/03_storage/06_best_practices.md b/docs/hpc/03_storage/06_best_practices.md index ac85630521..3533e41235 100644 --- a/docs/hpc/03_storage/06_best_practices.md +++ b/docs/hpc/03_storage/06_best_practices.md @@ -48,5 +48,5 @@ Your home directory has a relatively small number of inodes. If you create a conda or python environment in you home directory, this can eat up all the inodes. ::: -Please review the [Package Management section](../06_tools_and_software/05_software_on_greene.md#package-management-for-r-python--julia-and-conda-in-general) of the [Greene Software Page](../06_tools_and_software/05_software_on_greene.md). +Please review the [Package Management section](../06_tools_and_software/01_software_on_greene.md#package-management-for-r-python--julia-and-conda-in-general) of the [Greene Software Page](../06_tools_and_software/01_software_on_greene.md). diff --git a/docs/hpc/03_storage/07_large_number_of_small_files.md b/docs/hpc/03_storage/07_large_number_of_small_files.md index 3fc6f781c1..49aa009581 100644 --- a/docs/hpc/03_storage/07_large_number_of_small_files.md +++ b/docs/hpc/03_storage/07_large_number_of_small_files.md @@ -45,7 +45,7 @@ This option may require a lot of RAM - thus you may have to wait a long time for Special containers, which allow to either load data fast fully or access chosen elements without loading the whole dataset into RAM. ### SQLite -If you have structured data, a good option would be to use SQLite. Please see [SQLite: Handling Large Structured Data](../06_tools_and_software/07_sqlite_handling_large_structured_data.md) for more information. +If you have structured data, a good option would be to use SQLite. Please see [SQLite: Handling Large Structured Data](../06_tools_and_software/05_sqlite_handling_large_structured_data.md) for more information. ### HDF5 One can think about HDF5 file as a "container file" (database of a sort), which holds a lot of objects inside. diff --git a/docs/hpc/03_storage/08_transferring_cloud_storage_data_with_rclone.md b/docs/hpc/03_storage/08_transferring_cloud_storage_data_with_rclone.md index 4b679a48a5..74ba87801b 100644 --- a/docs/hpc/03_storage/08_transferring_cloud_storage_data_with_rclone.md +++ b/docs/hpc/03_storage/08_transferring_cloud_storage_data_with_rclone.md @@ -29,7 +29,7 @@ There are specific instruction on the [rclone web site](https://rclone.org/drive ### Step 1: Login to Greene: -Follow [instructions](../02_connecting_to_hpc/01_connecting_to_hpc.md) to log into the Greene HPC cluster. +Follow [instructions](../02_connecting_to_hpc/01_connecting_to_hpc.mdx) to log into the Greene HPC cluster. ### Step 2: Load the rclone module ```sh diff --git a/docs/hpc/05_submitting_jobs/01_slurm_submitting_jobs.md b/docs/hpc/05_submitting_jobs/01_slurm_submitting_jobs.md index b0ee3c3704..b00e495c20 100644 --- a/docs/hpc/05_submitting_jobs/01_slurm_submitting_jobs.md +++ b/docs/hpc/05_submitting_jobs/01_slurm_submitting_jobs.md @@ -6,7 +6,7 @@ - HPC workloads are usually better suited to *batch processing* than *interactive* working. - A batch job is sent to the system when submitted with an **sbatch** command. - The working pattern we are all familiar with is *interactive* - where we type ( or click ) something interactively, and the computer performs the associated action. Then we type ( or click ) the next thing. -- Comments at the start of the script, which match a special pattern ( #SBATCH ) are read as Slurm options. +- Comments at the start of the script, which match a special pattern ( `#SBATCH` ) are read as Slurm options. ### The trouble with interactive environments @@ -15,8 +15,7 @@ There is a reason why GUIs are less common in HPC environments: **point-and-clic :::tip The job might not start immediately, and might take hours or days, so we prefer a *batch* approach: -- Plan the sequence of commands which will perform the actions we need - - Write them into a script. +- Plan the sequence of commands which will perform the actions we need and write the commands into a script. You can now run the script interactively, which is a great way to save effort if i frequently use the same workflow, or ... - Submit the script to a batch system, to run on dedicated resources when they become available. diff --git a/docs/hpc/06_tools_and_software/01_intro.md b/docs/hpc/06_tools_and_software/01_intro.md deleted file mode 100644 index 297962f3b6..0000000000 --- a/docs/hpc/06_tools_and_software/01_intro.md +++ /dev/null @@ -1 +0,0 @@ -# Tools and Software diff --git a/docs/hpc/06_tools_and_software/05_software_on_greene.md b/docs/hpc/06_tools_and_software/01_software_on_greene.md similarity index 99% rename from docs/hpc/06_tools_and_software/05_software_on_greene.md rename to docs/hpc/06_tools_and_software/01_software_on_greene.md index 8d09f1b89d..77b886e7fb 100644 --- a/docs/hpc/06_tools_and_software/05_software_on_greene.md +++ b/docs/hpc/06_tools_and_software/01_software_on_greene.md @@ -7,7 +7,7 @@ There are different types of software packages available - This includes the licensed software listed below - Singularity Containers - You can find those already built and ready to use, at location `/scratch/work/public/singularity/` - - For more information on running software with Singularity, [click here](../06_tools_and_software/06_singularity_run_custom_applications_with_containers.md). + - For more information on running software with Singularity, [click here](../07_containers/01_intro.md). - Python/R/Julia packages can be installed by a user If you need another linux program installed, please contact us at [hpc@nyu.edu](mailto:hpc@nyu.edu) diff --git a/docs/hpc/06_tools_and_software/02_conda_environments.md b/docs/hpc/06_tools_and_software/02_conda_environments.md index dca14deced..000b02edbb 100644 --- a/docs/hpc/06_tools_and_software/02_conda_environments.md +++ b/docs/hpc/06_tools_and_software/02_conda_environments.md @@ -9,7 +9,7 @@ Conda provides a great way to install packages that are already compiled, so you :::note Reproducibility: -One of the ways to ensure the reproducibility of your results is to have an independent conda environment in the directory of each project (one of the options shown below). This will also keep conda environment files away from your /home/$USER directory. +One of the ways to ensure the reproducibility of your results is to have an independent conda environment in the directory of each project (one of the options shown below). This will also keep conda environment files away from your `/home/$USER` directory. ::: ## Advantages/disadvantages of using Conda diff --git a/docs/hpc/06_tools_and_software/07_sqlite_handling_large_structured_data.md b/docs/hpc/06_tools_and_software/05_sqlite_handling_large_structured_data.md similarity index 100% rename from docs/hpc/06_tools_and_software/07_sqlite_handling_large_structured_data.md rename to docs/hpc/06_tools_and_software/05_sqlite_handling_large_structured_data.md diff --git a/docs/hpc/06_tools_and_software/06_singularity_run_custom_applications_with_containers.md b/docs/hpc/06_tools_and_software/06_singularity_run_custom_applications_with_containers.md deleted file mode 100644 index a8452b813c..0000000000 --- a/docs/hpc/06_tools_and_software/06_singularity_run_custom_applications_with_containers.md +++ /dev/null @@ -1,89 +0,0 @@ -# Singularity: Run Custom Applications with Containers - -## What is Singularity -Singularity is a container based Linux kernel workspace that works just like docker. You can run pre-built programs in containers without having to worry about the pre-install environment. - -For users who are familiar with Docker containers, Singularity works very similarly, and can even run Docker containers. - -For a detailed introduction on Singularity, visit their official site [here](https://apptainer.org/documentation/) - -## why do we use Singularity -There are multiple reasons to use Singularity on the HPC clusters: -1. Security: Singularity provides a layer of security as it does not require any root access on our clusters. This makes it safer against malware and bad scripts that might jeopardize the outer system. Thus we only support Singularity on our clusters(there are not other options such as Kubernetes or Docker on our clusters right now) -2. Containerization: Singularity will run all your images(packaged and pre-built programs) inside of its containers, each container works like a small vm. They contain all the required environment and files of a single Linux kernel and you don't have to worry about any pre-installation nonsense -3. Inter-connectivity: Containers are able to talk to each other, as well as the home system, so while each container has its own small space, they are still a part of a big interconnected structure. Thus enabling you to connect your programs. -4. Accessibility: Probably the most important feature of all, Singularity allows you to run your program in 2 to 3 simple steps, as shown in the topic how to run a singularity container. - -## how to run a singularity container -There are 3 steps to run a Singularity container on our clusters: - -pulling a image from Singularity hub or Docker hub -```sh -$ singularity pull -# image name can be for example shub://vsoch/hello-world or docker://godlovedc/lolcow -``` - -![singularity1](./static/singularity1.png) - -build the image -```sh -$ singularity build .simg:rw -# the image name can be a local image or an image from a hub -``` -We add the :rw tag at the end of the .simg to explicitly give it "read and write" permissions while building. - -![singularity2](./static/singularity2.png) - -You can now run your container using the built image: - -run container -```sh -# this is one way of running a container -$ singularity run .simg:ro -# this is another way to run a container -$ ./.simg:ro -``` - -Unlike in the build phase, we add the :ro tag which means "read only" - as we are now just executing the image, not building it, and thus do not need it to be written. Writing access causes the Singularity image to be locked and it can become inaccessible while it is in read/write mode, so read only mode is best for executing commands. - -running this would yield a menu for output: - -![singularity3](./static/singularity3.png) - -go into container -```sh -singularity shell .simg:ro -# after this step, you will be going into the container and start your programming -``` - -![singularity4](./static/singularity4.png) - -you can run commands for the container using exec arguments without actually going into the container -```sh -$ singularity exec .simg:ro -# adding commands to the back will return the display result of these commands in the container without actually going into the container -``` - -Example: - -![singularity5](./static/singularity5.png) - -That's it! Now you're good to go and can just use these simple steps to run singularity images and run your programs - -For full information and documentation on Singularity, visit their site [here](https://sylabs.io/docs/) - -## How to Create a Singularity Container -So what if you want to create an image from your container and save it for a rainy day? - -The instructions are [here](https://apptainer.org/documentation/) for your convenience, read through them to create your own Singularity container and package it into an image! - -For those that know how docker containers are built, you can build docker containers using the information here and upload them onto docker hub and pulling them using Singularity. Singularity supports all docker images! - -## Singularity vs Docker -Why are there so many mentions of Docker? The reason is that Singularity is essentially the same as Docker and you don't need to relearn Singularity if you already have experience with Docker. Now let's get into some pros and cons between the two programs. -1. Docker is more accepted commercially than Singularity. You can download and run Docker on your own computer with any operating system and build containers with ease while Singularity is used in a more academic setting. Singularity only supports Linux operating systems and cannot run on a windows linux kernel(your windows ubuntu), so it is much more limited. -2. However, Docker requires root or admin access for the operating system it deploys on, and our clusters do not offer that access to any software that requires this criteria. Thus Docker is not available on the clusters and Singularity is. -3. A silver lining in all of this is that Singularity fully supports docker images and you can do everything in docker and push your image to docker hub and pull them on the clusters. Thus making sure that you don't need to relearn Singularity all over again and can just use it through the simplest of commands in this wiki. - -Good luck with Singularity, and have fun! - diff --git a/docs/hpc/07_containers/01_intro.md b/docs/hpc/07_containers/01_intro.md index fb04ed4eb6..a8452b813c 100644 --- a/docs/hpc/07_containers/01_intro.md +++ b/docs/hpc/07_containers/01_intro.md @@ -1 +1,89 @@ -# Apptainer/Singularity +# Singularity: Run Custom Applications with Containers + +## What is Singularity +Singularity is a container based Linux kernel workspace that works just like docker. You can run pre-built programs in containers without having to worry about the pre-install environment. + +For users who are familiar with Docker containers, Singularity works very similarly, and can even run Docker containers. + +For a detailed introduction on Singularity, visit their official site [here](https://apptainer.org/documentation/) + +## why do we use Singularity +There are multiple reasons to use Singularity on the HPC clusters: +1. Security: Singularity provides a layer of security as it does not require any root access on our clusters. This makes it safer against malware and bad scripts that might jeopardize the outer system. Thus we only support Singularity on our clusters(there are not other options such as Kubernetes or Docker on our clusters right now) +2. Containerization: Singularity will run all your images(packaged and pre-built programs) inside of its containers, each container works like a small vm. They contain all the required environment and files of a single Linux kernel and you don't have to worry about any pre-installation nonsense +3. Inter-connectivity: Containers are able to talk to each other, as well as the home system, so while each container has its own small space, they are still a part of a big interconnected structure. Thus enabling you to connect your programs. +4. Accessibility: Probably the most important feature of all, Singularity allows you to run your program in 2 to 3 simple steps, as shown in the topic how to run a singularity container. + +## how to run a singularity container +There are 3 steps to run a Singularity container on our clusters: + +pulling a image from Singularity hub or Docker hub +```sh +$ singularity pull +# image name can be for example shub://vsoch/hello-world or docker://godlovedc/lolcow +``` + +![singularity1](./static/singularity1.png) + +build the image +```sh +$ singularity build .simg:rw +# the image name can be a local image or an image from a hub +``` +We add the :rw tag at the end of the .simg to explicitly give it "read and write" permissions while building. + +![singularity2](./static/singularity2.png) + +You can now run your container using the built image: + +run container +```sh +# this is one way of running a container +$ singularity run .simg:ro +# this is another way to run a container +$ ./.simg:ro +``` + +Unlike in the build phase, we add the :ro tag which means "read only" - as we are now just executing the image, not building it, and thus do not need it to be written. Writing access causes the Singularity image to be locked and it can become inaccessible while it is in read/write mode, so read only mode is best for executing commands. + +running this would yield a menu for output: + +![singularity3](./static/singularity3.png) + +go into container +```sh +singularity shell .simg:ro +# after this step, you will be going into the container and start your programming +``` + +![singularity4](./static/singularity4.png) + +you can run commands for the container using exec arguments without actually going into the container +```sh +$ singularity exec .simg:ro +# adding commands to the back will return the display result of these commands in the container without actually going into the container +``` + +Example: + +![singularity5](./static/singularity5.png) + +That's it! Now you're good to go and can just use these simple steps to run singularity images and run your programs + +For full information and documentation on Singularity, visit their site [here](https://sylabs.io/docs/) + +## How to Create a Singularity Container +So what if you want to create an image from your container and save it for a rainy day? + +The instructions are [here](https://apptainer.org/documentation/) for your convenience, read through them to create your own Singularity container and package it into an image! + +For those that know how docker containers are built, you can build docker containers using the information here and upload them onto docker hub and pulling them using Singularity. Singularity supports all docker images! + +## Singularity vs Docker +Why are there so many mentions of Docker? The reason is that Singularity is essentially the same as Docker and you don't need to relearn Singularity if you already have experience with Docker. Now let's get into some pros and cons between the two programs. +1. Docker is more accepted commercially than Singularity. You can download and run Docker on your own computer with any operating system and build containers with ease while Singularity is used in a more academic setting. Singularity only supports Linux operating systems and cannot run on a windows linux kernel(your windows ubuntu), so it is much more limited. +2. However, Docker requires root or admin access for the operating system it deploys on, and our clusters do not offer that access to any software that requires this criteria. Thus Docker is not available on the clusters and Singularity is. +3. A silver lining in all of this is that Singularity fully supports docker images and you can do everything in docker and push your image to docker hub and pull them on the clusters. Thus making sure that you don't need to relearn Singularity all over again and can just use it through the simplest of commands in this wiki. + +Good luck with Singularity, and have fun! + diff --git a/docs/hpc/06_tools_and_software/static/singularity1.png b/docs/hpc/07_containers/static/singularity1.png similarity index 100% rename from docs/hpc/06_tools_and_software/static/singularity1.png rename to docs/hpc/07_containers/static/singularity1.png diff --git a/docs/hpc/06_tools_and_software/static/singularity2.png b/docs/hpc/07_containers/static/singularity2.png similarity index 100% rename from docs/hpc/06_tools_and_software/static/singularity2.png rename to docs/hpc/07_containers/static/singularity2.png diff --git a/docs/hpc/06_tools_and_software/static/singularity3.png b/docs/hpc/07_containers/static/singularity3.png similarity index 100% rename from docs/hpc/06_tools_and_software/static/singularity3.png rename to docs/hpc/07_containers/static/singularity3.png diff --git a/docs/hpc/06_tools_and_software/static/singularity4.png b/docs/hpc/07_containers/static/singularity4.png similarity index 100% rename from docs/hpc/06_tools_and_software/static/singularity4.png rename to docs/hpc/07_containers/static/singularity4.png diff --git a/docs/hpc/06_tools_and_software/static/singularity5.png b/docs/hpc/07_containers/static/singularity5.png similarity index 100% rename from docs/hpc/06_tools_and_software/static/singularity5.png rename to docs/hpc/07_containers/static/singularity5.png diff --git a/docs/hpc/09_ood/01_open_on_demand.md b/docs/hpc/09_ood/01_open_on_demand.md index 1c1b8b2ba3..70572ec6c5 100644 --- a/docs/hpc/09_ood/01_open_on_demand.md +++ b/docs/hpc/09_ood/01_open_on_demand.md @@ -10,7 +10,7 @@ If you're not on the NYU network, you may need to connect via NYU's Virtual Priv This page describes how to use your Singularity with conda environment in Open OnDemand (OOD) GUI at Greene. ### Log Into Greene via the Terminal -The following commands must be run from the terminal. Information on accessing via the terminal can be found at the [Connecting to the HPC page](../02_connecting_to_hpc/01_connecting_to_hpc.md). +The following commands must be run from the terminal. Information on accessing via the terminal can be found at the [Connecting to the HPC page](../02_connecting_to_hpc/01_connecting_to_hpc.mdx). ### Preinstallation Warning :::warning