diff --git a/.gitignore b/.gitignore index 8a5c8f5f..c2141982 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ venv site - +**.DS_Store** diff --git a/.markdownlint.jsonc b/.markdownlint.jsonc index b542a039..65c5cc30 100644 --- a/.markdownlint.jsonc +++ b/.markdownlint.jsonc @@ -18,6 +18,9 @@ "code_block_line_length": 500 // some example console output is wide }, + // MD024/no-duplicate-heading - No duplicate headings + "MD024": false, + // MD046/code-block-style - Code block style // Disable consistency checks between fenced/indented code blocks. // Standard code blocks should use fences, while mkdocs admonitions require diff --git a/README.md b/README.md index 42d93934..b40717e6 100644 --- a/README.md +++ b/README.md @@ -39,4 +39,3 @@ Starting server at http://localhost:8000/ Press Ctrl+C to quit. CStopping server... ``` - diff --git a/docs/guides/ha-cluster/readme.md b/docs/guides/ha-cluster/readme.md index 1f88a993..dea2b3ef 100644 --- a/docs/guides/ha-cluster/readme.md +++ b/docs/guides/ha-cluster/readme.md @@ -1,92 +1,447 @@ --- -authors: Nate Thornton +authors: Tony Floeder categories: setup --- -# High Availability Cluster +# High Availability Cluster Setup for Rabbit Nodes -NNF software supports provisioning of Red Hat GFS2 (Global File System 2) storage. Per RedHat: -> GFS2 allows multiple nodes to share storage at a block level as if the storage were connected locally to each cluster node. GFS2 cluster file system requires a cluster infrastructure. +NNF software supports provisioning of Red Hat GFS2 (Global File System 2) storage. GFS2 allows multiple nodes to share storage at a block level as if the storage were connected locally to each cluster node. To use GFS2, each Rabbit node and its associated compute nodes must form a high-availability cluster using Pacemaker and Corosync. -Therefore, in order to use GFS2, the NNF node and its associated compute nodes must form a high availability cluster. +## Table of Contents + +- [Background: Pacemaker and Corosync](#background-pacemaker-and-corosync) +- [Cluster Architecture](#cluster-architecture) +- [Cluster Setup](#cluster-setup) +- [Quorum Configuration](#quorum-configuration) + - [Why the Rabbit Needs 17 Votes](#why-the-rabbit-needs-17-votes) + - [Configuring Quorum Votes](#configuring-quorum-votes) +- [Fencing with fence_recorder](#fencing-with-fence_recorder) + - [How It Works](#how-it-works) + - [Installation](#installation) + - [STONITH Configuration](#stonith-configuration) + - [Configuration Options](#configuration-options) + - [Verifying Configuration](#verifying-configuration) +- [Request/Response Protocol](#requestresponse-protocol) + - [Request File Format](#request-file-format) + - [Response File Format](#response-file-format) + - [Atomic File Writing](#atomic-file-writing) +- [Log Files](#log-files) +- [Troubleshooting](#troubleshooting) + - [Check STONITH Status](#check-stonith-status) + - [Common Issues](#common-issues) + - [Testing Fence Operations](#testing-fence-operations) +- [GFS2 File System Configuration](#gfs2-file-system-configuration) +- [Dynamic Cluster Lifecycle](#dynamic-cluster-lifecycle) +- [References](#references) + +## Background: Pacemaker and Corosync + +**Corosync** provides the cluster communication layer—it handles node membership, messaging between nodes, and quorum decisions. When a node becomes unresponsive, Corosync detects this and informs Pacemaker. + +**Pacemaker** is the cluster resource manager. It decides where resources run, handles failover when nodes fail, and coordinates fencing (STONITH) to protect shared storage from corruption. + +**Fencing (STONITH)** ensures that when a node fails, it is forcibly removed from the cluster before other nodes access its shared resources. Without fencing, a failed node could corrupt shared storage if it continues writing after the cluster assumes it's dead. + +For comprehensive documentation, see: + +- [Red Hat: Overview of High Availability](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_and_managing_high_availability_clusters/assembly_overview-of-high-availability-configuring-and-managing-high-availability-clusters) +- [Red Hat: Fencing in a Red Hat High Availability Cluster](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_and_managing_high_availability_clusters/assembly_configuring-fencing-configuring-and-managing-high-availability-clusters) + +## Cluster Architecture + +Each Rabbit node forms a separate HA cluster with its 16 compute nodes: + +```text +┌─────────────────────────────────────────────────────────────────────────┐ +│ Rabbit Cluster (17 nodes) │ +│ │ +│ ┌──────────────┐ │ +│ │ rabbit-node-1│ │ +│ │ (Rabbit) │ │ +│ └──────────────┘ │ +│ │ │ +│ ┌───────────┬───────────┼───────────┬────────────┐ │ +│ │ │ │ │ │ │ +│ ▼ ▼ ▼ ▼ ▼ │ +│ ┌─────────┐ ┌─────────┐ ┌─────────┐ ┌──────────┐ ┌──────────┐ │ +│ │compute-1│ │compute-2│ │ · · · │ │compute-15│ │compute-16│ │ +│ └─────────┘ └─────────┘ └─────────┘ └──────────┘ └──────────┘ │ +│ │ +│ All 17 nodes communicate via Corosync │ +└─────────────────────────────────────────────────────────────────────────┘ +``` ## Cluster Setup -Red Hat provides instructions for [creating a high availability cluster with Pacemaker](https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/8/html/configuring_and_managing_high_availability_clusters/assembly_creating-high-availability-cluster-configuring-and-managing-high-availability-clusters#doc-wrapper), including instructions for [installing cluster software](https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/8/html/configuring_and_managing_high_availability_clusters/assembly_creating-high-availability-cluster-configuring-and-managing-high-availability-clusters#proc_installing-cluster-software-creating-high-availability-cluster) and -[creating a high availability cluster](https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/8/html/configuring_and_managing_high_availability_clusters/assembly_creating-high-availability-cluster-configuring-and-managing-high-availability-clusters). When following these instructions, each of the high availability clusters that are created should be named after the hostname of the NNF node. In the Red Hat examples the cluster name is `my_cluster`. +Red Hat provides comprehensive instructions for cluster setup: + +- [Installing Cluster Software](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_and_managing_high_availability_clusters/assembly_creating-high-availability-cluster-configuring-and-managing-high-availability-clusters#proc_installing-cluster-software-creating-high-availability-cluster) +- [Creating a High Availability Cluster](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_and_managing_high_availability_clusters/assembly_creating-high-availability-cluster-configuring-and-managing-high-availability-clusters) + +Each cluster should be named after the Rabbit node hostname. The following examples use `rabbit-node-1` with compute nodes numbered 1-16: + +```bash +# On each node, install cluster packages +dnf install pcs pacemaker fence-agents-all + +# Enable and start pcsd +systemctl enable --now pcsd + +# Set hacluster password (same on all nodes) +echo 'password' | passwd --stdin hacluster + +# From the Rabbit node, authenticate all 17 nodes +pcs host auth rabbit-node-1 \ + rabbit-compute-1 rabbit-compute-2 rabbit-compute-3 rabbit-compute-4 \ + rabbit-compute-5 rabbit-compute-6 rabbit-compute-7 rabbit-compute-8 \ + rabbit-compute-9 rabbit-compute-10 rabbit-compute-11 rabbit-compute-12 \ + rabbit-compute-13 rabbit-compute-14 rabbit-compute-15 rabbit-compute-16 \ + -u hacluster -p password + +# Create the cluster with all nodes +pcs cluster setup rabbit-node-1 \ + rabbit-node-1 \ + rabbit-compute-1 rabbit-compute-2 rabbit-compute-3 rabbit-compute-4 \ + rabbit-compute-5 rabbit-compute-6 rabbit-compute-7 rabbit-compute-8 \ + rabbit-compute-9 rabbit-compute-10 rabbit-compute-11 rabbit-compute-12 \ + rabbit-compute-13 rabbit-compute-14 rabbit-compute-15 rabbit-compute-16 + +pcs cluster start --all +pcs cluster enable --all +``` + +## Quorum Configuration + +Quorum determines how many nodes must agree before the cluster can take action. By default, each node gets one vote and quorum requires a majority. However, in the Rabbit cluster architecture, the Rabbit node owns all NVMe storage—compute nodes only access storage through the Rabbit. This creates an asymmetric relationship that requires special quorum configuration. + +### Why the Rabbit Needs 17 Votes + +The Rabbit node must **always** have quorum because: + +1. **Storage ownership**: The Rabbit owns all NVMe namespaces. If compute nodes could fence the Rabbit, they would lose access to all storage anyway. + +2. **Preventing split-brain**: If the Rabbit becomes isolated from compute nodes (network partition), compute nodes should not be able to form their own quorum and attempt to fence the Rabbit. + +3. **Fencing authority**: Only the Rabbit (via NNF software) can properly detach storage from failed compute nodes. Compute nodes cannot fence each other directly. + +With 17 votes for the Rabbit and 1 vote per compute node: + +| Scenario | Votes | Quorum (17 required) | Result | +|----------|-------|----------------------|--------| +| Rabbit alone | 17 | ✓ Yes | Rabbit can fence any compute node | +| All 16 computes, no Rabbit | 16 | ✗ No | Computes cannot take action | +| Rabbit + any computes | 17+ | ✓ Yes | Normal operation | + +### Configuring Quorum Votes + +After cluster setup, configure voting weights: + +```bash +# Stop the cluster to modify corosync.conf +pcs cluster stop --all + +# Edit corosync.conf on the Rabbit node to set votes +# The nodelist section should look like: +# +# nodelist { +# node { +# ring0_addr: rabbit-node-1 +# nodeid: 1 +# quorum_votes: 17 +# } +# node { +# ring0_addr: rabbit-compute-1 +# nodeid: 2 +# quorum_votes: 1 +# } +# ... (repeat for all compute nodes) +# } + +# After editing, sync the configuration to all nodes +pcs cluster sync + +# Start the cluster +pcs cluster start --all + +# Verify quorum configuration +corosync-quorumtool +``` + +The output should show the Rabbit with 17 votes and each compute with 1 vote, for a total of 33 votes and quorum at 17. + +## Fencing with fence_recorder + +The `fence_recorder` agent coordinates fencing with external NNF software using a request/response file pattern. When Pacemaker decides to fence a compute node, `fence_recorder`: + +1. Writes a fence request file +2. Waits for the NNF software to process the request and write a response +3. Returns success or failure to Pacemaker + +This allows the NNF software to perform storage cleanup (detaching NVMe namespaces) before the fence operation completes. + +> **Note:** Although the fence action is named "reboot", the NNF software does **not** actually reboot the compute node. Instead, the Rabbit's NNF software detaches all NVMe namespaces from the target compute node, preventing it from accessing any shared storage. This is sufficient for GFS2's requirements—the failed node can no longer corrupt shared data, regardless of whether it is still running. + +### How It Works + +```text +┌──────────────────────┐ +│ Pacemaker/Corosync │ +│ (Cluster Manager) │ +└──────────┬───────────┘ + │ Calls fence_recorder + ▼ +┌──────────────────────┐ ┌───────────────────────────────┐ +│ fence_recorder │──────▶│ Request File │ +│ (Fence Agent) │ │ requests/-.json │ +└──────────────────────┘ └───────────────────────────────┘ + │ │ + │ Waits for response │ NNF software reads request + │ ▼ + │ ┌───────────────────────────────┐ + │ │ NNF Software │ + │ │ - Detaches storage │ + │ │ - Updates node status │ + │ └───────────────────────────────┘ + │ │ + ▼ │ Writes response +┌───────────────────────────────┐ │ +│ Response File │◀────────────┘ +│ responses/-.json│ +└───────────────────────────────┘ + │ + │ Exit 0 (success) or 1 (failure) + ▼ +┌──────────────────────┐ +│ Pacemaker │ +└──────────────────────┘ +``` + +### Installation + +Install `fence_recorder` on all nodes in the cluster: +```bash +# Copy the agent to each node +sudo cp fence_recorder /usr/sbin/fence_recorder +sudo chmod 755 /usr/sbin/fence_recorder -## Fencing Agents +# Create the request/response directories on each rabbit node +sudo mkdir -p /localdisk/fence-recorder/{requests,responses} +sudo chmod 755 /localdisk/fence-recorder/{requests,responses} -Fencing is the process of restricting and releasing access to resources that a failed cluster node may have access to. Since a failed node may be unresponsive, an external device must exist that can restrict access to shared resources of that node, or to issue a hard reboot of the node. More information can be found form Red Hat: [1.2.1 Fencing](https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/8/html/configuring_and_managing_high_availability_clusters/assembly_overview-of-high-availability-configuring-and-managing-high-availability-clusters#fencing). +# Create log directory +sudo mkdir -p /var/log/cluster +sudo chmod 755 /var/log/cluster +``` + +### STONITH Configuration + +Create a STONITH resource for each compute node. Run these commands from any node in the cluster: + +```bash +# Create STONITH resources for all 16 compute nodes (1-16) +for i in $(seq 1 16); do + pcs stonith create compute-${i}-fence-recorder fence_recorder \ + port=compute-${i} \ + pcmk_host_list=compute-${i} \ + request_dir=/localdisk/fence-recorder/requests \ + response_dir=/localdisk/fence-recorder/responses \ + log_dir=/var/log/cluster \ + op monitor interval=120s timeout=10s +done + +# Enable fencing +pcs property set stonith-enabled=true +``` + +Alternatively, create resources individually: + +```bash +# Example: Create STONITH for rabbit-compute-1 +pcs stonith create compute-1-fence-recorder fence_recorder \ + port=rabbit-compute-1 \ + pcmk_host_list=compute-1 \ + request_dir=/localdisk/fence-recorder/requests \ + response_dir=/localdisk/fence-recorder/responses \ + log_dir=/var/log/cluster \ + op monitor interval=120s timeout=10s +``` + +### Configuration Options + +| Option | Default | Description | +|--------|---------|-------------| +| `port` | (required) | Target node name to fence | +| `pcmk_host_list` | (required) | Node this STONITH resource can fence | +| `request_dir` | `/var/run/fence_recorder/requests` | Directory for fence request files | +| `response_dir` | `/var/run/fence_recorder/responses` | Directory for fence response files | +| `log_dir` | `/var/log/cluster` | Directory for log files | + +### Verifying Configuration -HPE hardware implements software known as the Hardware System Supervisor (HSS), which itself conforms to the SNIA Redfish/Swordfish standard. This provides the means to manage hardware outside the host OS. +```bash +# Check STONITH status +pcs stonith status -### NNF Fencing +# View STONITH configuration +pcs stonith config -#### Source -The NNF Fencing agent is available at https://github.com/NearNodeFlash/fence-agents under the `nnf` branch. +# Test that fence_recorder can generate metadata +fence_recorder --action metadata +``` + +## Request/Response Protocol + +### Request File Format + +When a fence operation is triggered, `fence_recorder` writes a JSON request file: + +**Location**: `/-.json` + +```json +{ + "request_id": "550e8400-e29b-41d4-a716-446655440000", + "timestamp": "2025-01-10T14:30:00-06:00", + "action": "reboot", + "target_node": "rabbit-compute-1", + "recorder_node": "rabbit-node-1" +} +``` + +### Response File Format + +The NNF software writes a response file after processing: -```shell -git clone https://github.com/NearNodeFlash/fence-agents --branch nnf +**Location**: `/-.json` + +```json +{ + "request_id": "550e8400-e29b-41d4-a716-446655440000", + "success": true, + "action_performed": "reboot", + "target_node": "rabbit-compute-1", + "message": "Successfully fenced node", + "timestamp": "2025-01-10T14:30:15-06:00" +} ``` -#### Build -Refer to the `NNF.md file` at the root directory of the fence-agents repository. The fencing agents must be installed on every node in the cluster. +### Atomic File Writing + +Both request and response files use atomic rename to ensure consumers only see complete files: + +1. Write to temporary file: `..tmp` +2. Close the file +3. Rename to final name: `` -#### Setup -Configure the NNF agent with the following parameters: +File watchers should ignore files starting with `.` (hidden/temporary files). -| Argument | Definition | -| -------- | ---------- | -| `kubernetes-service-host=[ADDRESS]` | The IP address of the kubeapi server | -| `kubernetes-service-port=[PORT]` | The listening port of the kubeapi server | -| `service-token-file=[PATH]` | The location of the service token file. The file must be present on all nodes within the cluster | -| `service-cert-file=[PATH]` | The location of the service certificate file. The file must be present on all nodes within the cluster | -| `nnf-node-name=[NNF-NODE-NAME]` | Name of the NNF node as it is appears in the System Configuration | -| `api-version=[VERSION]` | The API Version of the NNF Node resource. Defaults to "v1alpha1" | +## Log Files -The token and certificate can be found in the Kubernetes Secrets resource for the nnf-system/nnf-fencing-agent ServiceAccount. This provides RBAC rules to limit the fencing agent to only the Kubernetes resources it needs access to. +Log files are written to the configured log directory: -For example, setting up the NNF fencing agent on `rabbit-node-1` with a kubernetes service API running at `192.168.0.1:6443` and the service token and certificate copied to `/etc/nnf/fence/`. This needs to be run on one node in the cluster. +| File | Format | Description | +|------|--------|-------------| +| `fence-events.log` | Timestamped text | Main operational log | +| `fence-events-readable.log` | Key=value | Grep-friendly format | +| `fence-events-detailed.jsonl` | JSON Lines | Machine-parseable format | +## Troubleshooting + +### Check STONITH Status + +```bash +# View current status +pcs stonith status + +# Check for failed resources +pcs status + +# View detailed configuration +pcs stonith config ``` -pcs stonith create rabbit-node-1 fence_nnf pcmk_host_list=rabbit-node-1 kubernetes-service-host=192.168.0.1 kubernetes-service-port=6443 service-token-file=/etc/nnf/fence/service.token service-cert-file=/etc/nnf/fence/service.cert nnf-node-name=rabbit-node-1 + +### Common Issues + +**STONITH resource stopped or failed:** + +```bash +# Clean up failed resource state +pcs resource cleanup + +# Check logs for errors +journalctl -u pacemaker | grep -i stonith ``` -#### Recovery -Since the NNF node is connected to 16 compute blades, careful coordination around fencing of a NNF node is required to minimize the impact of the outage. When a Rabbit node is fenced, the corresponding DWS Storage resource (`storages.dws.cray.hpe.com`) status changes. The workload manager must observe this change and follow the procedure below to recover from the fencing status. +**Timeout waiting for response:** -1. Observed the `storage.Status` changed and that `storage.Status.RequiresReboot == True` -2. Set the `storage.Spec.State := Disabled` -4. Wait for a change to the Storage status `storage.Status.State == Disabled` -5. Reboot the NNF node -6. Set the `storage.Spec.State := Enabled` -7. Wait for `storage.Status.State == Enabled` +```bash +# Check if NNF software is processing requests +ls -la /localdisk/fence-recorder/requests/ +ls -la /localdisk/fence-recorder/responses/ -### Compute Fencing +# Check fence_recorder logs +tail -f /var/log/cluster/fence-events.log +``` -The [Redfish fencing agent](https://github.com/ClusterLabs/fence-agents/tree/main/agents/redfish) from [ClusterLabs](https://github.com/ClusterLabs/fence-agents) should be used for Compute nodes in the cluster. It is also included at https://github.com/NearNodeFlash/fence-agents, and can be built at the same time as the NNF fencing agent. Configure the agent with the following parameters: +**Module not found error:** +Ensure the fencing library path is correct in `/usr/sbin/fence_recorder`. The `sys.path.append` line should point to `/usr/share/fence`. -| Argument | Definition | -| -------- | ---------- | -| `ip=[ADDRESS]` | The IP address or hostname of the HSS controller | -| `port=80` | The Port of the HSS controller. Must be `80` | -| `systems-uri=/redfish/v1/Systems/1` | The URI of the Systems object. Must be `/redfish/v1/Systems/1` | -| `ssl-insecure=true` | Instructs the use of an insecure SSL exchange. Must be `true` | -| `username=[USER]` | The user name for connecting to the HSS controller | -| `password=[PASSWORD]` | the password for connecting to the HSS controller | +### Testing Fence Operations -For example, setting up the Redfish fencing agent on `rabbit-compute-2` with the redfish service at `192.168.0.1`. This needs to be run on one node in the cluster. +```bash +# Test metadata generation +fence_recorder --action metadata -```shell -pcs stonith create rabbit-compute-2 fence_redfish pcmk_host_list=rabbit-compute-2 ip=192.168.0.1 systems-uri=/redfish/v1/Systems/1 username=root password=password ssl_insecure=true +# Test monitor action (non-destructive) +fence_recorder --action monitor -n rabbit-compute-1 \ + --request-dir=/localdisk/fence-recorder/requests \ + --response-dir=/localdisk/fence-recorder/responses ``` -### Dummy Fencing +## GFS2 File System Configuration + +After the cluster is configured with fencing, you can configure GFS2 file systems. See Red Hat documentation: + +- [Configuring a GFS2 File System in a Cluster](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_and_managing_high_availability_clusters/assembly_configuring-gfs2-in-a-cluster-configuring-and-managing-high-availability-clusters) + +## Dynamic Cluster Lifecycle + +Unlike traditional HA clusters that run continuously on all nodes, NNF manages Pacemaker cluster services dynamically on compute nodes based on workflow requirements. Cluster services run **continuously on the Rabbit** but are **started and stopped dynamically on compute nodes**. + +### When Cluster Services Start on Compute Nodes + +NNF software starts Pacemaker/Corosync cluster services on compute nodes when: + +1. A workflow requests GFS2 storage +2. The NNF software provisions the GFS2 file system +3. Compute nodes need to mount the shared storage + +At this point, cluster services are started on participating compute nodes, they join the Rabbit's cluster, fencing is enabled, and the GFS2 file system is mounted. + +> See [Storage Profiles](https://nearnodeflash.github.io/latest/guides/storage-profiles/readme/) for more information about how to start cluster services using the `PreActivate` command. + +### When Cluster Services Stop on Compute Nodes + +When the GFS2 workflow completes: + +1. The GFS2 file system is unmounted from all compute nodes +2. Storage resources are cleaned up +3. Pacemaker/Corosync cluster services are stopped on compute nodes +4. The Rabbit continues running cluster services + +> See [Storage Profiles](https://nearnodeflash.github.io/latest/guides/storage-profiles/readme/) for more information about how to stop cluster services using the `PostDeactivate` command. + +This dynamic lifecycle means: + +- **Rabbit runs cluster services continuously**: Always ready to accept compute nodes +- **Compute node services are transient**: They run only for the duration of GFS2 workflows +- **Resource efficiency**: Cluster overhead on compute nodes is incurred only when needed +- **Reduced complexity**: No long-running cluster services to maintain on compute nodes between jobs -The [dummy fencing agent](https://github.com/ClusterLabs/fence-agents/tree/main/agents/dummy) from ClusterLabs can be used for nodes in the cluster for an early access development system. +> **Note:** The cluster configuration (node membership, quorum votes, STONITH resources) is set up during system provisioning and persists on all nodes. Only the cluster *services* on compute nodes are started and stopped dynamically. -## Configuring a GFS2 file system in a cluster +## References -Follow steps 1-8 of the procedure from Red Hat: [Configuring a GFS2 file system in a cluster](https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/8/html/configuring_and_managing_high_availability_clusters/assembly_configuring-gfs2-in-a-cluster-configuring-and-managing-high-availability-clusters#doc-wrapper). +- [Red Hat: Configuring and Managing High Availability Clusters](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_and_managing_high_availability_clusters/) +- [Red Hat: Fencing in a Cluster](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_and_managing_high_availability_clusters/assembly_configuring-fencing-configuring-and-managing-high-availability-clusters) +- [ClusterLabs fence-agents](https://github.com/ClusterLabs/fence-agents) diff --git a/docs/guides/storage-profiles/readme.md b/docs/guides/storage-profiles/readme.md index 7a4e52ea..d53fd615 100644 --- a/docs/guides/storage-profiles/readme.md +++ b/docs/guides/storage-profiles/readme.md @@ -1,454 +1,905 @@ --- -authors: Nate Thornton , Dean Roehrich +authors: Nate Thornton , Dean Roehrich , Matt Richerson categories: provisioning --- -# Storage Profile Overview +# Storage Profiles -Storage Profiles allow for customization of the Rabbit storage provisioning process. Examples of content that can be customized via storage profiles is +Storage Profiles provide a way to customize how storage is provisioned and configured on Rabbit nodes. They allow administrators to define specific configurations for different file system types, RAID configurations, and storage layouts that users can select when submitting jobs. -1. The RAID type used for storage -2. Any mkfs or LVM args used -3. An external MGS NID for Lustre -4. A boolean value indicating the Lustre MGT and MDT should be combined on the same target device +## What Are Storage Profiles? -DW directives that allocate storage on Rabbit nodes allow a `profile` parameter to be specified to control how the storage is configured. NNF software provides a set of canned profiles to choose from, and the administrator may create more profiles. +An `NnfStorageProfile` is a Kubernetes Custom Resource that defines how storage should be configured on Rabbit nodes. Storage profiles control: -The administrator shall choose one profile to be the default profile that is used when a profile parameter is not specified. +- **File system type configuration** - Commands and options for XFS, GFS2, Raw, and Lustre file systems +- **Block device configuration** - LVM commands for creating physical volumes, volume groups, and logical volumes +- **RAID configurations** - Settings for redundant storage using LVM RAID or ZFS RAID +- **Target layouts** - How Lustre targets (MGT, MDT, OST) are distributed across Rabbit nodes +- **User commands** - Custom commands that run at various points in the storage lifecycle -## Specifying a Profile +Storage profiles are stored in the `nnf-system` namespace and are referenced by name in `#DW` directives. -To specify a profile name on a #DW directive, use the `profile` option +## Default vs. Non-Default Profiles -```shell -#DW jobdw type=lustre profile=durable capacity=5GB name=example -``` +### Default Profile -## Setting A Default Profile +Every NNF system must have exactly one storage profile marked as the default. The default profile is used when a `#DW` directive does not specify a profile. If zero or more than one profile is marked as default, new workflows will be rejected. -A default profile must be defined at all times. Any #DW line that does not specify a profile will use the default profile. If a default profile is not defined, then any new workflows will be rejected. If more than one profile is marked as default then any new workflows will be rejected. +A profile is marked as default by setting `data.default: true`: -To query existing profiles - -```shell -$ kubectl get nnfstorageprofiles -A -NAMESPACE NAME DEFAULT AGE -nnf-system durable true 14s -nnf-system performance false 6s +```yaml +apiVersion: nnf.cray.hpe.com/v1alpha9 +kind: NnfStorageProfile +metadata: + name: default + namespace: nnf-system +data: + default: true + # ... rest of profile configuration ``` -To set the default flag on a profile +### Querying Profiles + +To list all storage profiles and see which is the default: ```shell -kubectl patch nnfstorageprofile performance -n nnf-system --type merge -p '{"data":{"default":true}}' +kubectl get nnfstorageprofiles -n nnf-system ``` -To clear the default flag on a profile +Example output: -```shell -kubectl patch nnfstorageprofile durable -n nnf-system --type merge -p '{"data":{"default":false}}' +``` +NAME DEFAULT AGE +default true 14d +high-perf false 7d +durable false 7d +template false 14d ``` -## Creating The Initial Default Profile - -Create the initial default profile from scratch or by using the [NnfStorageProfile/template](https://github.com/NearNodeFlash/nnf-sos/blob/master/config/examples/nnf_v1alpha1_nnfstorageprofile.yaml) resource as a template. If `nnf-deploy` was used to install nnf-sos then the default profile described below will have been created automatically. +### Setting the Default Profile -To use the `template` resource begin by obtaining a copy of it either from the nnf-sos repo or from a live system. To get it from a live system use the following command: +To set a profile as the default: ```shell -kubectl get nnfstorageprofile -n nnf-system template -o yaml > profile.yaml +kubectl patch nnfstorageprofile high-perf -n nnf-system --type merge -p '{"data":{"default":true}}' ``` -Edit the `profile.yaml` file to trim the metadata section to contain only a name and namespace. The namespace must be left as nnf-system, but the name should be set to signify that this is the new default profile. In this example we will name it `default`. The metadata section will look like the following, and will contain no other fields: +To clear the default flag from a profile: -```yaml -metadata: - name: default - namespace: nnf-system +```shell +kubectl patch nnfstorageprofile default -n nnf-system --type merge -p '{"data":{"default":false}}' ``` -Mark this new profile as the default profile by setting `default: true` in the data section of the resource: +> **Note:** Ensure exactly one profile is marked as default at all times. -```yaml -data: - default: true -``` +## Specifying a Profile in #DW Directives -Apply this resource to the system and verify that it is the only one marked as the default resource: +To use a non-default storage profile, add the `profile` parameter to your `#DW` directive: ```shell -kubectl get nnfstorageprofile -A +#DW jobdw type=xfs profile=high-perf capacity=100GB name=my-storage ``` -The output will appear similar to the following: +```shell +#DW jobdw type=lustre profile=durable capacity=1TB name=my-lustre +``` ```shell -NAMESPACE NAME DEFAULT AGE -nnf-system default true 9s -nnf-system template false 11s +#DW create_persistent type=lustre profile=persistent-lustre capacity=10TB name=shared-fs ``` -The administrator should edit the `default` profile to record any cluster-specific settings. -Maintain a copy of this resource YAML in a safe place so it isn't lost across upgrades. +If no `profile` parameter is specified, the default profile is used. -### Keeping The Default Profile Updated +## File System Configuration -An upgrade of nnf-sos may include updates to the `template` profile. It may be necessary to manually copy these updates into the `default` profile. +Storage profiles contain configuration sections for each supported file system type: -## Profile Parameters +- `xfsStorage` - XFS file system configuration +- `gfs2Storage` - GFS2 file system configuration +- `rawStorage` - Raw block device configuration +- `lustreStorage` - Lustre file system configuration -### XFS +### XFS Storage -The following shows how to specify command line options for pvcreate, vgcreate, lvcreate, and mkfs for XFS storage. Optional mount options are specified one per line +XFS is a high-performance journaling file system suitable for single-node or exclusive access workloads. ```yaml -apiVersion: nnf.cray.hpe.com/v1alpha1 +apiVersion: nnf.cray.hpe.com/v1alpha9 kind: NnfStorageProfile metadata: - name: xfs-stripe-example + name: xfs-example namespace: nnf-system data: -[...] + default: false xfsStorage: - commandlines: - pvCreate: $DEVICE - vgCreate: $VG_NAME $DEVICE_LIST - lvCreate: -l 100%VG --stripes $DEVICE_NUM --stripesize=32KiB --name $LV_NAME $VG_NAME - mkfs: $DEVICE - options: - mountRabbit: - - noatime - - nodiratime -[...] + # Block device configuration (LVM) + blockDeviceCommands: + sharedVg: true + rabbitCommands: + pvCreate: $DEVICE + pvRemove: $DEVICE + vgCreate: --addtag $JOBID $VG_NAME $DEVICE_LIST + vgRemove: $VG_NAME + lvCreate: --zero n --activate n --size $LV_SIZE --stripes $DEVICE_NUM --stripesize=32KiB --name $LV_NAME $VG_NAME + lvRemove: $VG_NAME/$LV_NAME + lvChange: + activate: --activate y $VG_NAME/$LV_NAME + deactivate: --activate n $VG_NAME/$LV_NAME + computeCommands: + lvChange: + activate: --activate y $VG_NAME/$LV_NAME + deactivate: --activate n $VG_NAME/$LV_NAME + + # File system commands + fileSystemCommands: + rabbitCommands: + mkfs: $DEVICE + mount: $DEVICE $MOUNT_PATH + computeCommands: + mount: $DEVICE $MOUNT_PATH + + # User commands run during setup/teardown + userCommands: + postSetup: + - chown $USERID:$GROUPID $MOUNT_PATH + + # Capacity scaling factor (1.0 = request exactly what user specified) + capacityScalingFactor: "1.0" + + # Extra allocation padding for block device overhead + allocationPadding: 300MiB ``` -### GFS2 +### GFS2 Storage -The following shows how to specify command line options for pvcreate, lvcreate, and mkfs for GFS2. +GFS2 is a shared-disk cluster file system that allows multiple nodes to access the same file system simultaneously. ```yaml -apiVersion: nnf.cray.hpe.com/v1alpha1 +apiVersion: nnf.cray.hpe.com/v1alpha9 kind: NnfStorageProfile metadata: - name: gfs2-stripe-example + name: gfs2-example namespace: nnf-system data: -[...] + default: false gfs2Storage: - commandlines: - pvCreate: $DEVICE - vgCreate: $VG_NAME $DEVICE_LIST - lvCreate: -l 100%VG --stripes $DEVICE_NUM --stripesize=32KiB --name $LV_NAME $VG_NAME - mkfs: -j2 -p $PROTOCOL -t $CLUSTER_NAME:$LOCK_SPACE $DEVICE -[...] + blockDeviceCommands: + sharedVg: true + rabbitCommands: + pvCreate: $DEVICE + pvRemove: $DEVICE + # GFS2 requires shared VG with --shared flag + vgCreate: --shared --addtag $JOBID $VG_NAME $DEVICE_LIST + vgRemove: $VG_NAME + lvCreate: --zero n --activate n --size $LV_SIZE --stripes $DEVICE_NUM --stripesize=32KiB --name $LV_NAME $VG_NAME + lvRemove: $VG_NAME/$LV_NAME + lvChange: + # GFS2 uses shared activation (ys) + activate: --activate ys $VG_NAME/$LV_NAME + deactivate: --activate n $VG_NAME/$LV_NAME + vgChange: + lockStart: --lock-start $VG_NAME + lockStop: --lock-stop $VG_NAME + computeCommands: + lvChange: + activate: --activate ys $VG_NAME/$LV_NAME + deactivate: --activate n $VG_NAME/$LV_NAME + vgChange: + lockStart: --lock-start $VG_NAME + lockStop: --lock-stop $VG_NAME + + fileSystemCommands: + rabbitCommands: + # GFS2 mkfs requires journal count, protocol, cluster name, and lock space + mkfs: -j2 -p $PROTOCOL -t $CLUSTER_NAME:$LOCK_SPACE $DEVICE + mount: $DEVICE $MOUNT_PATH + computeCommands: + mount: $DEVICE $MOUNT_PATH + + userCommands: + postSetup: + - chown $USERID:$GROUPID $MOUNT_PATH + + capacityScalingFactor: "1.0" + allocationPadding: 300MiB ``` -### Lustre / ZFS +### Raw Storage -The following shows how to specify a zpool virtual device (vdev). In this case the default vdev is a stripe. See [zpoolconcepts(7)](https://openzfs.github.io/openzfs-docs/man/7/zpoolconcepts.7.html) for virtual device descriptions. +Raw storage provides direct block device access without a file system layer. ```yaml -apiVersion: nnf.cray.hpe.com/v1alpha1 +apiVersion: nnf.cray.hpe.com/v1alpha9 kind: NnfStorageProfile metadata: - name: zpool-stripe-example + name: raw-example namespace: nnf-system data: -[...] - lustreStorage: - mgtCommandlines: - zpoolCreate: -O canmount=off -o cachefile=none $POOL_NAME $DEVICE_LIST - mkfs: --mgs $VOL_NAME - mdtCommandlines: - zpoolCreate: -O canmount=off -o cachefile=none $POOL_NAME $DEVICE_LIST - mkfs: --mdt --fsname=$FS_NAME --mgsnode=$MGS_NID --index=$INDEX $VOL_NAME - mgtMdtCommandlines: - zpoolCreate: -O canmount=off -o cachefile=none $POOL_NAME $DEVICE_LIST - mkfs: --mgs --mdt --fsname=$FS_NAME --index=$INDEX $VOL_NAME - ostCommandlines: - zpoolCreate: -O canmount=off -o cachefile=none $POOL_NAME $DEVICE_LIST - mkfs: --ost --fsname=$FS_NAME --mgsnode=$MGS_NID --index=$INDEX $VOL_NAME -[...] + default: false + rawStorage: + blockDeviceCommands: + sharedVg: true + rabbitCommands: + pvCreate: $DEVICE + pvRemove: $DEVICE + vgCreate: --addtag $JOBID $VG_NAME $DEVICE_LIST + vgRemove: $VG_NAME + lvCreate: --zero n --activate n --size $LV_SIZE --stripes $DEVICE_NUM --stripesize=32KiB --name $LV_NAME $VG_NAME + lvRemove: $VG_NAME/$LV_NAME + lvChange: + activate: --activate y $VG_NAME/$LV_NAME + deactivate: --activate n $VG_NAME/$LV_NAME + computeCommands: + lvChange: + activate: --activate y $VG_NAME/$LV_NAME + deactivate: --activate n $VG_NAME/$LV_NAME + + fileSystemCommands: + rabbitCommands: + # Raw uses bind mount to expose the block device + mount: -o bind $DEVICE $MOUNT_PATH + computeCommands: + mount: -o bind $DEVICE $MOUNT_PATH + + capacityScalingFactor: "1.0" + allocationPadding: 300MiB ``` -#### ZFS dataset properties +### Lustre Storage -The following shows how to specify ZFS dataset properties in the `--mkfsoptions` arg for mkfs.lustre. See [zfsprops(7)](https://openzfs.github.io/openzfs-docs/man/7/zfsprops.7.html). +Lustre is a high-performance parallel distributed file system designed for large-scale cluster computing. ```yaml -apiVersion: nnf.cray.hpe.com/v1alpha1 +apiVersion: nnf.cray.hpe.com/v1alpha9 kind: NnfStorageProfile metadata: - name: zpool-stripe-example + name: lustre-example namespace: nnf-system data: -[...] + default: false lustreStorage: -[...] - ostCommandlines: + # Whether to combine MGT and MDT on the same target + combinedMgtMdt: true + + # Capacity for MGT device + capacityMgt: 5GiB + + # Capacity for MDT device (also used for combined MGT+MDT) + capacityMdt: 5GiB + + # MDT should not share a Rabbit with other targets + exclusiveMdt: false + + # Scaling factor for OST capacity + capacityScalingFactor: "1.0" + + # MGT target commands + mgtCommandlines: zpoolCreate: -O canmount=off -o cachefile=none $POOL_NAME $DEVICE_LIST - mkfs: --ost --mkfsoptions="recordsize=1024K -o compression=lz4" --fsname=$FS_NAME --mgsnode=$MGS_NID --index=$INDEX $VOL_NAME -[...] -``` - -#### Mount Options for Targets + mkfs: --mgs --backfstype=$BACKFS --mkfsoptions="nnf:jobid=$JOBID" $ZVOL_NAME + mountTarget: $ZVOL_NAME $MOUNT_PATH -##### Persistent Mount Options + # MDT target commands + mdtCommandlines: + zpoolCreate: -O canmount=off -o cachefile=none $POOL_NAME $DEVICE_LIST + mkfs: --mdt --backfstype=$BACKFS --fsname=$FS_NAME --mgsnode=$MGS_NID --index=$INDEX --mkfsoptions="nnf:jobid=$JOBID" $ZVOL_NAME + mountTarget: $ZVOL_NAME $MOUNT_PATH + postActivate: + - mountpoint $MOUNT_PATH -Use the mkfs.lustre `--mountfsoptions` parameter to set persistent mount options for Lustre targets. + # Combined MGT+MDT target commands + mgtMdtCommandlines: + zpoolCreate: -O canmount=off -o cachefile=none $POOL_NAME $DEVICE_LIST + mkfs: --mgs --mdt --backfstype=$BACKFS --fsname=$FS_NAME --index=$INDEX --mkfsoptions="nnf:jobid=$JOBID" $ZVOL_NAME + mountTarget: $ZVOL_NAME $MOUNT_PATH + postActivate: + - mountpoint $MOUNT_PATH -```yaml -apiVersion: nnf.cray.hpe.com/v1alpha1 -kind: NnfStorageProfile -metadata: - name: target-mount-option-example - namespace: nnf-system -data: -[...] - lustreStorage: -[...] + # OST target commands ostCommandlines: zpoolCreate: -O canmount=off -o cachefile=none $POOL_NAME $DEVICE_LIST - mkfs: --ost --mountfsoptions="errors=remount-ro,mballoc" --mkfsoptions="recordsize=1024K -o compression=lz4" --fsname=$FS_NAME --mgsnode=$MGS_NID --index=$INDEX $VOL_NAME -[...] + mkfs: --ost --backfstype=$BACKFS --fsname=$FS_NAME --mgsnode=$MGS_NID --index=$INDEX --mkfsoptions="nnf:jobid=$JOBID" $ZVOL_NAME + mountTarget: $ZVOL_NAME $MOUNT_PATH + postActivate: + - mountpoint $MOUNT_PATH + + # Client mount commands + clientCommandLines: + mountRabbit: $MGS_NID:/$FS_NAME $MOUNT_PATH + mountCompute: $MGS_NID:/$FS_NAME $MOUNT_PATH + rabbitPostSetup: + - lfs setstripe -E 64K -L mdt -E -1 -c -1 $MOUNT_PATH + + # Target placement options (see Target Layouts section) + mgtOptions: + colocateComputes: false + count: 1 + mdtOptions: + colocateComputes: false + count: 1 + mgtMdtOptions: + colocateComputes: false + count: 1 + ostOptions: + colocateComputes: true + scale: 5 + + # Commands to run on MGT after all targets are activated + preMountMGTCommands: + - lctl set_param -P osc.$FS_NAME-*.max_rpcs_in_flight=64 ``` -##### Non-Persistent Mount Options +## Target Layouts + +Target layout options control how Lustre targets (MGT, MDT, OST) are distributed across Rabbit nodes. These settings help optimize performance based on workload characteristics. + +### Layout Options + +Each target type (mgtOptions, mdtOptions, mgtMdtOptions, ostOptions) supports: -Non-persistent mount options can be specified with the ostOptions.mountTarget parameter to the NnfStorageProfile: +| Option | Description | +|--------|-------------| +| `count` | Static number of targets to create | +| `scale` | Dynamic value (1-10) that the WLM uses to determine target count | +| `colocateComputes` | If true, targets are placed on Rabbits connected to job's compute nodes | +| `storageLabels` | List of labels to restrict which Storage resources can be used | +> **Note:** Only one of `count` or `scale` can be set for each target type. + +### Understanding colocateComputes + +When `colocateComputes: true`: +- Storage is restricted to Rabbit nodes with physical connections to the job's compute nodes +- This typically means Rabbits in the same chassis as the compute nodes +- Best for minimizing network hops and maximizing bandwidth + +When `colocateComputes: false`: +- Storage can be placed on any available Rabbit node +- Useful for separating metadata targets from data targets +- Required for `create_persistent` directives since they may not have compute nodes + +### Scale vs Count + +**Scale** is useful when you want storage to automatically adjust based on job size: +- Value of 1: Minimum targets needed to satisfy capacity +- Value of 10: Maximum targets, potentially one per Rabbit connected to the job +- The WLM interprets scale values based on allocation size, compute count, and Rabbit count + +**Count** is useful when you need precise control: +- Specific number of targets regardless of job size +- Consistent performance characteristics across different jobs +- Useful for single-shared-file workloads with low metadata requirements + +### Example Layouts + +**High-performance scaled to job size:** ```yaml +ostOptions: + scale: 10 + colocateComputes: true +mdtOptions: + count: 2 + colocateComputes: true +``` -apiVersion: nnf.cray.hpe.com/v1alpha1 -kind: NnfStorageProfile -metadata: - name: target-mount-option-example - namespace: nnf-system -data: -[...] - lustreStorage: -[...] - ostCommandlines: - zpoolCreate: -O canmount=off -o cachefile=none $POOL_NAME $DEVICE_LIST - mkfs: --ost --mountfsoptions="errors=remount-ro" --mkfsoptions="recordsize=1024K -o compression=lz4" --fsname=$FS_NAME --mgsnode=$MGS_NID --index=$INDEX $VOL_NAME - ostOptions: - mountTarget: - - mballoc -[...] +**Static Configuration:** +```yaml +ostOptions: + count: 4 + colocateComputes: true +mdtOptions: + count: 1 + colocateComputes: true ``` -#### Target Layout +## RAID Configurations + +Storage profiles support RAID configurations for both LVM-based file systems (XFS, Raw) and ZFS-based Lustre targets. + +### LVM RAID (XFS and Raw) + +LVM RAID logical volumes provide redundancy for XFS and Raw allocations. -Users may want Lustre file systems with different performance characteristics. For example, a user job with a single compute node accessing the Lustre file system would see acceptable performance from a single OSS. An FPP workload might want as many OSSs as posible to avoid contention. +> **Note:** GFS2 cannot use RAID logical volumes because the LV is shared between multiple nodes. -The `NnfStorageProfile` allows admins to specify where and how many Lustre targets are allocated by the WLM. During the proposal phase of the workflow, the NNF software uses the information in the `NnfStorageProfile` to add extra constraints in the `DirectiveBreakdown`. The WLM uses these constraints when picking storage. +To create a RAID logical volume, specify `--type raid[x]`, `--activate y`, and `--nosync` in the `lvCreate` command: -The `NnfStorageProfile` has three fields in the `mgtOptions`, `mdtOptions`, and `ostOptions` to specify target layout. The fields are: +```yaml +xfsStorage: + blockDeviceCommands: + rabbitCommands: + pvCreate: $DEVICE + pvRemove: $DEVICE + vgCreate: --addtag $JOBID $VG_NAME $DEVICE_LIST + vgRemove: $VG_NAME + # RAID5 example: one parity device, remaining are data stripes + lvCreate: | + --activate y --zero n --nosync --type raid5 + --size $LV_SIZE --stripes $DEVICE_NUM-1 + --stripesize=32KiB --name $LV_NAME $VG_NAME + lvRemove: $VG_NAME/$LV_NAME + lvChange: + activate: --activate y $VG_NAME/$LV_NAME + deactivate: --activate n $VG_NAME/$LV_NAME + # Commands for rebuilding after drive replacement + lvmRebuild: + vgExtend: $VG_NAME $DEVICE + vgReduce: --removemissing $VG_NAME + lvRepair: $VG_NAME/$LV_NAME +``` -- `count` - A static value for how many Lustre targets to create. -- `scale` - A value from 1-10 that the WLM can use to determine how many Lustre targets to allocate. This is up to the WLM and the admins to agree on how to interpret this field. A value of 1 might indicate the minimum number of NNF nodes needed to reach the minimum capacity, while 10 might result in a Lustre target on every Rabbit attached to the computes in the job. Scale takes into account allocation size, compute node count, and Rabbit count. -- `colocateComputes` - true/false value. When "true", this adds a location constraint in the `DirectiveBreakdown` that limits the WLM to picking storage with a physical connection to the compute resources. In practice this means that Rabbit storage is restricted to the chassis used by the job. This can be set individually for each of the Lustre target types. When this is "false", any Rabbit storage can be picked, even if the Rabbit doesn't share a chassis with any of the compute nodes in the job. +> **Note:** The `--nosync` option allows the RAID volume to be used immediately without waiting for initial synchronization. -Only one of `scale` and `count` can be set for a particular target type. +### ZFS RAID (Lustre) -The `DirectiveBreakdown` for `create_persistent` #DWs won't include the constraint from `colocateCompute=true` since there may not be any compute nodes associated with the job. +ZFS RAID provides redundancy for Lustre targets using zpool virtual devices. ```yaml -apiVersion: nnf.cray.hpe.com/v1alpha1 -kind: NnfStorageProfile -metadata: - name: high-metadata - namespace: default -data: - default: false -... - lustreStorage: - combinedMgtMdt: false - capacityMdt: 500GiB - capacityMgt: 1GiB -[...] - ostOptions: - scale: 5 - colocateComputes: true - mdtOptions: - count: 10 +lustreStorage: + ostCommandlines: + # RAIDZ example (single parity, similar to RAID5) + zpoolCreate: -O canmount=off -o cachefile=none $POOL_NAME raidz $DEVICE_LIST + mkfs: --ost --backfstype=$BACKFS --fsname=$FS_NAME --mgsnode=$MGS_NID --index=$INDEX $ZVOL_NAME + mountTarget: $ZVOL_NAME $MOUNT_PATH + # Command for replacing a failed device + zpoolReplace: $POOL_NAME $OLD_DEVICE $NEW_DEVICE ``` -##### Example Layouts +## User Commands -`scale` with `colocateComputes=true` will likely be the most common layout type to use for `jobdw` directives. This will result in a Lustre file system whose performance scales with the number of compute nodes in the job. +Storage profiles support custom commands that run at various points during the storage lifecycle. These allow administrators to customize behavior beyond the standard provisioning steps. -`count` may be used when a specific performance characteristic is desired such as a single shared file workload that has low metadata requirements and only needs a single MDT. It may also be useful when a consistently performing file system is required across different jobs. +On the Rabbit nodes, all user command output is logged to the nnf-node-manager log. On the compute nodes, all user command output is logged to the system log. If a user command returns a non-zero return code, the stderr output is also logged, and the workflow will enter a `TransientCondition` state. -`colocatedComputes=false` may be useful for placing MDTs on NNF nodes without an OST (within the same file system). +There are two categories of user commands with different execution contexts: -The `count` field may be useful when creating a persistent file system since the job with the `create_persistent` directive may only have a single compute node. +1. **Block Device and File System Commands** - For compute nodes, these run during the PreRun and PostRun phases. For Rabbit nodes, these Run during PreRun, PostRun, DataIn, and DataOut phases depending on which other DW directives are specified (data movement and user containers). +2. **Storage-Level Commands** - Run during storage setup and teardown phases -In general, `scale` gives a simple way for users to get a filesystem that has performance consistent with their job size. `count` is useful for times when a user wants full control of the file system layout. +### Block Device and File System User Commands -### RAID Configurations +The user commands in `blockDeviceCommands` and `fileSystemCommands` are run when storage is being activated/deactivated and mounted/unmounted for use: -Allocations can be set up to use a RAID device to provide continued access in the event of a drive failure. Optionally, commands can be specified to rebuild the RAID device after a replacement drive has been added. The storage profile parameters differ depending on whether the allocation is using LVM or zpool. +| Location | Workflow Phase | Description | +|----------|----------------|-------------| +| Rabbit | DataIn | Storage is activated and mounted for data staging into the allocation | +| Rabbit | DataOut | Storage is activated and mounted for data staging out of the allocation | +| Rabbit | PreRun | Storage is activated and mounted for access in a user container | +| Rabbit | PostRun | Storage is unmounted and deactivated after the user container exits | +| Compute | PreRun | Storage is activated and mounted before the user's application runs | +| Compute | PostRun | Storage is unmounted and deactivated after the user's application completes | -#### Zpool +These commands are useful for operations that need to happen each time storage is accessed, such as: +- Setting up environment-specific configurations +- Running health checks before/after use +- Synchronizing data or caches -To create a Lustre confiuration with a redundant zpool, the `raidz` option is required in `zpoolCreate` command. To allow the zpool to be rebuilt with a new drive, the `zpoolReplace` command is required. +#### Block Device User Commands -The example below shows a RAID configuration for the OST, however, the same options can be specified for any of the Lustre targets. +```yaml +xfsStorage: + blockDeviceCommands: + # Commands run on Rabbit during PreRun/PostRun/DataIn/DataOut phases + rabbitCommands: + userCommands: + preActivate: + - echo "Rabbit: About to activate block device" + postActivate: + - echo "Rabbit: Block device activated" + preDeactivate: + - echo "Rabbit: About to deactivate block device" + postDeactivate: + - echo "Rabbit: Block device deactivated" + # Commands run on Compute during PreRun/PostRun phases + computeCommands: + userCommands: + preActivate: + - echo "Compute: About to activate block device" + postActivate: + - echo "Compute: Block device activated" + preDeactivate: + - echo "Compute: About to deactivate block device" + postDeactivate: + - echo "Compute: Block device deactivated" +``` + +#### File System User Commands ```yaml -apiVersion: nnf.cray.hpe.com/v1alpha8 -kind: NnfStorageProfile -metadata: - name: lustre-raid-example - namespace: nnf-system -data: -[...] - lustreStorage: -[...] - ostCommandlines: - mkfs: --ost --backfstype=$BACKFS --fsname=$FS_NAME --mgsnode=$MGS_NID --index=$INDEX - --mkfsoptions="nnf:jobid=$JOBID" $ZVOL_NAME - mountTarget: $ZVOL_NAME $MOUNT_PATH - postActivate: - - mountpoint $MOUNT_PATH - zpoolCreate: -O canmount=off -o cachefile=none $POOL_NAME raidz $DEVICE_LIST - zpoolReplace: $POOL_NAME $OLD_DEVICE $NEW_DEVICE +xfsStorage: + fileSystemCommands: + # Commands run on Rabbit during DataIn/DataOut phases + rabbitCommands: + userCommands: + preMount: + - echo "Rabbit: About to mount file system" + postMount: + - echo "Rabbit: File system mounted" + preUnmount: + - echo "Rabbit: About to unmount file system" + postUnmount: + - echo "Rabbit: File system unmounted" + # Commands run on Compute during PreRun/PostRun phases + computeCommands: + userCommands: + preMount: + - echo "Compute: About to mount file system" + postMount: + - echo "Compute: File system mounted" + preUnmount: + - echo "Compute: About to unmount file system" + postUnmount: + - echo "Compute: File system unmounted" ``` -#### LVM +### Storage-Level User Commands -A RAID logical volume can be used with XFS and Raw allocations. -NOTE: gfs2 allocations cannot use RAID logical volumes because the LV is shared. +The `userCommands` section at the storage level (e.g., `xfsStorage.userCommands`) contains commands that run during the **setup and teardown phases** of the workflow. These run on the Rabbit nodes when storage is first provisioned and when it is finally destroyed. -To create a redundant LV, `--type raid[x]` and `--nosync` should be specified in the `lvcreate` command. Also, the `--stripes` parameter should be adjusted accordingly to specify the number of data stripes. For `raid5`, `$DEVICE_NUM-1` is used. +| Command | Phase | Description | +|---------|-------|-------------| +| `postSetup` | Setup | Runs after storage is fully provisioned and the file system is mounted on the Rabbit | +| `preTeardown` | Teardown | Runs before storage is destroyed, while the file system is still mounted | +| `postActivate` | Setup | Runs after the file system is activated during initial setup | +| `preDeactivate` | Teardown | Runs before the file system is deactivated during final teardown | -To allow the LV to rebuild after a drive is replaced, `vgExtend`, `lvRepair`, and `vgReduce` should be specified in the `lvmRebuild` section. +These commands are useful for one-time operations such as: +- Setting ownership and permissions on newly created storage +- Initializing directory structures +- Cleaning up or archiving data before destruction ```yaml -apiVersion: nnf.cray.hpe.com/v1alpha8 -kind: NnfStorageProfile -metadata: - name: xfs-raid-example - namespace: nnf-system -data: -[...] - xfsStorage: - commandlines: - lvChange: - activate: --activate y $VG_NAME/$LV_NAME - deactivate: --activate n $VG_NAME/$LV_NAME - lvmRebuild: - vgExtend: $VG_NAME $DEVICE - vgReduce: --removemissing $VG_NAME - lvRepair: $VG_NAME/$LV_NAME - lvCreate: --activate n --zero n --nosync --type raid5 --extents $PERCENT_VG --stripes $DEVICE_NUM-1 - --stripesize=32KiB --name $LV_NAME $VG_NAME - lvRemove: $VG_NAME/$LV_NAME - mkfs: $DEVICE - mountCompute: $DEVICE $MOUNT_PATH - mountRabbit: $DEVICE $MOUNT_PATH - postMount: - - chown $USERID:$GROUPID $MOUNT_PATH - pvCreate: $DEVICE - pvRemove: $DEVICE - sharedVg: true - vgChange: - lockStart: --lock-start $VG_NAME - lockStop: --lock-stop $VG_NAME - vgCreate: --shared --addtag $JOBID $VG_NAME $DEVICE_LIST - vgRemove: $VG_NAME +xfsStorage: + userCommands: + # Run once after storage is fully set up (file system mounted on Rabbit) + postSetup: + - chown $USERID:$GROUPID $MOUNT_PATH + - chmod 750 $MOUNT_PATH + - mkdir -p $MOUNT_PATH/input $MOUNT_PATH/output + + # Run once before storage is torn down (file system still mounted) + preTeardown: + - echo "Final cleanup of $MOUNT_PATH" + + # Run once after file system is activated during setup + postActivate: + - echo "File system activated for first time" + + # Run once before file system is deactivated during teardown + preDeactivate: + - echo "About to deactivate file system for last time" +``` + +### Lustre-Specific User Commands + +User commands can also be specified for Lustre file systems, however there are no block device activate/deactivate hooks. + +```yaml +lustreStorage: + # Commands for each target type + mgtCommandlines: + postActivate: + - mountpoint $MOUNT_PATH + preDeactivate: + - echo "Deactivating MGT" + + mdtCommandlines: + postActivate: + - mountpoint $MOUNT_PATH + + ostCommandlines: + postActivate: + - mountpoint $MOUNT_PATH + + # Client-side commands + clientCommandLines: + rabbitPreMount: + - echo "About to mount Lustre client on Rabbit" + rabbitPostMount: + - lfs setstripe -c -1 $MOUNT_PATH + rabbitPreUnmount: + - sync + rabbitPostUnmount: + - echo "Lustre client unmounted" + + computePreMount: + - echo "About to mount Lustre client on Compute" + computePostMount: + - echo "Lustre client mounted" + computePreUnmount: + - sync + computePostUnmount: + - echo "Lustre client unmounted" + + # Setup/teardown with Lustre client mounted on Rabbit + rabbitPostSetup: + - lfs setstripe -E 64K -L mdt -E -1 -c -1 $MOUNT_PATH + rabbitPreTeardown: + - lfs getstripe $MOUNT_PATH + + # Commands run on MGT after all targets are up + preMountMGTCommands: + - lctl set_param -P osc.$FS_NAME-*.max_rpcs_in_flight=64 + - lctl set_param -P osc.$FS_NAME-*.max_dirty_mb=2000 +``` ## Command Line Variables -### global -- `$JOBID` - expands to the Job ID from the Workflow -- `$USERID` - expands to the User ID of the user who submitted the job -- `$GROUPID` - expands to the Group ID of the user who submitted the job +Storage profile commands can use variables that are expanded at runtime. Variables use the `$VARIABLE_NAME` syntax. -### LVM PV commands +### Global Variables -- `$DEVICE` - expands to the `/dev/` value for one device that has been allocated +Available in all commands: -### LVM VG commands +| Variable | Description | +|----------|-------------| +| `$JOBID` | Job ID from the Workflow | +| `$USERID` | User ID of the job submitter | +| `$GROUPID` | Group ID of the job submitter | + +### LVM Variables -- `$VG_NAME` - expands to a volume group name that is controlled by Rabbit software. -- `$DEVICE_LIST` - expands to a list of space-separated `/dev/` devices. This list will contain the devices that were iterated over for the pvcreate step. -- `$DEVICE_NUM` - expands to the count of devices in `$DEVICE_LIST` -- `$DEVICE_NUM-1` - expands to the count of devices in `$DEVICE_LIST` minus 1 -- `$DEVICE_NUM-2` - expands to the count of devices in `$DEVICE_LIST` minus 2 -- `$DEVICE` - expands to the name of a new device. This is used by `vgextend` when repairing a RAID device +#### Physical Volume Commands -### LVM LV Commands +| Variable | Description | +|----------|-------------| +| `$DEVICE` | Path to allocated device (e.g., `/dev/nvme0n1`) | -- `$VG_NAME` - see vgcreate above. -- `$LV_NAME` - expands to a logical volume name that is controlled by Rabbit software. -- `$DEVICE_NUM` - expands to a number indicating the number of devices allocated for the volume group. -- `$DEVICE_NUM-1` - expands to a number indicating the number of devices allocated for the volume group minus 1. -- `$DEVICE_NUM-2` - expands to a number indicating the number of devices allocated for the volume group minus 2. -- `$DEVICE1, $DEVICE2, ..., $DEVICEn` - each expands to one of the devices from the `$DEVICE_LIST` above. -- `$PERCENT_VG` - expands to the size that each LV should be based on a percentage of the total VG size -- `$LV_SIZE` - expands to the size of the LV in kB in the format expected by `lvcreate` +#### Volume Group Commands -### XFS mkfs +| Variable | Description | +|----------|-------------| +| `$VG_NAME` | Volume group name (controlled by Rabbit software) | +| `$DEVICE_LIST` | Space-separated list of devices | +| `$DEVICE_NUM` | Count of devices | +| `$DEVICE_NUM-1` | Device count minus 1 (for RAID5) | +| `$DEVICE_NUM-2` | Device count minus 2 (for RAID6) | +| `$DEVICE` | New device path (used in vgExtend for RAID rebuild) | -- `$DEVICE` - expands to the `/dev/` value for the logical volume that was created by the lvcreate step above. +#### Logical Volume Commands -### GFS2 mkfs +| Variable | Description | +|----------|-------------| +| `$VG_NAME` | Volume group name | +| `$LV_NAME` | Logical volume name | +| `$DEVICE_NUM` | Count of devices | +| `$DEVICE_NUM-1` | Device count minus 1 | +| `$DEVICE_NUM-2` | Device count minus 2 | +| `$DEVICE1`, `$DEVICE2`, ..., `$DEVICEn` | Individual devices from `$DEVICE_LIST` | +| `$PERCENT_VG` | Size as percentage of VG | +| `$LV_SIZE` | Size in kB format for lvcreate | -- `$DEVICE` - expands to the `/dev/` value for the logical volume that was created by the lvcreate step above. -- `$CLUSTER_NAME` - expands to a cluster name that is controlled by Rabbit Software -- `$LOCK_SPACE` - expands to a lock space key that is controlled by Rabbit Software. -- `$PROTOCOL` - expands to a locking protocol that is controlled by Rabbit Software. +### File System Variables -### zpool create +#### XFS/Raw mkfs -- `$DEVICE_LIST` - expands to a list of space-separated `/dev/` devices. This list will contain the devices that were allocated for this storage request. -- `$POOL_NAME` - expands to a pool name that is controlled by Rabbit software. -- `$DEVICE_NUM` - expands to a number indicating the number of devices allocated for this storage request. -- `$DEVICE1, $DEVICE2, ..., $DEVICEn` - each expands to one of the devices from the `$DEVICE_LIST` above. +| Variable | Description | +|----------|-------------| +| `$DEVICE` | Path to the logical volume device | -### zpool replace +#### GFS2 mkfs -- `$DEVICE_NUM` - expands to a number indicating the number of devices allocated for this storage request. -- `$DEVICE_NUM-1` - expands to a number indicating the number of devices allocated for this storage request minus 1. -- `$DEVICE_NUM-2` - expands to a number indicating the number of devices allocated for this storage request minus 2. -- `$DEVICE_LIST` - expands to a list of space-separated `/dev/` devices. This list will contain the devices that were allocated for this storage request. -- `$POOL_NAME` - expands to a pool name that is controlled by Rabbit software. -- `$OLD_DEVICE` - expands to the name of a device that is degraded -- `$NEW_DEVICE` - expands to the name of a new device that can replace the degraded device +| Variable | Description | +|----------|-------------| +| `$DEVICE` | Path to the logical volume device | +| `$CLUSTER_NAME` | Cluster name (controlled by Rabbit software) | +| `$LOCK_SPACE` | Lock space key (controlled by Rabbit software) | +| `$PROTOCOL` | Locking protocol (controlled by Rabbit software) | -### lustre mkfs +#### Mount/Unmount -- `$FS_NAME` - expands to the filesystem name that was passed to Rabbit software from the workflow's #DW line. -- `$MGS_NID` - expands to the NID of the MGS. If the MGS was orchestrated by nnf-sos then an appropriate internal value will be used. -- `$ZVOL_NAME` - expands to the volume name that will be created. This value will be `/`, and is controlled by Rabbit software. -- `$INDEX` - expands to the index value of the target and is controlled by Rabbit software. -- `$TARGET_NAME` - expands to the name of the lustre target of the form `[fsname]-[target-type][index]` (e.g., `mylus-OST0003`) -- `$BACKFS` - expands to the type of file system backing the Lustre target +| Variable | Description | +|----------|-------------| +| `$DEVICE` | Device path to mount | +| `$MOUNT_PATH` | Path to mount on | -### Mount/Unmount +### ZFS/Lustre Variables -- `$DEVICE` - expands to the device path to mount -- `$MOUNT_PATH` - expands to the path to mount on +#### zpool create -### PostMount/PreUnmount and PostActivate/PreDeactivate +| Variable | Description | +|----------|-------------| +| `$POOL_NAME` | Pool name (controlled by Rabbit software) | +| `$DEVICE_LIST` | Space-separated list of devices | +| `$DEVICE_NUM` | Count of devices | +| `$DEVICE1`, `$DEVICE2`, ..., `$DEVICEn` | Individual devices | -- `$MOUNT_PATH` - expands to the mount path of the fileystem to perform certain actions on the mounted filesystem +#### zpool replace -#### Lustre Specific +| Variable | Description | +|----------|-------------| +| `$POOL_NAME` | Pool name | +| `$DEVICE_LIST` | List of devices | +| `$DEVICE_NUM`, `$DEVICE_NUM-1`, `$DEVICE_NUM-2` | Device counts | +| `$OLD_DEVICE` | Degraded device to replace | +| `$NEW_DEVICE` | Replacement device | -These variables are for lustre only and can be used to perform PostMount activities such are setting lustre striping. +#### Lustre mkfs -- `$NUM_MDTS` - expands to the number of MDTs for the lustre filesystem -- `$NUM_MGTS` - expands to the number of MGTs for the lustre filesystem -- `$NUM_MGTMDTS` - expands to the number of combined MGTMDTs for the lustre filesystem -- `$NUM_OSTS` - expands to the number of OSTs for the lustre filesystem -- `$NUM_NNFNODES` - expands to the number of NNF Nodes for the lustre filesystem +| Variable | Description | +|----------|-------------| +| `$FS_NAME` | Lustre fsname picked by NNF software | +| `$MGS_NID` | NID of the MGS | +| `$ZVOL_NAME` | ZFS volume name (`pool/dataset`) | +| `$INDEX` | Target index number | +| `$TARGET_NAME` | Target name (e.g., `mylus-OST0003`) | +| `$BACKFS` | Backing file system type | -### NnfSystemStorage specific +#### Lustre Client -- `$COMPUTE_HOSTNAME` - Expands to the hostname of the compute node that will use the allocation. This can be used to add a tag during the lvcreate +| Variable | Description | +|----------|-------------| +| `$MGS_NID` | NID of the MGS | +| `$FS_NAME` | File system name | +| `$MOUNT_PATH` | Client mount path | +| `$NUM_MDTS` | Number of MDTs | +| `$NUM_MGTS` | Number of MGTs | +| `$NUM_MGTMDTS` | Number of combined MGT/MDTs | +| `$NUM_OSTS` | Number of OSTs | +| `$NUM_NNFNODES` | Number of NNF nodes | + +### NnfSystemStorage Variables + +For system storage allocations: + +| Variable | Description | +|----------|-------------| +| `$COMPUTE_HOSTNAME` | Hostname of the compute node using the allocation | + +### User Command Variables + +Different variables are available depending on which user command hook is being executed. + +#### Block Device User Commands + +The following variables are available to `blockDeviceCommands.rabbitCommands.userCommands` and `blockDeviceCommands.computeCommands.userCommands` (preActivate, postActivate, preDeactivate, postDeactivate): + +| Variable | Description | +|----------|-------------| +| `$JOBID` | Job ID from the Workflow | +| `$USERID` | User ID of the job submitter | +| `$GROUPID` | Group ID of the job submitter | +| `$VG_NAME` | Volume group name | +| `$LV_NAME` | Logical volume name | + +#### File System User Commands + +The following variables are available to `fileSystemCommands.rabbitCommands.userCommands` and `fileSystemCommands.computeCommands.userCommands` (preMount, postMount, preUnmount, postUnmount): + +| Variable | Description | +|----------|-------------| +| `$JOBID` | Job ID from the Workflow | +| `$USERID` | User ID of the job submitter | +| `$GROUPID` | Group ID of the job submitter | +| `$DEVICE` | Device path being mounted | +| `$MOUNT_PATH` | Path where the file system is mounted | + +#### Storage-Level User Commands + +The following variables are available to `userCommands` at the storage level (postSetup, preTeardown, postActivate, preDeactivate): + +| Variable | Description | +|----------|-------------| +| `$JOBID` | Job ID from the Workflow | +| `$USERID` | User ID of the job submitter | +| `$GROUPID` | Group ID of the job submitter | +| `$MOUNT_PATH` | Path where the file system is mounted | + +#### Lustre Target User Commands + +The following variables are available to Lustre target commands (mgtCommandlines, mdtCommandlines, mgtMdtCommandlines, ostCommandlines) for postActivate and preDeactivate: + +| Variable | Description | +|----------|-------------| +| `$JOBID` | Job ID from the Workflow | +| `$MOUNT_PATH` | Path where the target is mounted | +| `$FS_NAME` | Lustre file system name | +| `$TARGET_NAME` | Target name (e.g., `mylus-OST0003`) | + +#### Lustre Client User Commands + +The following variables are available to `clientCommandLines` user commands: + +**For rabbitPreMount, rabbitPostMount, rabbitPreUnmount, rabbitPostUnmount, computePreMount, computePostMount, computePreUnmount, computePostUnmount:** + +| Variable | Description | +|----------|-------------| +| `$JOBID` | Job ID from the Workflow | +| `$USERID` | User ID of the job submitter | +| `$GROUPID` | Group ID of the job submitter | +| `$MGS_NID` | NID of the MGS | +| `$FS_NAME` | Lustre file system name | +| `$MOUNT_PATH` | Path where the client is mounted | + +**For rabbitPostSetup and rabbitPreTeardown:** + +| Variable | Description | +|----------|-------------| +| `$JOBID` | Job ID from the Workflow | +| `$USERID` | User ID of the job submitter | +| `$GROUPID` | Group ID of the job submitter | +| `$MGS_NID` | NID of the MGS | +| `$FS_NAME` | Lustre file system name | +| `$MOUNT_PATH` | Path where the client is mounted | +| `$NUM_MDTS` | Number of MDTs | +| `$NUM_MGTS` | Number of MGTs | +| `$NUM_MGTMDTS` | Number of combined MGT/MDTs | +| `$NUM_OSTS` | Number of OSTs | +| `$NUM_NNFNODES` | Number of NNF nodes | + +#### Lustre preMountMGTCommands + +The following variables are available to `preMountMGTCommands`: + +| Variable | Description | +|----------|-------------| +| `$JOBID` | Job ID from the Workflow | +| `$FS_NAME` | Lustre file system name | + +## Advanced Configuration + +### External MGS + +To use an existing external MGS instead of creating one: + +```yaml +lustreStorage: + # Use existing MGS by NID + externalMgs: "10.0.0.1@tcp" + + # Or reference an MGS pool created with standaloneMgtPoolName + externalMgs: "pool:my-mgs-pool" +``` + +### ZFS Dataset Properties + +Set ZFS properties via `--mkfsoptions`: + +```yaml +lustreStorage: + ostCommandlines: + mkfs: --ost --mkfsoptions="recordsize=1024K -o compression=lz4" --fsname=$FS_NAME --mgsnode=$MGS_NID --index=$INDEX $ZVOL_NAME +``` + +### Persistent Lustre Mount Options + +Use `--mountfsoptions` in mkfs for persistent mount options: + +```yaml +lustreStorage: + ostCommandlines: + mkfs: --ost --mountfsoptions="errors=remount-ro,mballoc" --fsname=$FS_NAME --mgsnode=$MGS_NID --index=$INDEX $ZVOL_NAME ``` -lvCreate --zero n --activate n --extents $PERCENT_VG --addtag $COMPUTE_HOSTNAME ... -``` \ No newline at end of file + +### Capacity Scaling and Padding + +```yaml +xfsStorage: + # Request 10% more capacity than specified by user + capacityScalingFactor: "1.1" + + # Add fixed padding for LVM/filesystem overhead + allocationPadding: 500MiB +``` + +### Storage Labels + +Restrict allocations to specific storage resources: + +```yaml +lustreStorage: + ostOptions: + storageLabels: + - high-performance + - nvme-only +``` + +## Pinned Profiles + +When a workflow references a storage profile, the NNF software creates a "pinned" copy of the profile. This ensures that: + +- Profile changes don't affect running workflows +- The exact configuration is preserved for the workflow's lifetime +- Profiles marked as `pinned: true` cannot also be `default: true` + +**Note:** Do not manually set `pinned: true` on profiles you create. \ No newline at end of file diff --git a/docs/guides/user-interactions/readme.md b/docs/guides/user-interactions/readme.md index cf67336e..5dc06b23 100644 --- a/docs/guides/user-interactions/readme.md +++ b/docs/guides/user-interactions/readme.md @@ -15,17 +15,18 @@ Once the job is running on compute nodes, the application can find access to Rab ### jobdw -The `jobdw` directive command tells the Rabbit software to create a file system on the Rabbit hardware for the lifetime of the user's job. At the end of the job, any data that is not moved off of the file system either by the application or through a `copy_out` directive will be lost. Multiple `jobdw` directives can be listed in the same job script. +The `jobdw` directive command tells the Rabbit software to create a file system on the Rabbit hardware for the lifetime of the user's job. At the end of the job, any data that is not moved off of the file system either by the application or through a `copy_out` directive is lost. Multiple `jobdw` directives can be listed in the same job script. #### Command Arguments + | Argument | Required | Value | Notes | |----------|----------|-------|-------| -| `type` | Yes | `raw`, `xfs`, `gfs2`, `lustre` | Type defines how the storage should be formatted. For Lustre file systems, a single file system is created that is mounted by all computes in the job. For raw, xfs, and GFS2 storage, a separate file system is allocated for each compute node. | +| `type` | Yes | `raw`, `xfs`, `gfs2`, `lustre` | Type defines how the storage should be formatted. For Lustre file systems, a single file system is created that is mounted by all computes in the job. For raw, xfs, and GFS2 storage, a separate file system is allocated for each compute node. | | `capacity` | Yes | Allocation size with units. `1TiB`, `100GB`, etc. | Capacity interpretation varies by storage type. For Lustre file systems, capacity is the aggregate OST capacity. For raw, xfs, and GFS2 storage, capacity is the capacity of the file system for a single compute node. Capacity suffixes are: `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB` | | `name` | Yes | String including numbers and '-' | This is a name for the storage allocation that is unique within a job | -| `profile` | No | Profile name | This specifies which profile to use when allocating storage. Profiles include `mkfs` and `mount` arguments, file system layout, and many other options. Profiles are created by admins. When no profile is specified, the default profile is used. More information about storage profiles can be found in the [Storage Profiles](../storage-profiles/readme.md) guide. | +| `profile` | No | Profile name | This specifies which profile to use when allocating storage. Profiles include `mkfs` and `mount` arguments, file system layout, and many other options. When no profile is specified, the default profile is used. More information about storage profiles can be found in the [Storage Profiles](../storage-profiles/readme.md) guide. **Note:** Admins are responsible for profiles.| | `requires` | No | `copy-offload` | Use this option with [Copy Offload](../data-movement/copy-offload.md). This is for users who want to initiate data movement to or from the Rabbit storage from within their application. | -| `requires` | No | `user-container-auth` | Use this option with [User Containers](../user-containers/readme.md) that have an application that expects to use the same kind of TLS certificate and [per-Workflow token](../data-movement/copy-offload.md#per-workflow-token) that is configured for [Copy Offload](../data-movement/copy-offload.md). | +| `requires` | No | `user-container-auth` | Use this option with [User Containers](../user-containers/readme.md) that have an application that expects to use the same kind of TLS certificate and [per-Workflow token](../data-movement/copy-offload.md#wlm-and-the-per-workflow-token) that is configured for [Copy Offload](../data-movement/copy-offload.md). | #### Examples @@ -52,12 +53,13 @@ This directive results in a 50GB GFS2 file system created for each compute node The `create_persistent` command results in a storage allocation on the Rabbit nodes that lasts beyond the lifetime of the job. This is useful for creating a file system that can share data between jobs. Only a single `create_persistent` directive is allowed in a job, and it cannot be in the same job as a `destroy_persistent` directive. See [persistentdw](readme.md#persistentdw) to utilize the storage in a job. #### Command Arguments + | Argument | Required | Value | Notes | |----------|----------|-------|-------| -| `type` | Yes | `raw`, `xfs`, `gfs2`, `lustre` | Type defines how the storage should be formatted. For Lustre file systems, a single file system is created. For raw, xfs, and GFS2 storage, a separate file system is allocated for each compute node in the job. | +| `type` | Yes | `raw`, `xfs`, `gfs2`, `lustre` | Type defines how the storage should be formatted. For Lustre file systems, a single file system is created. For raw, xfs, and GFS2 storage, a separate file system is allocated for each compute node in the job. | | `capacity` | Yes | Allocation size with units. `1TiB`, `100GB`, etc. | Capacity interpretation varies by storage type. For Lustre file systems, capacity is the aggregate OST capacity. For raw, xfs, and GFS2 storage, capacity is the capacity of the file system for a single compute node. Capacity suffixes are: `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB` | | `name` | Yes | Lowercase string including numbers and '-' | This is a name for the storage allocation that is unique within the system | -| `profile` | No | Profile name | This specifies which profile to use when allocating storage. Profiles include `mkfs` and `mount` arguments, file system layout, and many other options. Profiles are created by admins. When no profile is specified, the default profile is used. The profile used when creating the persistent storage allocation is the same profile used by jobs that use the persistent storage. More information about storage profiles can be found in the [Storage Profiles](../storage-profiles/readme.md) guide.| +| `profile` | No | Profile name | This specifies which profile to use when allocating storage. Profiles include `mkfs` and `mount` arguments, file system layout, and many other options. When no profile is specified, the default profile is used. The profile used when creating the persistent storage allocation is the same profile used by jobs that use the persistent storage. More information about storage profiles can be found in the [Storage Profiles](../storage-profiles/readme.md) guide. **Note:** Admins are responsible for profiles.| #### Examples @@ -74,12 +76,14 @@ This directive results in a 100GiB xfs file system created for each compute node This directive results in a single 10TiB Lustre file system being created that can be accessed later by any compute nodes in the system. Multiple jobs can access a Rabbit Lustre file system at the same time. This job can be scheduled with a single compute node (or zero compute nodes if the WLM allows), without any limitations on compute node counts for subsequent jobs using the persistent Lustre file system. ### destroy_persistent -The `destroy_persistent` command will delete persistent storage that was allocated by a corresponding `create_persistent`. If the persistent storage is currently in use by a job, then the job containing the `destroy_persistent` command will fail. Only a single `destroy_persistent` directive is allowed in a job, and it cannot be in the same job as a `create_persistent` directive. + +The `destroy_persistent` command deletes persistent storage that was allocated by a corresponding `create_persistent`. If the persistent storage is currently in use by a job, then the job containing the `destroy_persistent` command fails. Only a single `destroy_persistent` directive is allowed in a job, and it cannot be in the same job as a `create_persistent` directive. #### Command Arguments + | Argument | Required | Value | Notes | |----------|----------|-------|-------| -| `name` | Yes | Lowercase string including numbers and '-' | This is a name for the persistent storage allocation that will be destroyed | +| `name` | Yes | Lowercase string including numbers and '-' | This is a name for the persistent storage allocation that is destroyed | #### Examples @@ -87,19 +91,21 @@ The `destroy_persistent` command will delete persistent storage that was allocat #DW destroy_persistent name=shared-data ``` -This directive will delete the persistent storage allocation with the name `shared-data` +This directive deletes the persistent storage allocation with the name `shared-data` ### persistentdw + The `persistentdw` command makes an existing persistent storage allocation available to a job. The persistent storage must already be created from a `create_persistent` command in a different job script. Multiple `persistentdw` commands can be used in the same job script to request access to multiple persistent allocations. Persistent Lustre file systems can be accessed from any compute nodes in the system, and the compute node count for the job can vary as needed. Multiple jobs can access a persistent Lustre file system concurrently if desired. Raw, xfs, and GFS2 file systems can only be accessed by compute nodes that have a physical connection to the Rabbits hosting the storage, and jobs accessing these storage types must have the same compute node count as the job that made the persistent storage. #### Command Arguments + | Argument | Required | Value | Notes | |----------|----------|-------|-------| -| `name` | Yes | Lowercase string including numbers and '-' | This is a name for the persistent storage that will be accessed | +| `name` | Yes | Lowercase string including numbers and '-' | This is a name for the persistent storage that is accessed | | `requires` | No | `copy-offload` | Use this option with [Copy Offload](../data-movement/copy-offload.md). This is for users who want to initiate data movement to or from the Rabbit storage from within their application. | -| `requires` | No | `user-container-auth` | Use this option with [User Containers](../user-containers/readme.md) that have an application that expects to use the same kind of TLS certificate and [per-Workflow token](../data-movement/copy-offload.md#per-workflow-token) that is configured for [Copy Offload](../data-movement/copy-offload.md). | +| `requires` | No | `user-container-auth` | Use this option with [User Containers](../user-containers/readme.md) that have an application that expects to use the same kind of TLS certificate and [per-Workflow token](../data-movement/copy-offload.md#wlm-and-the-per-workflow-token) that is configured for [Copy Offload](../data-movement/copy-offload.md). | #### Examples @@ -111,14 +117,15 @@ This directive causes the `shared-data` persistent storage allocation to be moun ### copy_in/copy_out -The `copy_in` and `copy_out` directives are used to move data to and from the storage allocations on Rabbit nodes. The `copy_in` directive requests that data be moved into the Rabbit file system before application launch, and the `copy_out` directive requests data to be moved off of the Rabbit file system after application exit. This is different from data-movement that is requested through the copy-offload API, which occurs during application runtime. Multiple `copy_in` and `copy_out` directives can be included in the same job script. More information about data movement can be found in the [Data Movement](../data-movement/readme.md) documentation. +The `copy_in` and `copy_out` directives are used to move data to and from the storage allocations on Rabbit nodes. The `copy_in` directive requests that data be moved into the Rabbit file system before application launch, and the `copy_out` directive requests data to be moved off of the Rabbit file system after application exit. Multiple `copy_in` and `copy_out` directives can be included in the same job script. This is different from data-movement that is requested through the copy-offload API, which occurs during application runtime. More information about data movement can be found in the [Data Movement](../data-movement/readme.md) documentation. #### Command Arguments -| Argument | Required | Value | Notes | -|----------|----------|-------|-------| + +|Argument|Required|Value|Notes| +|--------|----------|-------|-------| | `source` | Yes | `[path]`, `$DW_JOB_[name]/[path]`, `$DW_PERSISTENT_[name]/[path]` | `[name]` is the name of the Rabbit persistent or job storage as specified in the `name` argument of the `jobdw` or `persistentdw` directive. Any `'-'` in the name from the `jobdw` or `persistentdw` directive should be changed to a `'_'` in the `copy_in` and `copy_out` directive. | | `destination` | Yes | `[path]`, `$DW_JOB_[name]/[path]`, `$DW_PERSISTENT_[name]/[path]` | `[name]` is the name of the Rabbit persistent or job storage as specified in the `name` argument of the `jobdw` or `persistentdw` directive. Any `'-'` in the name from the `jobdw` or `persistentdw` directive should be changed to a `'_'` in the `copy_in` and `copy_out` directive. | -| `profile` | No | Profile name | This specifies which profile to use when copying data. Profiles specify the copy command to use, MPI arguments, and how output gets logged. If no profile is specified then the default profile is used. Profiles are created by an admin. | +| `profile` | No | Profile name | This specifies the profile to use when copying data. Profiles specify the copy command to use, MPI arguments, and how output is logged. If no profile is specified then the default profile is used. More information about datamovement profiles can be found in the [DataMovement Profiles](../data-movement/readme.md#data-movement-profiles) guide. **Note:** Admins are responsible for profiles.| #### Examples @@ -156,13 +163,14 @@ This set of directives makes use of a persistent storage allocation and a job st The `container` directive is used to launch user containers on the Rabbit nodes. The containers have access to `jobdw`, `persistentdw`, or global Lustre storage as specified in the `container` directive. More documentation for user containers can be found in the [User Containers](../user-containers/readme.md) guide. Only a single `container` directive is allowed in a job. #### Command Arguments + | Argument | Required | Value | Notes | |----------|----------|-------|-------| | `name` | Yes | Lowercase string including numbers and '-' | This is a name for the container instance that is unique within a job | -| `profile` | Yes | Profile name | This specifies which container profile to use. The container profile contains information about which container to run, which file system types to expect, which network ports are needed, and many other options. An admin is responsible for creating the container profiles. | -| `DW_JOB_[expected]` | No | `jobdw` storage allocation `name` | The container profile will list `jobdw` file systems that the container requires. `[expected]` is the name as specified in the container profile | -| `DW_PERSISTENT_[expected]` | No | `persistentdw` storage allocation `name` | The container profile will list `persistentdw` file systems that the container requires. `[expected]` is the name as specified in the container profile | -| `DW_GLOBAL_[expected]` | No | Global lustre path | The container profile will list global Lustre file systems that the container requires. `[expected]` is the name as specified in the container profile | +| `profile` | Yes | Profile name | This specifies which container profile to use. The container profile contains information about which container to run, which file system types to expect, which network ports are needed, and many other options. **Note:** Admins are responsible for profiles.| +| `DW_JOB_[expected]` | No | `jobdw` storage allocation `name` | The container profile lists `jobdw` file systems that the container requires. `[expected]` is the name as specified in the container profile | +| `DW_PERSISTENT_[expected]` | No | `persistentdw` storage allocation `name` | The container profile lists `persistentdw` file systems that the container requires. `[expected]` is the name as specified in the container profile | +| `DW_GLOBAL_[expected]` | No | Global lustre path | The container profile lists global Lustre file systems that the container requires. `[expected]` is the name as specified in the container profile | #### Examples @@ -179,8 +187,8 @@ The WLM makes a set of environment variables available to the job application ru | Environment Variable | Value | Notes | |----------------------|-------|-------| -| `DW_JOB_[name]` | Mount path of a `jobdw` file system | `[name]` is from the `name` argument in the `jobdw` directive. Any `'-'` characters in the `name` will be converted to `'_'` in the environment variable. There will be one of these environment variables per `jobdw` directive in the job. | -| `DW_PERSISTENT_[name]` | Mount path of a `persistentdw` file system | `[name]` is from the `name` argument in the `persistentdw` directive. Any `'-'` characters in the `name` will be converted to `'_'` in the environment variable. There will be one of these environment variables per `persistentdw` directive in the job. | +| `DW_JOB_[name]` | Mount path of a `jobdw` file system | `[name]` is from the `name` argument in the `jobdw` directive. Any `'-'` characters in the `name` are converted to `'_'` in the environment variable. There is one of these environment variables per `jobdw` directive in the job. | +| `DW_PERSISTENT_[name]` | Mount path of a `persistentdw` file system | `[name]` is from the `name` argument in the `persistentdw` directive. Any `'-'` characters in the `name` are converted to `'_'` in the environment variable. There is one of these environment variables per `persistentdw` directive in the job. | | `NNF_CONTAINER_PORTS` | Comma separated list of ports | These ports are used together with the IP address of the local Rabbit to communicate with a user container specified by a `container` directive. More information can be found in the [User Containers](../user-containers/readme.md) guide. | | `DW_WORKFLOW_NAME` | Name of the Workflow | | | `DW_WORKFLOW_NAMESPACE` | Namespace of the Workflow | | diff --git a/mkdocs.yml b/mkdocs.yml index 53d4209c..f155b799 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,4 +1,4 @@ -# Release: 0.1.23 +# Release: 0.1.24 site_name: NNF site_description: 'Near Node Flash' docs_dir: docs/ @@ -34,7 +34,7 @@ nav: - 'CRD Version Bumper': 'repo-guides/crd-bumper/readme.md' - 'Editing APIs': 'repo-guides/crd-bumper/editing-apis.md' - 'CRD Upgrade Helpers': 'repo-guides/release-nnf-sw/crd-upgrade-helpers.md' - - 'Upgrade Tester': 'repo-guides/release-nnf-sw/upgrade-tester.md' + - 'Upgrade Tester': 'repo-guides/testing/upgrade-tester.md' theme: name: 'material' custom_dir: overrides @@ -51,6 +51,8 @@ theme: - search.share logo: img/logo.png favicon: img/logo.png +extra_css: + - stylesheets/extra.css plugins: - search: separator: '[\s\-,:!=\[\]()"/]+|(?!\b)(?=[A-Z][a-z])|\.(?!\d)|&[lg]t;' diff --git a/overrides/stylesheets/extra.css b/overrides/stylesheets/extra.css new file mode 100644 index 00000000..24665717 --- /dev/null +++ b/overrides/stylesheets/extra.css @@ -0,0 +1,27 @@ +/* Prevent word wrapping only in the first column (Argument) */ +.md-typeset table th:first-child, +.md-typeset table td:first-child { + white-space: nowrap; + word-break: keep-all; + min-width: 120px; +} + +/* Ensure code elements in the first column don't wrap */ +.md-typeset table th:first-child code, +.md-typeset table td:first-child code { + white-space: nowrap; + word-break: keep-all; +} + +/* Allow normal wrapping for other columns (especially Notes) */ +.md-typeset table th:not(:first-child), +.md-typeset table td:not(:first-child) { + white-space: normal; + word-break: normal; +} + +/* Keep table layout flexible */ +.md-typeset table { + table-layout: auto; + width: 100%; +} \ No newline at end of file