diff --git a/_topic_maps/_topic_map.yml b/_topic_maps/_topic_map.yml index 22d744080bd7..054c0008d333 100644 --- a/_topic_maps/_topic_map.yml +++ b/_topic_maps/_topic_map.yml @@ -1637,6 +1637,24 @@ Topics: Topics: - Name: DPU Operator File: dpu-operator + - Name: NVIDIA DPF Operator + Dir: dpf_operator + Distros: openshift-enterprise,openshift-origin + Topics: + - Name: About the NVIDIA DPF Operator + File: about-dpf-operator + - Name: Setting up the environment for DPF + File: dpf-environment-setup + - Name: Installing and configuring the DPF Operator + File: dpf-operator-installation + - Name: Provisioning the DPU hosted cluster + File: dpf-hosted-cluster-provisioning + - Name: Adding worker nodes and provisioning DPUs + File: dpf-worker-nodes-provisioning + - Name: Validating traffic and configuring telemetry + File: dpf-validation-telemetry + - Name: Troubleshooting DPF + File: dpf-troubleshooting - Name: Network Observability Operator Dir: network_observability_operator Distros: openshift-enterprise,openshift-origin diff --git a/images/nw-dpf-assisted-installer-dpu-interface.png b/images/nw-dpf-assisted-installer-dpu-interface.png new file mode 100644 index 000000000000..c94bfccf54d6 Binary files /dev/null and b/images/nw-dpf-assisted-installer-dpu-interface.png differ diff --git a/images/nw-dpf-lab-topology.png b/images/nw-dpf-lab-topology.png new file mode 100644 index 000000000000..bdbd2fffa85d Binary files /dev/null and b/images/nw-dpf-lab-topology.png differ diff --git a/modules/nw-dpf-adding-worker-nodes-overview.adoc b/modules/nw-dpf-adding-worker-nodes-overview.adoc new file mode 100644 index 000000000000..8be88c2fb21d --- /dev/null +++ b/modules/nw-dpf-adding-worker-nodes-overview.adoc @@ -0,0 +1,10 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-worker-nodes-provisioning.adoc + +:_mod-docs-content-type: CONCEPT +[id="adding-worker-nodes_{context}"] += Worker nodes + +[role="_abstract"] +You can add DPU-equipped worker nodes to the management cluster by using the Assisted Installer or the Bare Metal Operator. diff --git a/modules/nw-dpf-adding-workers-assisted-installer.adoc b/modules/nw-dpf-adding-workers-assisted-installer.adoc new file mode 100644 index 000000000000..bd8c5400b812 --- /dev/null +++ b/modules/nw-dpf-adding-workers-assisted-installer.adoc @@ -0,0 +1,38 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-worker-nodes-provisioning.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-adding-workers-assisted-installer_{context}"] += Add worker nodes by using the Assisted Installer + +[role="_abstract"] +You can add DPU-equipped worker nodes to the management cluster by using the Assisted Installer on the {hybrid-console}. + +.Prerequisites + +* You have access to the management cluster as a user with the `cluster-admin` role. +* The management cluster is registered on the {hybrid-console}. +* You have iDRAC or BMC access to the worker node. +* The worker node meets the hardware and network prerequisites. For details, see "DPF hardware requirements" and "DPF network infrastructure requirements". + +.Procedure + +. Log in to the link:https://console.redhat.com/openshift[{hybrid-console}]. + +. Select your cluster from the cluster list. + +. Click *Add hosts*. + +. Click *Download Discovery ISO* and save the discovery ISO to your local system. + +. Upload the ISO to the worker node by using the iDRAC or BMC interface and boot the server from the ISO. + +. Wait for the node to boot from the discovery ISO. + +. After the node is discovered, note the DPU interface name from the Assisted Installer interface. +This value is used for the `DPU_P0` environment variable. ++ +image::nw-dpf-assisted-installer-dpu-interface.png[Assisted Installer showing DPU interface name] + +. Click *Install ready host* to start the installation and monitor the installation progress. diff --git a/modules/nw-dpf-adding-workers-baremetal-operator.adoc b/modules/nw-dpf-adding-workers-baremetal-operator.adoc new file mode 100644 index 000000000000..68e49073e058 --- /dev/null +++ b/modules/nw-dpf-adding-workers-baremetal-operator.adoc @@ -0,0 +1,226 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-worker-nodes-provisioning.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-adding-workers-baremetal-operator_{context}"] += Add worker nodes by using the Bare Metal Operator + +[role="_abstract"] +You can add DPU-equipped worker nodes to the management cluster by using the Bare Metal Operator to automate provisioning through the cluster API. + +.Prerequisites + +* You have access to the management cluster as a user with the `cluster-admin` role. +* The OpenShift CLI (`oc`) is installed. +* The Bare Metal Operator is installed on the management cluster. +* Physical worker servers with Redfish-compatible BMC, iDRAC, or iLO access are available. +* Network connectivity exists from the management cluster to the worker BMC interfaces. +* You have the BMC IP address and access credentials for each server. +* You have the MAC address of the management network interface for each server. +* You have the name of the root disk device for each server. + +.Procedure + +. Set the following environment variables for the worker node: ++ +[source,terminal] +---- +$ export BMC_IP= +$ export BMC_USER= +$ export BMC_PASSWORD= +$ export WORKER_NAME= +$ export BOOT_MAC= +$ export ROOT_DEVICE= +---- ++ +where: ++ +-- +``:: Specifies the IP address of the worker node BMC interface. +``:: Specifies the username for BMC access. +``:: Specifies the password for BMC access. +``:: Specifies a name for the worker node, such as `worker-01`. +``:: Specifies the MAC address of the out-of-band management interface, such as `00:00:5E:00:53:01`. +``:: Specifies the path to the root disk device, such as `/dev/nvme0n1`. +-- + +. Verify BMC connectivity from one of the control plane nodes: ++ +[source,terminal] +---- +$ ping $BMC_IP +---- ++ +[source,terminal] +---- +$ curl -k https://$BMC_IP/redfish/v1/ +---- ++ +[source,terminal] +---- +$ curl -k -u $BMC_USER:$BMC_PASSWORD https://$BMC_IP/redfish/v1/Systems +---- + +. Verify that the Bare Metal Operator is available: ++ +[source,terminal] +---- +$ oc get clusteroperator baremetal +---- ++ +.Example output +[source,terminal] +---- +NAME VERSION AVAILABLE PROGRESSING DEGRADED SINCE MESSAGE +baremetal 4.22.x True False False ... +---- + +. Create a file named `provisioning.yaml` with the following content to disable the provisioning network: ++ +[source,yaml] +---- +apiVersion: metal3.io/v1alpha1 +kind: Provisioning +metadata: + name: provisioning-configuration +spec: + provisioningNetwork: "Disabled" + watchAllNamespaces: false +---- ++ +[IMPORTANT] +==== +When `provisioningNetwork` is set to `Disabled`, servers boot by using Redfish virtual media instead of PXE. +==== + +. Apply the `Provisioning` resource: ++ +[source,terminal] +---- +$ oc apply -f provisioning.yaml +---- + +. Create a file named `bmc-secret.yaml` with the following content to store the BMC credentials: ++ +[source,yaml] +---- +apiVersion: v1 +kind: Secret +metadata: + name: ${WORKER_NAME}-bmc-secret + namespace: openshift-machine-api +type: Opaque +stringData: + username: ${BMC_USER} + password: ${BMC_PASSWORD} +---- + +. Apply the BMC credentials secret: ++ +[source,terminal] +---- +$ envsubst < bmc-secret.yaml | oc apply -f - +---- + +. Create a file named `machineset-dpu.yaml` with the following content: ++ +[source,yaml] +---- +apiVersion: machine.openshift.io/v1beta1 +kind: MachineSet +metadata: + name: worker-dpu + namespace: openshift-machine-api +spec: + replicas: 1 + selector: + matchLabels: + machine.openshift.io/cluster-api-machineset: worker-dpu + template: + metadata: + labels: + machine.openshift.io/cluster-api-machineset: worker-dpu + node-role.kubernetes.io/worker-dpu: "" + spec: + metadata: + labels: + node-role.kubernetes.io/worker-dpu: "" + providerSpec: + value: + hostSelector: + matchLabels: + dpu-capable: "true" + customDeploy: + method: install_coreos + userData: + name: worker-dpu-user-data-managed + namespace: openshift-machine-api +---- ++ +The `MachineSet` must be created before the `BareMetalHost` resources. +It automatically selects `BareMetalHost` resources that are labeled `dpu-capable: "true"`, provisions them, and applies the `node-role.kubernetes.io/worker-dpu=""` label. +Set `replicas` to match the number of DPU worker nodes. + +. Apply the `MachineSet` resource: ++ +[source,terminal] +---- +$ oc apply -f machineset-dpu.yaml +---- ++ +.Example output +[source,terminal] +---- +machineset.machine.openshift.io/worker-dpu created +---- + +. Create a file named `baremetalhost.yaml` with the following content: ++ +[source,yaml] +---- +apiVersion: metal3.io/v1alpha1 +kind: BareMetalHost +metadata: + name: $WORKER_NAME + namespace: openshift-machine-api + labels: + dpu-capable: "true" + annotations: + inspect.metal3.io: disabled +spec: + online: false + bootMACAddress: $BOOT_MAC + rootDeviceHints: + deviceName: $ROOT_DEVICE + bmc: + address: redfish-virtualmedia+https://$BMC_IP + credentialsName: $WORKER_NAME-bmc-secret + disableCertificateVerification: true +---- + +. Apply the `BareMetalHost` resource: ++ +[source,terminal] +---- +$ envsubst < baremetalhost.yaml | oc apply -f - +---- + +.Verification + +* Monitor the provisioning progress: ++ +[source,terminal] +---- +$ oc get bmh -n openshift-machine-api -w +---- ++ +.Example output +[source,terminal] +---- +NAME STATE CONSUMER ONLINE ERROR AGE +worker-01 registering true 10s +worker-01 available true 30s +worker-01 provisioning true 1m +worker-01 provisioned true 10m +---- diff --git a/modules/nw-dpf-adjusting-ovnk-cni-overview.adoc b/modules/nw-dpf-adjusting-ovnk-cni-overview.adoc new file mode 100644 index 000000000000..7d0221c2ebba --- /dev/null +++ b/modules/nw-dpf-adjusting-ovnk-cni-overview.adoc @@ -0,0 +1,10 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-worker-nodes-provisioning.adoc + +:_mod-docs-content-type: CONCEPT +[id="ovnk-cni-adjustments_{context}"] += OVN-Kubernetes CNI settings for DPF + +[role="_abstract"] +You must adjust the cluster CNI so that it supports DPU acceleration. diff --git a/modules/nw-dpf-approving-dpu-csrs.adoc b/modules/nw-dpf-approving-dpu-csrs.adoc new file mode 100644 index 000000000000..e42410226e85 --- /dev/null +++ b/modules/nw-dpf-approving-dpu-csrs.adoc @@ -0,0 +1,72 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-worker-nodes-provisioning.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-approving-dpu-csrs_{context}"] += Approve DPU node CSRs + +[role="_abstract"] +You must approve the pending certificate signing requests (CSRs) for DPU nodes on the hosted cluster so that the DPU nodes can join the hosted cluster and complete provisioning. + +[NOTE] +==== +When you use the `dpf-hcp-provisioner-operator`, DPU CSR approval is handled automatically. +Manual approval is provided as a fallback if automatic approval is not functioning. +==== + +.Prerequisites + +* You have access to the hosted cluster as a user with the `cluster-admin` role. +* The OpenShift CLI (`oc`) is installed. +* The `KUBECONFIG` environment variable is set to the hosted cluster kubeconfig file. +* DPU provisioning has reached the `DPU Cluster Config` stage. + +.Procedure + +. Watch for pending CSRs from the DPU nodes: ++ +[source,terminal] +---- +$ oc get csr -w +---- ++ +The DPU node name typically follows the pattern `-`. + +. Approve all pending CSRs: ++ +[source,terminal] +---- +$ oc get csr -o go-template='{{range .items}}{{if not .status}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}' | xargs oc adm certificate approve +---- ++ +.Example output +[source,terminal] +---- +certificatesigningrequest.certificates.k8s.io/csr-6jx22 approved +certificatesigningrequest.certificates.k8s.io/csr-tb6nd approved +---- ++ +Repeat this step until no pending CSRs remain. + +.Verification + +* Verify that the DPU nodes joined the hosted cluster and are in a `Ready` state: ++ +[source,terminal] +---- +$ oc get nodes +---- ++ +.Example output +[source,terminal] +---- +NAME STATUS ROLES AGE VERSION +host-worker1-mt2413xz0b65 Ready worker 2m48s v1.33.5 +host-worker2-mt2413xz0aw6 Ready worker 2m45s v1.33.5 +---- ++ +[NOTE] +==== +After the DPU nodes join the hosted cluster, the DPU provisioning process continues to the remaining stages. +==== diff --git a/modules/nw-dpf-approving-worker-csrs.adoc b/modules/nw-dpf-approving-worker-csrs.adoc new file mode 100644 index 000000000000..5819805911a5 --- /dev/null +++ b/modules/nw-dpf-approving-worker-csrs.adoc @@ -0,0 +1,95 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-worker-nodes-provisioning.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-approving-worker-csrs_{context}"] += Approve worker node CSRs and apply DPU labels + +[role="_abstract"] +You must approve the pending certificate signing requests (CSRs) for worker nodes that join the management cluster and then label the nodes for the DPU `MachineConfigPool`. + +[NOTE] +==== +When you add worker nodes by using the Bare Metal Operator, CSRs are typically auto-approved and the `worker-dpu` label is applied automatically through the `MachineSet` template. +Manual approval is required when you use the Assisted Installer or as a fallback if auto-approval is not functioning. +==== + +.Prerequisites + +* You have access to the management cluster as a user with the `cluster-admin` role. +* The OpenShift CLI (`oc`) is installed. +* Worker nodes are booted and attempting to join the management cluster. + +.Procedure + +. Watch for pending CSRs: ++ +[source,terminal] +---- +$ oc get csr -w +---- + +. Approve all pending CSRs: ++ +[source,terminal] +---- +$ oc get csr -o go-template='{{range .items}}{{if not .status}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}' | xargs oc adm certificate approve +---- ++ +.Example output +[source,terminal] +---- +certificatesigningrequest.certificates.k8s.io/csr-27bgq approved +certificatesigningrequest.certificates.k8s.io/csr-69g65 approved +certificatesigningrequest.certificates.k8s.io/csr-7r862 approved +certificatesigningrequest.certificates.k8s.io/csr-f5vk7 approved +---- ++ +Repeat this step until no pending CSRs remain. +Each node typically generates multiple CSRs. + +. Verify that the worker nodes joined the cluster: ++ +[source,terminal] +---- +$ oc get nodes +---- ++ +.Example output +[source,terminal] +---- +NAME STATUS ROLES AGE VERSION +host-worker1 NotReady worker 68s v1.35 +host-worker2 NotReady worker 75s v1.35 +master-0 Ready control-plane,master,worker 4d22h v1.35 +master-1 Ready control-plane,master,worker 4d21h v1.35 +master-2 Ready control-plane,master,worker 4d22h v1.35 +---- ++ +[NOTE] +==== +The worker nodes show a status of `NotReady` until the DPU provisioning process is fully completed and all OVN-Kubernetes CNI components on the host and the DPU are running. +Do not proceed to the next steps until all pending CSRs are approved. +==== + +. Label each worker node for the DPU `MachineConfigPool`: ++ +In the following command, replace `` with the name of the worker node: ++ +[source,terminal] +---- +$ oc label node node-role.kubernetes.io/worker-dpu="" +---- ++ +.Example output +[source,terminal] +---- +node/ labeled +---- ++ +[NOTE] +==== +This label triggers the Machine Config Operator to apply the DPU-specific `MachineConfig` and reboot the node. +Wait for the reboot to complete before proceeding. +==== diff --git a/modules/nw-dpf-architecture-overview.adoc b/modules/nw-dpf-architecture-overview.adoc new file mode 100644 index 000000000000..8534f05bfcb5 --- /dev/null +++ b/modules/nw-dpf-architecture-overview.adoc @@ -0,0 +1,61 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/about-dpf-operator.adoc + +:_mod-docs-content-type: CONCEPT +[id="nw-dpf-architecture-overview_{context}"] += DPF architecture overview + +[role="_abstract"] +The NVIDIA DOCA Platform Framework (DPF) v26.4 deployment on {product-title} {product-version} creates a dual-cluster topology that offloads OVN-Kubernetes data plane operations to NVIDIA BlueField-3 DPUs. + +In Host Trusted mode, DPF uses BlueField DPUs as host accelerators. Administrators can orchestrate both workloads and DPU-accelerated infrastructure by using standard {product-title} APIs and custom resource definitions (CRDs). + +By offloading critical {product-title} networking functions, such as OVN-Kubernetes, to the DPU, the architecture frees host CPU resources for tenant applications. DPF also provides automated lifecycle management so that administrators can provision, configure, and update fleets of DPUs directly from {product-title}. + +== The topology + +The DPF deployment creates a specialized networking infrastructure consisting of two distinct cluster planes that work together to deliver hardware-accelerated networking: + +{product-title} management cluster:: +The management cluster runs on the server's main x86 CPU cores. It hosts the actual business logic, such as AI workloads or enterprise applications. ++ +The management cluster serves as the primary administrative interface and the host cluster that provisions and manages both the fleet of DPUs and user workloads. In a Host Trusted deployment, the worker nodes in this cluster are the physical servers that house the BlueField DPUs. ++ +The management cluster is responsible for the following functions: ++ +* Running user workloads on x86 host processors. +* Providing a single interface for defining the required state of the infrastructure by using Kubernetes CRDs such as `DPUSet`, `DPUDeployment`, and `DPUService`. +* Driving the discovery of DPUs, flashing BlueField Bootstream (BFB) images, and configuring host-to-DPU networking. +* Coordinating the deployment of services and network flows to the DPU cluster. + +{product-title} DPU hosted cluster:: +The DPU hosted cluster is a dedicated, secondary Kubernetes control plane for managing the fleet of NVIDIA BlueField DPUs. The DPUs function as the worker nodes of this hosted cluster, separate from the bare-metal hosts they are physically attached to. ++ +The DPU hosted cluster runs the following components: ++ +* OVN-Kubernetes running in DPU mode to offload flows. +* DOCA services such as Host-Based Networking (HBN) for BGP routing, DOCA Telemetry Service (DTS) for monitoring, and Firefly for time synchronization. +* System components including NVIDIA IPAM, Multus, and SR-IOV Device Plugins to manage the DPU hardware resources. + +== Example lab topology + +The following diagram illustrates the physical connectivity for a reference lab environment. It serves as a baseline example to demonstrate the core components and their interactions. + +image::nw-dpf-lab-topology.png[DPF lab topology diagram] + +== Architecture characteristics + +Host-managed DPU lifecycle:: +In a Host Trusted deployment, the DPU is managed from the host. This model enables cloud operators to manage BlueField-bound services directly from their standard {product-title} control plane. DPF automates the discovery and provisioning of DPUs: the DPF Operator detects worker nodes, creates DPU objects, and deploys the DOCA Management Service (DMS) to install the BFB firmware and configure networking between the host and DPU. This approach reduces manual low-level device configuration by using standard Kubernetes APIs and workflows. + +Infrastructure service offloading:: +In Host Trusted mode, infrastructure services such as networking, storage, and security are offloaded from the host CPU to the DPU. This frees host CPU resources for applications. The framework routes data center traffic through dedicated ports on the BlueField DPU. + +Kubernetes-native orchestration:: +DPF extends the Kubernetes control plane to the DPUs, enabling administrators to deploy and orchestrate NVIDIA DOCA services and third-party applications directly on the BlueField DPU by using familiar Kubernetes constructs. The architecture supports automated rolling updates, scaling, and rollbacks for services without disrupting ongoing operations. + +[NOTE] +==== +In Host Trusted mode, the host is part of the trusted domain. +==== diff --git a/modules/nw-dpf-component-placement.adoc b/modules/nw-dpf-component-placement.adoc new file mode 100644 index 000000000000..891f96e1f344 --- /dev/null +++ b/modules/nw-dpf-component-placement.adoc @@ -0,0 +1,36 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/about-dpf-operator.adoc + +:_mod-docs-content-type: CONCEPT +[id="nw-dpf-component-placement_{context}"] += DPF component placement + +[role="_abstract"] +The software components and operators run on the management cluster and the DPU hosted cluster to separate workload management from infrastructure acceleration. + +== Management cluster components + +The following operators and services run within the management cluster: + +NVIDIA DPF Operator:: The core operator that manages DPU services and configurations within the hosted cluster, including DPU provisioning, networking acceleration, and DOCA service orchestration. + +dpf-hcp-provisioner-operator:: Automates the hosted control planes cluster lifecycle for DPU nodes, replacing the manual cluster creation workflow from previous versions. + +MultiCluster Engine (MCE) and hosted control planes:: Provide the control plane and management framework for the DPU hosted cluster. + +Node Feature Discovery (NFD) Operator:: Discovers and labels hardware features on the nodes, including the presence of DPUs. + +MetalLB Operator:: Provides load balancing services for the management cluster. + +GitOps Operator:: Facilitates ArgoCD-based deployment of applications and configurations. + +cert-manager Operator:: Automates the management, issuance, and renewal of TLS certificates within the cluster. + +NVIDIA Maintenance Operator:: Assists in performing maintenance tasks and gracefully draining DPU worker nodes. + +{lvms}:: Provides persistent ReadWriteMany (RWX) storage required for various components, such as the etcd database of the hosted cluster. + +NodeSRIOVDevicePluginConfig:: A DPF-managed CRD that configures SR-IOV device plugin pods on worker nodes. It defines VF allocation ranges for management and workload traffic, replacing the standalone SR-IOV device plugin `DaemonSet` from previous versions. + +Bare Metal Operator:: Provisions and adds worker nodes with DPUs to the management cluster. diff --git a/modules/nw-dpf-configuring-cno.adoc b/modules/nw-dpf-configuring-cno.adoc new file mode 100644 index 000000000000..6330d1b8065a --- /dev/null +++ b/modules/nw-dpf-configuring-cno.adoc @@ -0,0 +1,36 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-environment-setup.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-configuring-cno_{context}"] += Configure the Cluster Network Operator for IP forwarding + +[role="_abstract"] +You must enable global IP forwarding for OVN-Kubernetes to allow IP packet forwarding between the different networks managed by OVN-Kubernetes. + +.Prerequisites + +* You have access to the cluster as a user with the `cluster-admin` role. +* You have installed the OpenShift CLI (`oc`). + +.Procedure + +* Enable global IP forwarding on the OVN-Kubernetes configuration: ++ +[source,terminal] +---- +$ oc patch network.operator.openshift.io cluster --type=merge -p \ + '{"spec":{"defaultNetwork":{"ovnKubernetesConfig":{"gatewayConfig":{"ipForwarding":"Global"}}}}}' +---- + +.Verification + +* Verify that the patch was applied: ++ +[source,terminal] +---- +$ oc get network.operator.openshift.io cluster -o jsonpath='{.spec.defaultNetwork.ovnKubernetesConfig.gatewayConfig.ipForwarding}' +---- ++ +The command returns `Global` if the patch was applied successfully. diff --git a/modules/nw-dpf-configuring-dpu-telemetry.adoc b/modules/nw-dpf-configuring-dpu-telemetry.adoc new file mode 100644 index 000000000000..47a213dee284 --- /dev/null +++ b/modules/nw-dpf-configuring-dpu-telemetry.adoc @@ -0,0 +1,10 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-validation-telemetry.adoc + +:_mod-docs-content-type: CONCEPT +[id="dpu-telemetry_{context}"] += DPU telemetry configuration + +[role="_abstract"] +The DOCA Telemetry Service (DTS) exposes metrics from the BlueField DPUs. You can enable observability by turning on user workload monitoring and creating a `ServiceMonitor` resource. diff --git a/modules/nw-dpf-configuring-dts-servicemonitor.adoc b/modules/nw-dpf-configuring-dts-servicemonitor.adoc new file mode 100644 index 000000000000..ce5879b3a311 --- /dev/null +++ b/modules/nw-dpf-configuring-dts-servicemonitor.adoc @@ -0,0 +1,63 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-validation-telemetry.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-configuring-dts-servicemonitor_{context}"] += Configure the DTS ServiceMonitor + +[role="_abstract"] +You can create a `ServiceMonitor` resource to instruct the user workload monitoring Prometheus instance to scrape the DTS metrics endpoint. +The `ServiceMonitor` selects the mirrored DTS service in the `dpf-operator-system` namespace and scrapes its `/metrics` path on the `httpserverport` every 30 seconds. + +.Prerequisites + +* User workload monitoring is enabled in {product-title}. +* The DTS `DPUServiceTemplate`, `DPUServiceConfiguration`, and `DPUDeployment` resources are applied. ++ +For details, see "DPU telemetry observability with DTS". +* You have access to the management cluster as a user with the `cluster-admin` role. + +.Procedure + +. Create a file named `dts-servicemonitor.yaml` with the following content: ++ +[source,yaml] +---- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: dts-servicemonitor + namespace: dpf-operator-system +spec: + endpoints: + - port: httpserverport + interval: 30s + path: /metrics + selector: + matchLabels: + dpu.nvidia.com/exposed-port-for-dpucluster: "" +---- + +. Apply the `ServiceMonitor`: ++ +[source,terminal] +---- +$ oc apply -f dts-servicemonitor.yaml +---- + +.Verification + +* Verify that the `ServiceMonitor` is created in the `dpf-operator-system` namespace: ++ +[source,terminal] +---- +$ oc -n dpf-operator-system get servicemonitor dts-servicemonitor +---- ++ +.Example output +[source,terminal] +---- +NAME AGE +dts-servicemonitor ... +---- diff --git a/modules/nw-dpf-configuring-gitops.adoc b/modules/nw-dpf-configuring-gitops.adoc new file mode 100644 index 000000000000..a481af47d0b4 --- /dev/null +++ b/modules/nw-dpf-configuring-gitops.adoc @@ -0,0 +1,97 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-environment-setup.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-configuring-gitops_{context}"] += Configure the GitOps Operator + +[role="_abstract"] +You must create an `ArgoCD` instance in the `dpf-operator-system` namespace to manage the DPF components lifecycle. + +.Prerequisites + +* You have access to the cluster as a user with the `cluster-admin` role. +* You have installed the OpenShift CLI (`oc`). +* The GitOps Operator is installed. + +.Procedure + +. Create a file named `argocd-instance.yaml` with the following `ArgoCD` resource definition: ++ +[source,yaml] +---- +apiVersion: argoproj.io/v1beta1 +kind: ArgoCD +metadata: + name: argocd + namespace: dpf-operator-system +spec: + nodePlacement: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + server: + route: + enabled: true + labels: + ovn.dpu.nvidia.com/skip-injection: "" + controller: + labels: + ovn.dpu.nvidia.com/skip-injection: "" + repo: + labels: + ovn.dpu.nvidia.com/skip-injection: "" + applicationSet: + enabled: false + sso: + dex: + openShiftOAuth: true + notifications: + enabled: false +---- + +. Apply the file: ++ +[source,terminal] +---- +$ oc apply -f argocd-instance.yaml +---- + +. Wait for the ArgoCD Redis deployment to become available: ++ +[source,terminal] +---- +$ oc wait deployment argocd-redis -n dpf-operator-system \ + --for=condition=Available --timeout=120s +---- + +. Add the `skip-injection` label to the Redis deployment to prevent the OVN resource injector from modifying Redis pods: ++ +[source,terminal] +---- +$ oc patch deployment argocd-redis -n dpf-operator-system \ + --type=merge -p '{"spec":{"template":{"metadata":{"labels":{"ovn.dpu.nvidia.com/skip-injection":""}}}}}' +---- ++ +[IMPORTANT] +==== +The {product-title} GitOps Operator `ArgoCD` CR does not support custom labels on the Redis component. +You must apply this label manually. +The ArgoCD operator does not override this patch. +==== + +.Verification + +* Verify that the ArgoCD pods are running: ++ +[source,terminal] +---- +$ oc get pods -n dpf-operator-system -l app.kubernetes.io/part-of=argocd +---- diff --git a/modules/nw-dpf-configuring-hosted-cluster-auth.adoc b/modules/nw-dpf-configuring-hosted-cluster-auth.adoc new file mode 100644 index 000000000000..a2062cb5be7f --- /dev/null +++ b/modules/nw-dpf-configuring-hosted-cluster-auth.adoc @@ -0,0 +1,61 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-worker-nodes-provisioning.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-configuring-hosted-cluster-auth_{context}"] += Configure authorization for the hosted cluster + +[role="_abstract"] +DPF services running on DPU nodes require privileged access to host networking and devices. +You must create a `ClusterRoleBinding` on the hosted cluster that grants the `privileged` security context constraint (SCC) to all service accounts in the `dpf-operator-system` namespace. + +.Prerequisites + +* You have access to the hosted cluster as a user with the `cluster-admin` role. +* The OpenShift CLI (`oc`) is installed. +* The hosted cluster kubeconfig file is available. +* DPU provisioning has reached the `DPU Cluster Config` stage. + +.Procedure + +. Switch to the hosted cluster context: ++ +[source,terminal] +---- +$ export KUBECONFIG=$HOSTED_CLUSTER_NAME.kubeconfig +---- + +. Create a file named `dpu-cluster-scc.yaml` with the following content: ++ +[source,yaml] +---- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: dpf-system-scc-privileged + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/part-of: dpu-services +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:openshift:scc:privileged +subjects: +- kind: Group + apiGroup: rbac.authorization.k8s.io + name: system:serviceaccounts:dpf-operator-system +---- + +. Apply the resource file on the hosted cluster: ++ +[source,terminal] +---- +$ oc apply -f dpu-cluster-scc.yaml +---- ++ +.Example output +[source,terminal] +---- +clusterrolebinding.rbac.authorization.k8s.io/dpf-system-scc-privileged created +---- diff --git a/modules/nw-dpf-configuring-mce.adoc b/modules/nw-dpf-configuring-mce.adoc new file mode 100644 index 000000000000..62d48b76ee5c --- /dev/null +++ b/modules/nw-dpf-configuring-mce.adoc @@ -0,0 +1,57 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-environment-setup.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-configuring-mce_{context}"] += Configure the multicluster engine + +[role="_abstract"] +You must create a `MultiClusterEngine` instance and enable the hosted control planes component to provide the control plane and management framework for the DPU-hosted cluster. + +.Prerequisites + +* You have access to the cluster as a user with the `cluster-admin` role. +* You have installed the OpenShift CLI (`oc`). +* The multicluster engine for Kubernetes Operator is installed. You can install it by using the Assisted Installer during cluster creation. +* The hosted control planes feature is supported on your cluster. + +.Procedure + +. Create a file named `multiclusterengine-instance.yaml` with the following content: ++ +[source,yaml] +---- +apiVersion: multicluster.openshift.io/v1 +kind: MultiClusterEngine +metadata: + name: multiclusterengine +spec: + overrides: + components: + - name: hypershift + enabled: true +---- + +. Apply the file: ++ +[source,terminal] +---- +$ oc apply -f multiclusterengine-instance.yaml +---- + +.Verification + +* Verify that the `MultiClusterEngine` instance is available and hosted control planes are enabled: ++ +[source,terminal] +---- +$ oc get multiclusterengine multiclusterengine +---- ++ +.Example output +[source,terminal] +---- +NAME STATUS AGE CURRENTVERSION DESIREDVERSION MESSAGE +multiclusterengine Available 4m58s 2.17.0 2.17.0 All components available +---- diff --git a/modules/nw-dpf-configuring-metallb.adoc b/modules/nw-dpf-configuring-metallb.adoc new file mode 100644 index 000000000000..d210f8f89f36 --- /dev/null +++ b/modules/nw-dpf-configuring-metallb.adoc @@ -0,0 +1,70 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-environment-setup.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-configuring-metallb_{context}"] += Configure the MetalLB Operator + +[role="_abstract"] +You must create a `MetalLB` instance to deploy the MetalLB load balancer in the cluster and provide external IP addresses for services. + +.Prerequisites + +* You have access to the cluster as a user with the `cluster-admin` role. +* You have installed the OpenShift CLI (`oc`). +* The MetalLB Operator is installed. + +.Procedure + +. Define the hosted cluster variables used by MetalLB: ++ +[source,terminal] +---- +$ export HOSTED_CLUSTER_VIP= +$ export HOSTED_CLUSTER_NAME="dpf-hosted" +---- ++ +where: ++ +``:: Specifies the virtual IP for the hosted DPU cluster, allocated from the management cluster subnet, for example `203.0.113.200/32`. +`HOSTED_CLUSTER_NAME`:: Specifies the DPU hosted cluster name. ++ +[NOTE] +==== +These variables are also used in later installation steps. +==== + +. Create a file named `metallb-config.yaml` with the following `MetalLB` resource definition: ++ +[source,yaml] +---- +apiVersion: metallb.io/v1beta1 +kind: MetalLB +metadata: + name: metallb + namespace: openshift-operators +spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + speakerTolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule +---- + +. Apply the resource file: ++ +[source,terminal] +---- +$ envsubst < metallb-config.yaml | oc apply -f - +---- + +.Verification + +* Verify that the `MetalLB` instance was created: ++ +[source,terminal] +---- +$ oc get metallb -n openshift-operators +---- diff --git a/modules/nw-dpf-configuring-nfd.adoc b/modules/nw-dpf-configuring-nfd.adoc new file mode 100644 index 000000000000..579e7de0f1e5 --- /dev/null +++ b/modules/nw-dpf-configuring-nfd.adoc @@ -0,0 +1,128 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-environment-setup.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-configuring-nfd_{context}"] += Configure the Node Feature Discovery Operator + +[role="_abstract"] +You must configure the Node Feature Discovery (NFD) Operator to discover hardware features on the nodes and to label worker nodes that have DPUs installed. + +.Prerequisites + +* You have access to the cluster as a user with the `cluster-admin` role. +* You have installed the OpenShift CLI (`oc`). +* The Node Feature Discovery Operator is installed. You can install it by using the Assisted Installer during cluster creation. + +.Procedure + +. Define the cluster variables: ++ +[source,terminal] +---- +$ export CLUSTER_NAME="doca-mgmt" +$ export BASE_DOMAIN="example.com" +$ export HOST_CLUSTER_API="api.${CLUSTER_NAME}.${BASE_DOMAIN}" +---- ++ +where: ++ +`CLUSTER_NAME`:: Specifies the management cluster name. +`BASE_DOMAIN`:: Specifies the management cluster base domain. +`HOST_CLUSTER_API`:: Specifies the management cluster API endpoint. ++ +[NOTE] +==== +These variables are also used in later installation steps. +==== + +. Create a file named `nfd-instance.yaml` with the following `NodeFeatureDiscovery` resource definition. +The `HOST_CLUSTER_API` variable is used in the resource file. ++ +[source,yaml] +---- +apiVersion: nfd.openshift.io/v1 +kind: NodeFeatureDiscovery +metadata: + name: nfd-instance + namespace: openshift-nfd +spec: + operand: + workerEnvs: + - name: KUBERNETES_SERVICE_HOST + value: $HOST_CLUSTER_API + - name: KUBERNETES_SERVICE_PORT + value: "6443" + workerConfig: + configData: | + sources: + pci: + deviceClassWhitelist: + - "0200" + - "03" + - "12" + - "0207" + deviceLabelFields: + - "vendor" + - "device" + - "class" +---- + +. Apply the file by using `envsubst` to substitute the environment variables: ++ +[source,terminal] +---- +$ envsubst < nfd-instance.yaml | oc apply -f - +---- + +. Create a file named `nfd-rule.yaml` with the following `NodeFeatureRule` resource definition to detect worker nodes with DPUs and label them with a `dpu-enabled` label: ++ +[source,yaml] +---- +apiVersion: nfd.openshift.io/v1alpha1 +kind: NodeFeatureRule +metadata: + name: dpu-detection-rule + namespace: openshift-nfd +spec: + rules: + - labels: + dpu-enabled: "" + matchFeatures: + - feature: pci.device + matchExpressions: + device: + op: In + value: + - a2d6 + - a2dc + vendor: + op: In + value: + - 15b3 + name: DPU-detection-rule +---- + +. Apply the file: ++ +[source,terminal] +---- +$ oc apply -f nfd-rule.yaml +---- + +.Verification + +* Verify that the `NodeFeatureDiscovery` instance is configured: ++ +[source,terminal] +---- +$ oc get nodefeaturediscovery -n openshift-nfd +---- + +* Verify that the `NodeFeatureRule` was created: ++ +[source,terminal] +---- +$ oc get nodefeaturerule -n openshift-nfd +---- diff --git a/modules/nw-dpf-configuring-required-operators.adoc b/modules/nw-dpf-configuring-required-operators.adoc new file mode 100644 index 000000000000..8194412d2c80 --- /dev/null +++ b/modules/nw-dpf-configuring-required-operators.adoc @@ -0,0 +1,10 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-environment-setup.adoc + +:_mod-docs-content-type: CONCEPT +[id="configuring-operators_{context}"] += Required operator configuration + +[role="_abstract"] +After the required operators are installed, you configure the MultiCluster Engine, Node Feature Discovery, MetalLB, GitOps, and Cluster Network Operator for the DPF environment. diff --git a/modules/nw-dpf-configuring-worker-nodes.adoc b/modules/nw-dpf-configuring-worker-nodes.adoc new file mode 100644 index 000000000000..986e5973d977 --- /dev/null +++ b/modules/nw-dpf-configuring-worker-nodes.adoc @@ -0,0 +1,10 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-environment-setup.adoc + +:_mod-docs-content-type: CONCEPT +[id="configuring-worker-nodes_{context}"] += Worker node configuration + +[role="_abstract"] +Before you install the DPF Operator, worker nodes in the management cluster require a `MachineConfig` resource and the DPF namespace. diff --git a/modules/nw-dpf-creating-bfb.adoc b/modules/nw-dpf-creating-bfb.adoc new file mode 100644 index 000000000000..bf2cb9f3a142 --- /dev/null +++ b/modules/nw-dpf-creating-bfb.adoc @@ -0,0 +1,58 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-operator-installation.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-creating-bfb_{context}"] += Create the BFB resource + +[role="_abstract"] +You can create a `BFB` custom resource to define the DPU image, known as a BlueField Bootstream File, that is downloaded and placed on shared storage for DPU provisioning. + +.Prerequisites + +* The DPF Operator is installed. +* The `DPFOperatorConfig` resource is created. +* The DPF Operator environment variables are set. For details, see "DPF Operator installation environment variables". + +.Procedure + +. Create a file named `bfb.yaml` with the following content: ++ +[source,yaml] +---- +apiVersion: provisioning.dpu.nvidia.com/v1alpha1 +kind: BFB +metadata: + name: bf-bundle + namespace: dpf-operator-system +spec: + url: $BFB_URL + versions: + atf: 4.13.1-0-g5fcb148df + bsp: 4.13.1.13827 + doca: 3.4.0 + uefi: 4.13.1-14-g8a01157b7f +---- + +. Apply the resource file: ++ +[source,terminal] +---- +$ envsubst < bfb.yaml | oc apply -f - +---- + +.Verification + +* Verify that the BFB image phase is `Ready`: ++ +[source,terminal] +---- +$ oc get bfbs.provisioning.dpu.nvidia.com -n dpf-operator-system bf-bundle -o yaml | grep phase +---- ++ +.Example output +[source,terminal] +---- +phase: Ready +---- diff --git a/modules/nw-dpf-creating-dpf-namespace.adoc b/modules/nw-dpf-creating-dpf-namespace.adoc new file mode 100644 index 000000000000..f9a582299f72 --- /dev/null +++ b/modules/nw-dpf-creating-dpf-namespace.adoc @@ -0,0 +1,33 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-environment-setup.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-creating-dpf-namespace_{context}"] += Create the DPF namespace + +[role="_abstract"] +You must create a dedicated namespace for the DPF Operator and its components before installing the operator. + +.Prerequisites + +* You have access to the cluster as a user with the `cluster-admin` role. +* You have installed the OpenShift CLI (`oc`). + +.Procedure + +* Create the `dpf-operator-system` namespace: ++ +[source,terminal] +---- +$ oc create namespace dpf-operator-system +---- + +.Verification + +* Verify that the namespace was created: ++ +[source,terminal] +---- +$ oc get namespace dpf-operator-system +---- diff --git a/modules/nw-dpf-creating-dpf-resources-overview.adoc b/modules/nw-dpf-creating-dpf-resources-overview.adoc new file mode 100644 index 000000000000..87645af63ad7 --- /dev/null +++ b/modules/nw-dpf-creating-dpf-resources-overview.adoc @@ -0,0 +1,10 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-operator-installation.adoc + +:_mod-docs-content-type: CONCEPT +[id="creating-dpf-resources_{context}"] += DPF resources + +[role="_abstract"] +After you install the DPF Operator, the deployment requires these DPF custom resources: `DPFOperatorConfig`, `NodeSRIOVDevicePluginConfig`, `DPUFlavor`, `BFB`, and `DPUDeployment`. diff --git a/modules/nw-dpf-creating-dpfhcpprovisioner.adoc b/modules/nw-dpf-creating-dpfhcpprovisioner.adoc new file mode 100644 index 000000000000..d26cc49d4fd5 --- /dev/null +++ b/modules/nw-dpf-creating-dpfhcpprovisioner.adoc @@ -0,0 +1,72 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-hosted-cluster-provisioning.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-creating-dpfhcpprovisioner_{context}"] += Create the DPFHCPProvisioner resource + +[role="_abstract"] +The `DPFHCPProvisioner` custom resource triggers the creation of the complete hosted cluster infrastructure. +This includes the `HostedCluster`, the MetalLB `IPAddressPool`, the `L2Advertisement`, and kubeconfig injection into the `DPUCluster`. + +.Prerequisites + +* The environment variables described in "Hosted cluster provisioning environment variables" are set. +* The `dpf-hcp-provisioner-operator` is installed and running. +* The pull secret and SSH key secret are created in the clusters namespace. +* The `DPUCluster` resource is created in the `dpf-operator-system` namespace. + +.Procedure + +. Create a file named `dpfhcpprovisioner.yaml` with the following content: ++ +[source,yaml] +---- +apiVersion: provisioning.dpu.hcp.io/v1alpha1 +kind: DPFHCPProvisioner +metadata: + name: $HOSTED_CLUSTER_NAME + namespace: $CLUSTERS_NAMESPACE +spec: + baseDomain: $BASE_DOMAIN + dpuClusterRef: + name: $HOSTED_CLUSTER_NAME + namespace: dpf-operator-system + dpuDeploymentRef: + name: dpudeployment + namespace: dpf-operator-system + machineOSURL: $BLUEFIELD_OCP_IMAGE + etcdStorageClass: $ETCD_STORAGE_CLASS + ocpReleaseImage: $OCP_RELEASE_IMAGE + pullSecretRef: + name: $PULL_SECRET_NAME + sshKeySecretRef: + name: $SSH_KEY_SECRET_NAME + virtualIP: $HOSTED_CLUSTER_VIP +---- ++ +where: + +`baseDomain`:: Specifies the base DNS domain for the hosted cluster. +`dpuClusterRef`:: Specifies a reference to the `DPUCluster` resource that represents the DPU hosted cluster. +`dpuDeploymentRef`:: Specifies a reference to the `DPUDeployment` resource in the `dpf-operator-system` namespace. +`machineOSURL`:: Specifies the BlueField {product-title} image URL. When set, this skips automatic image lookup. +`etcdStorageClass`:: Specifies the storage class for etcd persistent volume claims. +`ocpReleaseImage`:: Specifies the {product-title} release image for the hosted cluster. +`pullSecretRef`:: Specifies the name of the pull secret in the same namespace. +`sshKeySecretRef`:: Specifies the name of the SSH key secret in the same namespace. +`virtualIP`:: Specifies the virtual IP address for the hosted cluster API server. + +. Apply the resource with variable substitution: ++ +[source,terminal] +---- +$ envsubst < dpfhcpprovisioner.yaml | oc apply -f - +---- ++ +.Example output +[source,terminal] +---- +dpfhcpprovisioner.provisioning.dpu.hcp.io/dpf-hosted created +---- diff --git a/modules/nw-dpf-creating-dpfoperatorconfig.adoc b/modules/nw-dpf-creating-dpfoperatorconfig.adoc new file mode 100644 index 000000000000..5aafd0892aa7 --- /dev/null +++ b/modules/nw-dpf-creating-dpfoperatorconfig.adoc @@ -0,0 +1,124 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-operator-installation.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-creating-dpfoperatorconfig_{context}"] += Create the DPFOperatorConfig resource + +[role="_abstract"] +You can create a `DPFOperatorConfig` custom resource to configure the DPF Operator components, including the provisioning controller, static cluster manager, and SR-IOV device plugin controller. + +.Prerequisites + +* The DPF Operator is installed. +* The DPF Operator environment variables are set. For details, see "DPF Operator installation environment variables". + +.Procedure + +. Create a file named `dpfoperatorconfig.yaml` with the following content: ++ +[source,yaml] +---- +apiVersion: operator.dpu.nvidia.com/v1alpha1 +kind: DPFOperatorConfig +metadata: + name: dpfoperatorconfig + namespace: dpf-operator-system +spec: + kamajiClusterManager: + disable: true + multus: + disable: true + cniInstaller: + disable: true + networking: + controlPlaneMTU: $NODES_MTU + highSpeedMTU: $NODES_MTU + overrides: + dpuCNIBinPath: /var/lib/cni/bin/ + dpuCNIPath: /run/multus/cni/net.d/ + dpuOpenvSwitchSystemSharedLib64Path: /lib64 + flannelSkipCNIConfigInstallation: false + kubernetesAPIServerPort: $TARGETCLUSTER_API_SERVER_PORT + kubernetesAPIServerVIP: $HOST_CLUSTER_API + dpuLinkerCachePath: /etc/ld.so.cache + dpuOptLibraryPath: /usr/opt + provisioningController: + enableDynamicBFCFGTemplates: true + hostAgentDNSPolicy: Default + dmsTimeout: 900 + nodeSRIOVDevicePluginController: + devicePlugin: + defaultResourcePrefix: openshift.io + disable: false + replicas: 1 + staticClusterManager: + disable: false +---- + +. Apply the resource file: ++ +[source,terminal] +---- +$ envsubst < dpfoperatorconfig.yaml | oc apply -f - +---- + +.Verification + +. Verify that the provisioning controller manager deployment has rolled out: ++ +[source,terminal] +---- +$ oc rollout status deployment --namespace dpf-operator-system dpf-provisioning-controller-manager +---- ++ +.Example output +[source,terminal] +---- +deployment "dpf-provisioning-controller-manager" successfully rolled out +---- + +. Verify that the DPU service controller manager deployment has rolled out: ++ +[source,terminal] +---- +$ oc rollout status deployment --namespace dpf-operator-system dpuservice-controller-manager +---- ++ +.Example output +[source,terminal] +---- +deployment "dpuservice-controller-manager" successfully rolled out +---- + +. Verify that all pods in the `dpf-operator-system` namespace are running: ++ +[source,terminal] +---- +$ oc get pods -n dpf-operator-system +---- ++ +.Example output +[source,terminal] +---- +NAME READY STATUS RESTARTS AGE +argocd-application-controller-0 1/1 Running 0 13h +argocd-redis-7565dcc7fc-nz92w 1/1 Running 0 13h +argocd-repo-server-5cd89d9dd8-l9s58 1/1 Running 0 13h +argocd-server-6cbf77db84-gzrl9 1/1 Running 0 13h +dpf-dpu-detector-kwqtm 1/1 Running 0 2m46s +dpf-dpu-detector-mqbn4 1/1 Running 0 2m46s +dpf-dpu-detector-vt968 1/1 Running 0 2m46s +dpf-operator-controller-manager-7bf9744c5f-zdvhw 1/1 Running 0 6m46s +dpf-provisioning-controller-manager-57d4767fdc-ftm7q 1/1 Running 0 2m47s +dpf-nodesriovdeviceplugin-controller-584696485b-ffnsk 1/1 Running 0 41h +dpuservice-controller-manager-7459985c77-bxsqp 1/1 Running 0 2m46s +maintenance-operator-585767f779-4qvvn 1/1 Running 0 21h +static-cm-controller-manager-6d977d5f84-756pw 1/1 Running 0 2m47s +---- ++ +[NOTE] +==== +The `servicechainset-controller-manager` pod might show a status of `ContainerCreating` at this phase. This is expected behavior. +==== diff --git a/modules/nw-dpf-creating-dpu-services-overview.adoc b/modules/nw-dpf-creating-dpu-services-overview.adoc new file mode 100644 index 000000000000..f7b55b948a53 --- /dev/null +++ b/modules/nw-dpf-creating-dpu-services-overview.adoc @@ -0,0 +1,10 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-operator-installation.adoc + +:_mod-docs-content-type: CONCEPT +[id="creating-dpu-services_{context}"] += DPU services + +[role="_abstract"] +DPU services run on BlueField DPUs and include the Host-Based Networking (HBN) service, OVN-Kubernetes DPU service, and DOCA Telemetry Service (DTS). diff --git a/modules/nw-dpf-creating-dpucluster.adoc b/modules/nw-dpf-creating-dpucluster.adoc new file mode 100644 index 000000000000..053925ef3146 --- /dev/null +++ b/modules/nw-dpf-creating-dpucluster.adoc @@ -0,0 +1,55 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-hosted-cluster-provisioning.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-creating-dpucluster_{context}"] += Create the DPUCluster resource + +[role="_abstract"] +The `DPUCluster` resource tells the DPF Operator about the hosted cluster where DPU services will run. +The `dpf-hcp-provisioner-operator` automatically injects the kubeconfig into this resource after the hosted cluster is created. + +.Prerequisites + +* The environment variables described in "Hosted cluster provisioning environment variables" are set. +* The DPF Operator is installed. + +.Procedure + +. Create a file named `dpucluster.yaml` with the following content: ++ +[source,yaml] +---- +apiVersion: provisioning.dpu.nvidia.com/v1alpha1 +kind: DPUCluster +metadata: + name: $HOSTED_CLUSTER_NAME + namespace: dpf-operator-system +spec: + type: static + maxNodes: 10 + kubeconfig: ${HOSTED_CLUSTER_NAME}-admin-kubeconfig +---- + +. Apply the resource with variable substitution: ++ +[source,terminal] +---- +$ envsubst < dpucluster.yaml | oc apply -f - +---- ++ +.Example output +[source,terminal] +---- +dpucluster.provisioning.dpu.nvidia.com/dpf-hosted created +---- + +.Verification + +* Verify that the `DPUCluster` resource was created: ++ +[source,terminal] +---- +$ oc get dpucluster -n dpf-operator-system +---- diff --git a/modules/nw-dpf-creating-dpudeployment.adoc b/modules/nw-dpf-creating-dpudeployment.adoc new file mode 100644 index 000000000000..61699a0fa3e8 --- /dev/null +++ b/modules/nw-dpf-creating-dpudeployment.adoc @@ -0,0 +1,108 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-operator-installation.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-creating-dpudeployment_{context}"] += Create the DPUDeployment resource + +[role="_abstract"] +You can create a `DPUDeployment` custom resource as the main orchestration object that connects DPU services with specific BFB images and DPU flavors. +The `DPUDeployment` defines DPU sets for DPU provisioning and configures service chains to deploy services across DPUs. + +.Prerequisites + +* The DPF Operator is installed. +* The `DPFOperatorConfig` resource is created. +* The `NodeSRIOVDevicePluginConfig` resource is created. +* The `DPUFlavor` resource is created. +* The `BFB` resource is created and in `Ready` phase. + +.Procedure + +. Create a file named `dpudeployment.yaml` with the following content: ++ +[source,yaml] +---- +apiVersion: svc.dpu.nvidia.com/v1alpha1 +kind: DPUDeployment +metadata: + name: dpudeployment + namespace: dpf-operator-system +spec: + dpus: + nodeEffect: + drain: true + dpuSetStrategy: + type: RollingUpdate + bfb: bf-bundle + flavor: hbn-ovnk + dpuSets: + - nameSuffix: "dpuset1" + dpuAnnotations: + noderesources.dpu.nvidia.com/nodesriovdevicepluginconfig: bf3-p0-vfs + nodeSelector: + matchLabels: + feature.node.kubernetes.io/dpu-enabled: "" + services: + hbn: + serviceTemplate: hbn + serviceConfiguration: hbn + ovn: + serviceTemplate: ovn + serviceConfiguration: ovn + doca-telemetry-service: + serviceTemplate: doca-telemetry-service + serviceConfiguration: doca-telemetry-service + serviceChains: + switches: + - ports: + - serviceInterface: + matchLabels: + uplink: p0 + - service: + name: hbn + interface: p0_if + - ports: + - serviceInterface: + matchLabels: + uplink: p1 + - service: + name: hbn + interface: p1_if + - ports: + - serviceInterface: + matchLabels: + port: ovn + - service: + name: hbn + interface: pf2dpu2_if +---- + +. Apply the resource file: ++ +[source,terminal] +---- +$ oc apply -f dpudeployment.yaml +---- + +.Verification + +* Verify the `DPUDeployment` state: ++ +[source,terminal] +---- +$ oc get DPUDeployment -n dpf-operator-system +---- ++ +.Example output +[source,terminal] +---- +NAME READY PHASE AGE +dpudeployment False Pending 2m32s +---- ++ +[NOTE] +==== +A `Pending` phase is expected at this stage. The `DPUDeployment` transitions to `Ready` after DPU provisioning is complete and all services are deployed. +==== diff --git a/modules/nw-dpf-creating-dpuflavor.adoc b/modules/nw-dpf-creating-dpuflavor.adoc new file mode 100644 index 000000000000..4dd8ab9f699c --- /dev/null +++ b/modules/nw-dpf-creating-dpuflavor.adoc @@ -0,0 +1,220 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-operator-installation.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-creating-dpuflavor_{context}"] += Create the DPUFlavor resource + +[role="_abstract"] +You can create a `DPUFlavor` custom resource to define the DPU configuration, including NVConfig parameters, kernel arguments, hugepages settings, and the OVS initialization script. +The `DPUFlavor` also supports an optional `configFiles` field for custom DPU configuration files. + +.Prerequisites + +* The DPF Operator is installed. +* The `DPFOperatorConfig` resource is created. +* The DPF Operator environment variables are set. For details, see "DPF Operator installation environment variables". + +.Procedure + +. Create a file named `dpuflavor.yaml` with the content for your MTU configuration. ++ +Select the `DPUFlavor` that matches your MTU configuration: ++ +-- +*Standard MTU (1500):* + +[source,yaml,subs="verbatim"] +---- +apiVersion: provisioning.dpu.nvidia.com/v1alpha1 +kind: DPUFlavor +metadata: + name: hbn-ovnk + namespace: dpf-operator-system + annotations: + provisioning.dpu.nvidia.com/skip-bfcfg-size-check: "" +spec: + grub: + kernelParameters: + - console=hvc0 + - console=ttyAMA0 + - earlycon=pl011,0x13010000 + - iommu.passthrough=1 + - cgroup_no_v1=net_prio,net_cls + - hugepagesz=2048kB + - hugepages=250 + nvconfig: + - device: '*' + parameters: + - PF_BAR2_ENABLE=0 + - PER_PF_NUM_SF=1 + - PF_TOTAL_SF=20 + - PF_SF_BAR_SIZE=10 + - NUM_PF_MSIX_VALID=0 + - PF_NUM_PF_MSIX_VALID=1 + - PF_NUM_PF_MSIX=228 + - INTERNAL_CPU_MODEL=1 + - INTERNAL_CPU_OFFLOAD_ENGINE=0 + - SRIOV_EN=1 + - NUM_OF_VFS=$NUM_VFS + - LAG_RESOURCE_ALLOCATION=1 + - LINK_TYPE_P1=ETH + - LINK_TYPE_P2=ETH + ovs: + rawConfigScript: | + _ovs-vsctl() { + ovs-vsctl --timeout 15 "$@" + } + _ovs-vsctl set Open_vSwitch . other_config:doca-init=true + _ovs-vsctl set Open_vSwitch . other_config:dpdk-max-memzones=50000 + _ovs-vsctl set Open_vSwitch . other_config:hw-offload=true + _ovs-vsctl set Open_vSwitch . other_config:pmd-quiet-idle=true + _ovs-vsctl set Open_vSwitch . other_config:max-idle=20000 + _ovs-vsctl set Open_vSwitch . other_config:max-revalidator=5000 + _ovs-vsctl set Open_vSwitch . other_config:doca-congestion-threshold=60 + _ovs-vsctl set Open_vSwitch . other_config:flow-limit=500000 + _ovs-vsctl set Open_vSwitch . other_config:hw-offload-ct-unidir-udp-enabled=true + _ovs-vsctl remove Open_vSwitch . other_config default-datapath-type || true + if systemctl list-unit-files openvswitch-switch.service &>/dev/null; then + systemctl restart openvswitch-switch + elif systemctl list-unit-files openvswitch.service &>/dev/null; then + systemctl restart openvswitch + fi + _ovs-vsctl --may-exist add-br br-sfc + _ovs-vsctl set bridge br-sfc datapath_type=netdev + _ovs-vsctl set bridge br-sfc fail_mode=secure + _ovs-vsctl --if-exists del-br br-hbn + _ovs-vsctl --may-exist add-br br-hbn + _ovs-vsctl set bridge br-hbn datapath_type=netdev + _ovs-vsctl set bridge br-hbn fail_mode=secure + _ovs-vsctl --may-exist add-port br-sfc p0 + _ovs-vsctl set Interface p0 type=dpdk + _ovs-vsctl set Interface p0 mtu_request=9216 + _ovs-vsctl set Port p0 external_ids:dpf-type=physical + # Activate DOCA for OVNK + _ovs-vsctl set Open_vSwitch . external-ids:ovn-bridge-datapath-type=netdev + # setup ovnkube managed bridge, br-dpu (this corresponds to br-ex on ovnk docs) + _ovs-vsctl --may-exist add-br br-dpu + _ovs-vsctl br-set-external-id br-dpu bridge-id br-dpu + _ovs-vsctl br-set-external-id br-dpu bridge-uplink pbrdputobrovn + _ovs-vsctl set bridge br-dpu datapath_type=netdev + _ovs-vsctl --may-exist add-port br-dpu pf0hpf + _ovs-vsctl set Interface pf0hpf type=dpdk + # Create OVS bridge (br-ovn) in between the SC managed bridge and OVNK + _ovs-vsctl --may-exist add-br br-ovn + _ovs-vsctl set bridge br-ovn datapath_type=netdev + _ovs-vsctl --may-exist add-port br-ovn pbrovntobrdpu + _ovs-vsctl --may-exist add-port br-dpu pbrdputobrovn + # Patch br-ovn and br-dpu together + _ovs-vsctl set Interface pbrovntobrdpu type=patch options:peer=pbrdputobrovn + _ovs-vsctl set Interface pbrdputobrovn type=patch options:peer=pbrovntobrdpu +---- + +*Jumbo frames (MTU 9000):* + +[source,yaml,subs="verbatim"] +---- +apiVersion: provisioning.dpu.nvidia.com/v1alpha1 +kind: DPUFlavor +metadata: + name: hbn-ovnk + namespace: dpf-operator-system + annotations: + provisioning.dpu.nvidia.com/skip-bfcfg-size-check: "" +spec: + grub: + kernelParameters: + - console=hvc0 + - console=ttyAMA0 + - earlycon=pl011,0x13010000 + - iommu.passthrough=1 + - cgroup_no_v1=net_prio,net_cls + - hugepagesz=2048kB + - hugepages=250 + nvconfig: + - device: '*' + parameters: + - PF_BAR2_ENABLE=0 + - PER_PF_NUM_SF=1 + - PF_TOTAL_SF=20 + - PF_SF_BAR_SIZE=10 + - NUM_PF_MSIX_VALID=0 + - PF_NUM_PF_MSIX_VALID=1 + - PF_NUM_PF_MSIX=228 + - INTERNAL_CPU_MODEL=1 + - INTERNAL_CPU_OFFLOAD_ENGINE=0 + - SRIOV_EN=1 + - NUM_OF_VFS=$NUM_VFS + - LAG_RESOURCE_ALLOCATION=1 + - NUM_VF_MSIX=30 + - LINK_TYPE_P1=ETH + - LINK_TYPE_P2=ETH + ovs: + rawConfigScript: | + _ovs-vsctl() { + ovs-vsctl --timeout 15 "$@" + } + _ovs-vsctl set Open_vSwitch . other_config:doca-init=true + _ovs-vsctl set Open_vSwitch . other_config:dpdk-max-memzones=50000 + _ovs-vsctl set Open_vSwitch . other_config:hw-offload=true + _ovs-vsctl set Open_vSwitch . other_config:pmd-quiet-idle=true + _ovs-vsctl set Open_vSwitch . other_config:max-idle=20000 + _ovs-vsctl set Open_vSwitch . other_config:max-revalidator=5000 + _ovs-vsctl set Open_vSwitch . other_config:doca-congestion-threshold=60 + _ovs-vsctl set Open_vSwitch . other_config:flow-limit=500000 + _ovs-vsctl set Open_vSwitch . other_config:hw-offload-ct-unidir-udp-enabled=true + _ovs-vsctl remove Open_vSwitch . other_config default-datapath-type || true + if systemctl list-unit-files openvswitch-switch.service &>/dev/null; then + systemctl restart openvswitch-switch + elif systemctl list-unit-files openvswitch.service &>/dev/null; then + systemctl restart openvswitch + fi + _ovs-vsctl --may-exist add-br br-sfc + _ovs-vsctl set bridge br-sfc datapath_type=netdev + _ovs-vsctl set bridge br-sfc fail_mode=secure + _ovs-vsctl --if-exists del-br br-hbn + _ovs-vsctl --may-exist add-br br-hbn + _ovs-vsctl set bridge br-hbn datapath_type=netdev + _ovs-vsctl set bridge br-hbn fail_mode=secure + _ovs-vsctl --may-exist add-port br-sfc p0 + _ovs-vsctl set Interface p0 type=dpdk + _ovs-vsctl set Interface p0 mtu_request=9216 + _ovs-vsctl set Port p0 external_ids:dpf-type=physical + # Activate DOCA for OVNK + _ovs-vsctl set Open_vSwitch . external-ids:ovn-bridge-datapath-type=netdev + # setup ovnkube managed bridge, br-dpu (this corresponds to br-ex on ovnk docs) + _ovs-vsctl --may-exist add-br br-dpu + _ovs-vsctl br-set-external-id br-dpu bridge-id br-dpu + _ovs-vsctl br-set-external-id br-dpu bridge-uplink pbrdputobrovn + _ovs-vsctl set bridge br-dpu datapath_type=netdev + _ovs-vsctl --may-exist add-port br-dpu pf0hpf + _ovs-vsctl set Interface pf0hpf type=dpdk + _ovs-vsctl set Interface br-dpu mtu_request=9000 + # Create OVS bridge (br-ovn) in between the SC managed bridge and OVNK + _ovs-vsctl --may-exist add-br br-ovn + _ovs-vsctl set bridge br-ovn datapath_type=netdev + _ovs-vsctl set Interface br-ovn mtu_request=9000 + _ovs-vsctl --may-exist add-port br-ovn pbrovntobrdpu + _ovs-vsctl --may-exist add-port br-dpu pbrdputobrovn + # Patch br-ovn and br-dpu together + _ovs-vsctl set Interface pbrovntobrdpu type=patch options:peer=pbrdputobrovn + _ovs-vsctl set Interface pbrdputobrovn type=patch options:peer=pbrovntobrdpu +---- +-- + +. Apply the resource file: ++ +[source,terminal] +---- +$ envsubst < dpuflavor.yaml | oc apply -f - +---- + +.Verification + +* Verify that the `DPUFlavor` resource is created: ++ +[source,terminal] +---- +$ oc get dpuflavor -n dpf-operator-system +---- diff --git a/modules/nw-dpf-creating-dts-service.adoc b/modules/nw-dpf-creating-dts-service.adoc new file mode 100644 index 000000000000..df096deaa0b0 --- /dev/null +++ b/modules/nw-dpf-creating-dts-service.adoc @@ -0,0 +1,80 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-operator-installation.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-creating-dts-service_{context}"] += Create the DOCA Telemetry Service DPU service resources + +[role="_abstract"] +You can create the DOCA Telemetry Service by defining a `DPUServiceTemplate` and a `DPUServiceConfiguration` custom resource. +The DOCA Telemetry Service provides metrics collection from the DPUs by using Prometheus. + +.Prerequisites + +* The DPF Operator is installed. +* The `DPFOperatorConfig` resource is created. +* The DPF Operator environment variables are set. For details, see "DPF Operator installation environment variables". + +.Procedure + +. Create a file named `dts.yaml` with the following content: ++ +[source,yaml] +---- +apiVersion: svc.dpu.nvidia.com/v1alpha1 +kind: DPUServiceTemplate +metadata: + name: doca-telemetry-service + namespace: dpf-operator-system +spec: + deploymentServiceName: "doca-telemetry-service" + resourceRequirements: + cpu: 1 + memory: 1Gi + storage: 1Gi + helmChart: + source: + repoURL: $DTS_HELM_REPO_URL + chart: doca-telemetry + version: $DTS_HELM_CHART_VERSION + values: + configMapData: + prometheus: + port: 9189 + hostVolumePrefix: "/var/lib" + imageDTS: $DTS_IMAGE + imagePullSecrets: + - name: dpf-pull-secret +--- +apiVersion: svc.dpu.nvidia.com/v1alpha1 +kind: DPUServiceConfiguration +metadata: + name: doca-telemetry-service + namespace: dpf-operator-system +spec: + deploymentServiceName: "doca-telemetry-service" + serviceConfiguration: + configPorts: + ports: + - name: httpserverport + port: 9189 + protocol: TCP + serviceType: None +---- + +. Apply the resource file: ++ +[source,terminal] +---- +$ envsubst < dts.yaml | oc apply -f - +---- + +.Verification + +* Verify that the DOCA Telemetry Service resources are created: ++ +[source,terminal] +---- +$ oc get dpuservicetemplate,dpuserviceconfiguration -n dpf-operator-system | grep doca-telemetry +---- diff --git a/modules/nw-dpf-creating-hbn-service.adoc b/modules/nw-dpf-creating-hbn-service.adoc new file mode 100644 index 000000000000..745a738b75e8 --- /dev/null +++ b/modules/nw-dpf-creating-hbn-service.adoc @@ -0,0 +1,152 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-operator-installation.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-creating-hbn-service_{context}"] += Create the HBN DPU service resources + +[role="_abstract"] +You can create the Host-Based Networking DPU service by defining a `DPUServiceTemplate` and a `DPUServiceConfiguration` custom resource. +The HBN service provides BGP-based networking on the DPU with ECMP routing support. + +.Prerequisites + +* The DPF Operator is installed. +* The `DPFOperatorConfig` resource is created. +* The DPF Operator environment variables are set. For details, see "DPF Operator installation environment variables". + +.Procedure + +. Create a file named `hbn.yaml` with the following content: ++ +[source,yaml] +---- +apiVersion: svc.dpu.nvidia.com/v1alpha1 +kind: DPUServiceTemplate +metadata: + name: hbn + namespace: dpf-operator-system +spec: + deploymentServiceName: "hbn" + helmChart: + source: + repoURL: $HBN_HELM_REPO_URL + version: $HBN_HELM_CHART_VERSION + chart: doca-hbn + values: + image: + repository: $HBN_IMAGE_REPO + tag: $HBN_IMAGE_TAG + imagePullSecrets: + - name: dpf-pull-secret + resources: + memory: 6Gi + nvidia.com/bf_sf: 3 +--- +apiVersion: svc.dpu.nvidia.com/v1alpha1 +kind: DPUServiceConfiguration +metadata: + name: hbn + namespace: dpf-operator-system +spec: + deploymentServiceName: "hbn" + serviceConfiguration: + serviceDaemonSet: + annotations: + k8s.v1.cni.cncf.io/networks: |- + [ + {"name": "iprequest", "interface": "ip_lo", "cni-args": {"poolNames": ["loopback"], "poolType": "cidrpool"}}, + {"name": "iprequest", "interface": "ip_pf2dpu2", "cni-args": {"poolNames": ["pool1"], "poolType": "cidrpool", "allocateDefaultGateway": true}} + ] + helmChart: + values: + configuration: + perDPUValuesYAML: | + - hostnamePattern: "*" + values: + bgp_peer_group: hbn + startupYAMLJ2: | + - header: + model: BLUEFIELD + nvue-api-version: nvue_v1 + rev-id: 1.0 + version: HBN 2.4.0 + - set: + interface: + lo: + ip: + address: + {{ ipaddresses.ip_lo.ip }}/32: {} + type: loopback + p0_if,p1_if: + type: swp + link: + mtu: 9216 + pf2dpu2_if: + ip: + address: + {{ ipaddresses.ip_pf2dpu2.cidr }}: {} + type: swp + link: + mtu: 9216 + router: + bgp: + autonomous-system: {{ ( ipaddresses.ip_lo.ip.split(".")[3] | int ) + 65101 }} + enable: on + graceful-restart: + mode: full + router-id: {{ ipaddresses.ip_lo.ip }} + vrf: + default: + router: + bgp: + address-family: + ipv4-unicast: + enable: on + redistribute: + connected: + enable: on + ipv6-unicast: + enable: on + redistribute: + connected: + enable: on + enable: on + neighbor: + p0_if: + peer-group: {{ config.bgp_peer_group }} + type: unnumbered + p1_if: + peer-group: {{ config.bgp_peer_group }} + type: unnumbered + path-selection: + multipath: + aspath-ignore: on + peer-group: + {{ config.bgp_peer_group }}: + remote-as: external + interfaces: + - name: p0_if + network: mybrhbn + - name: p1_if + network: mybrhbn + - name: pf2dpu2_if + network: mybrhbn +---- + +. Apply the resource file: ++ +[source,terminal] +---- +$ envsubst < hbn.yaml | oc apply -f - +---- + +.Verification + +* Verify that the HBN service resources are created: ++ +[source,terminal] +---- +$ oc get dpuservicetemplate,dpuserviceconfiguration -n dpf-operator-system | grep hbn +---- diff --git a/modules/nw-dpf-creating-hcp-secrets.adoc b/modules/nw-dpf-creating-hcp-secrets.adoc new file mode 100644 index 000000000000..638b1dfbd01e --- /dev/null +++ b/modules/nw-dpf-creating-hcp-secrets.adoc @@ -0,0 +1,79 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-hosted-cluster-provisioning.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-creating-hcp-secrets_{context}"] += Create secrets for the hosted cluster + +[role="_abstract"] +You must create a pull secret and an SSH key secret in the clusters namespace before provisioning the hosted cluster. +The `DPFHCPProvisioner` resource references these secrets during hosted cluster creation. + +.Prerequisites + +* The environment variables described in "Hosted cluster provisioning environment variables" are set. +* A valid {product-title} pull secret file exists at the path specified by `OPENSHIFT_PULL_SECRET`. +* An SSH public key file exists at the path specified by `SSH_KEY`. + +.Procedure + +. Create the clusters namespace: ++ +[source,terminal] +---- +$ oc create namespace $CLUSTERS_NAMESPACE +---- + +. Create the pull secret: ++ +[source,terminal] +---- +$ oc create secret generic $PULL_SECRET_NAME \ + --from-file=.dockerconfigjson=$OPENSHIFT_PULL_SECRET \ + --type=kubernetes.io/dockerconfigjson \ + -n $CLUSTERS_NAMESPACE +---- ++ +.Example output +[source,terminal] +---- +secret/pull-secret created +---- + +. Create the SSH key secret: ++ +[source,terminal] +---- +$ oc create secret generic $SSH_KEY_SECRET_NAME \ + --from-file=id_rsa.pub=$SSH_KEY \ + -n $CLUSTERS_NAMESPACE +---- ++ +.Example output +[source,terminal] +---- +secret/ssh-key created +---- + +.Verification + +* Verify that the secrets were created in the clusters namespace: ++ +[source,terminal] +---- +$ oc get secrets -n $CLUSTERS_NAMESPACE +---- ++ +.Example output +[source,terminal] +---- +NAME TYPE DATA AGE +pull-secret kubernetes.io/dockerconfigjson 1 10s +ssh-key Opaque 1 5s +---- + +[NOTE] +==== +The BlueField {product-title} layer image referenced by `BLUEFIELD_OCP_IMAGE` might require authentication to the Quay or Red{nbsp}Hat registry. Ensure that the pull secret includes credentials for that image registry. +==== diff --git a/modules/nw-dpf-creating-ovnk-credentials.adoc b/modules/nw-dpf-creating-ovnk-credentials.adoc new file mode 100644 index 000000000000..a8d4f18a3982 --- /dev/null +++ b/modules/nw-dpf-creating-ovnk-credentials.adoc @@ -0,0 +1,89 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-operator-installation.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-creating-ovnk-credentials_{context}"] += Create the OVN-Kubernetes credential request and role bindings + +[role="_abstract"] +You can create a `DPUServiceCredentialRequest` custom resource and its associated role bindings to enable the OVN-Kubernetes DPU service on the hosted cluster to authenticate with the management cluster API server. +The `ClusterRoleBinding` grants the required permissions for OVN node network operations. + +.Prerequisites + +* The DPF Operator is installed. +* The `DPFOperatorConfig` resource is created. + +.Procedure + +. Create a file named `dpucredentialreq.yaml` with the following content: ++ +[source,yaml] +---- +apiVersion: svc.dpu.nvidia.com/v1alpha1 +kind: DPUServiceCredentialRequest +metadata: + name: ovn-dpu + namespace: dpf-operator-system +spec: + serviceAccount: + name: ovn-kubernetes-node-dpu-service + namespace: openshift-ovn-kubernetes + duration: 24h + type: tokenFile + secret: + name: ovn-dpu + namespace: dpf-operator-system + metadata: + labels: + dpu.nvidia.com/image-pull-secret: "" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: openshift-ovn-kubernetes-node-limited-dpu-service + namespace: openshift-ovn-kubernetes +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: openshift-ovn-kubernetes-node-limited +subjects: +- kind: ServiceAccount + name: ovn-kubernetes-node-dpu-service + namespace: openshift-ovn-kubernetes +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ovn-kubernetes-node-limited-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: openshift-ovn-kubernetes-node-limited +subjects: +- kind: ServiceAccount + name: ovn-kubernetes-node-dpu-service + namespace: openshift-ovn-kubernetes +---- + +. Apply the resource file: ++ +[source,terminal] +---- +$ oc apply -f dpucredentialreq.yaml +---- + +.Verification + +* Verify that the credential request and role bindings are created: ++ +[source,terminal] +---- +$ oc get dpuservicecredentialrequest -n dpf-operator-system +---- ++ +[source,terminal] +---- +$ oc get clusterrolebinding ovn-kubernetes-node-limited-binding +---- diff --git a/modules/nw-dpf-creating-ovnk-service.adoc b/modules/nw-dpf-creating-ovnk-service.adoc new file mode 100644 index 000000000000..3d0decf280f3 --- /dev/null +++ b/modules/nw-dpf-creating-ovnk-service.adoc @@ -0,0 +1,91 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-operator-installation.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-creating-ovnk-service_{context}"] += Create the OVN-Kubernetes DPU service resources + +[role="_abstract"] +You can create the OVN-Kubernetes DPU service by defining a `DPUServiceTemplate` and a `DPUServiceConfiguration` custom resource. +The OVN-Kubernetes service provides pod networking on the DPU. + +.Prerequisites + +* The DPF Operator is installed. +* The `DPFOperatorConfig` resource is created. +* The DPF Operator environment variables are set. For details, see "DPF Operator installation environment variables". + +.Procedure + +. Create a file named `ovn-k.yaml` with the following content: ++ +[source,yaml] +---- +apiVersion: svc.dpu.nvidia.com/v1alpha1 +kind: DPUServiceTemplate +metadata: + name: ovn + namespace: dpf-operator-system +spec: + deploymentServiceName: "ovn" + helmChart: + source: + repoURL: $OVN_TEMPLATE_CHART_URL + chart: ovn-kubernetes-chart + version: $OVN_CHART_VERSION + values: + commonManifests: + enabled: true + dpuManifests: + enabled: true + cniBinDir: /var/lib/cni/bin/ + cniConfDir: /run/multus/cni/net.d + leaseNamespace: "openshift-ovn-kubernetes" + gatewayOpts: "--gateway-interface=br-dpu" +--- +apiVersion: svc.dpu.nvidia.com/v1alpha1 +kind: DPUServiceConfiguration +metadata: + name: ovn + namespace: dpf-operator-system +spec: + deploymentServiceName: "ovn" + serviceConfiguration: + helmChart: + values: + global: + enableOvnKubeIdentity: false + imagePullSecretName: "dpf-pull-secret" + k8sAPIServer: https://$HOST_CLUSTER_API:$TARGETCLUSTER_API_SERVER_PORT + podNetwork: $POD_CIDR/23 + serviceNetwork: $SERVICE_CIDR + mtu: $OVN_MTU + dpuManifests: + ovnMultiNetworkEnable: "false" + kubernetesSecretName: "ovn-dpu" + vtepCIDR: $VTEP_CIDR + hostCIDR: $TARGETCLUSTER_NODE_CIDR + ipamPool: "pool1" + ipamPoolType: "cidrpool" + ipamVTEPIPIndex: 0 + ipamPFIPIndex: 1 + cniBinDir: "/var/lib/cni/bin/" + cniConfDir: "/run/multus/cni/net.d" +---- + +. Apply the resource file: ++ +[source,terminal] +---- +$ envsubst < ovn-k.yaml | oc apply -f - +---- + +.Verification + +* Verify that the OVN-Kubernetes service resources are created: ++ +[source,terminal] +---- +$ oc get dpuservicetemplate,dpuserviceconfiguration -n dpf-operator-system | grep ovn +---- diff --git a/modules/nw-dpf-creating-service-interfaces.adoc b/modules/nw-dpf-creating-service-interfaces.adoc new file mode 100644 index 000000000000..a4437d750b0b --- /dev/null +++ b/modules/nw-dpf-creating-service-interfaces.adoc @@ -0,0 +1,100 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-operator-installation.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-creating-service-interfaces_{context}"] += Create the DPUServiceInterface resources + +[role="_abstract"] +You can create `DPUServiceInterface` custom resources to define interface objects that are specified in service chains. +You must create physical interface resources for the DPU ports and an OVN-Kubernetes interface resource for host workloads. + +.Prerequisites + +* The DPF Operator is installed. +* The `DPFOperatorConfig` resource is created. + +.Procedure + +. Create a file named `physical-if.yaml` with the following content to define the physical DPU port interfaces: ++ +[source,yaml] +---- +apiVersion: svc.dpu.nvidia.com/v1alpha1 +kind: DPUServiceInterface +metadata: + name: p0 + namespace: dpf-operator-system +spec: + template: + spec: + template: + metadata: + labels: + uplink: "p0" + spec: + interfaceType: physical + physical: + interfaceName: p0 +--- +apiVersion: svc.dpu.nvidia.com/v1alpha1 +kind: DPUServiceInterface +metadata: + name: p1 + namespace: dpf-operator-system +spec: + template: + spec: + template: + metadata: + labels: + uplink: "p1" + spec: + interfaceType: physical + physical: + interfaceName: p1 +---- + +. Apply the physical interface resource file: ++ +[source,terminal] +---- +$ oc apply -f physical-if.yaml +---- + +. Create a file named `ovnk-if.yaml` with the following content to define the OVN-Kubernetes interface: ++ +[source,yaml] +---- +apiVersion: svc.dpu.nvidia.com/v1alpha1 +kind: DPUServiceInterface +metadata: + name: ovn + namespace: dpf-operator-system +spec: + template: + spec: + template: + metadata: + labels: + port: ovn + spec: + interfaceType: ovn +---- + +. Apply the OVN-Kubernetes interface resource file: ++ +[source,terminal] +---- +$ oc apply -f ovnk-if.yaml +---- + +.Verification + +* Verify that all `DPUServiceInterface` resources are created: ++ +[source,terminal] +---- +$ oc get dpuserviceinterface -n dpf-operator-system +---- diff --git a/modules/nw-dpf-creating-service-ipam.adoc b/modules/nw-dpf-creating-service-ipam.adoc new file mode 100644 index 000000000000..5f30938064bc --- /dev/null +++ b/modules/nw-dpf-creating-service-ipam.adoc @@ -0,0 +1,62 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-operator-installation.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-creating-service-ipam_{context}"] += Create the DPUServiceIPAM resources + +[role="_abstract"] +You can create `DPUServiceIPAM` custom resources to configure IP address management for DPU services. +Two IPAM pools are required: one for the VTEP network used by the high-speed data plane, and one for loopback addresses used by the HBN service. + +.Prerequisites + +* The DPF Operator is installed. +* The `DPFOperatorConfig` resource is created. +* The DPF Operator environment variables are set. For details, see "DPF Operator installation environment variables". + +.Procedure + +. Create a file named `dpuservice-ipam.yaml` with the following content: ++ +[source,yaml] +---- +--- +apiVersion: svc.dpu.nvidia.com/v1alpha1 +kind: DPUServiceIPAM +metadata: + name: pool1 + namespace: dpf-operator-system +spec: + ipv4Network: + network: $VTEP_CIDR + gatewayIndex: 3 + prefixSize: 29 +--- +apiVersion: svc.dpu.nvidia.com/v1alpha1 +kind: DPUServiceIPAM +metadata: + name: loopback + namespace: dpf-operator-system +spec: + ipv4Network: + network: "11.0.0.0/24" + prefixSize: 32 +---- + +. Apply the resource file: ++ +[source,terminal] +---- +$ envsubst < dpuservice-ipam.yaml | oc apply -f - +---- + +.Verification + +* Verify that the `DPUServiceIPAM` resources are created: ++ +[source,terminal] +---- +$ oc get dpuserviceipam -n dpf-operator-system +---- diff --git a/modules/nw-dpf-creating-service-nads.adoc b/modules/nw-dpf-creating-service-nads.adoc new file mode 100644 index 000000000000..18fc4151aeb4 --- /dev/null +++ b/modules/nw-dpf-creating-service-nads.adoc @@ -0,0 +1,69 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-operator-installation.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-creating-service-nads_{context}"] += Create the DPUServiceNAD resources + +[role="_abstract"] +You can create `DPUServiceNAD` custom resources to define the network attachments available to DPU services on the hosted cluster. +Each NAD maps to an OVS bridge on the DPU and specifies the resource type, IPAM mode, and MTU configuration. + +The following network attachments are required: + +`mybrhbn`:: Maps to the `br-hbn` bridge, used by the HBN service. IPAM is disabled because IP allocation is handled by `DPUServiceIPAM`. +`mybrsfc`:: Maps to the `br-sfc` bridge, used for service function chaining. IPAM is enabled. + +Both use sub-function resource type. The MTU is set according to the `NODES_MTU` environment variable. + +.Prerequisites + +* The DPF Operator is installed. +* The `DPFOperatorConfig` resource is created. +* The DPF Operator environment variables are set. For details, see "DPF Operator installation environment variables". + +.Procedure + +. Create a file named `dpuservice-nad.yaml` with the following content: ++ +[source,yaml] +---- +apiVersion: svc.dpu.nvidia.com/v1alpha1 +kind: DPUServiceNAD +metadata: + name: mybrhbn + namespace: dpf-operator-system +spec: + resourceType: sf + ipam: false + bridge: "br-hbn" + serviceMTU: $NODES_MTU +--- +apiVersion: svc.dpu.nvidia.com/v1alpha1 +kind: DPUServiceNAD +metadata: + name: mybrsfc + namespace: dpf-operator-system +spec: + resourceType: sf + ipam: true + bridge: "br-sfc" + serviceMTU: $NODES_MTU +---- + +. Apply the resource file: ++ +[source,terminal] +---- +$ envsubst < dpuservice-nad.yaml | oc apply -f - +---- + +.Verification + +* Verify that the `DPUServiceNAD` resources are created: ++ +[source,terminal] +---- +$ oc get dpuservicenad -n dpf-operator-system +---- diff --git a/modules/nw-dpf-creating-sriov-config.adoc b/modules/nw-dpf-creating-sriov-config.adoc new file mode 100644 index 000000000000..c4e95a0dbf61 --- /dev/null +++ b/modules/nw-dpf-creating-sriov-config.adoc @@ -0,0 +1,72 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-operator-installation.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-creating-sriov-config_{context}"] += Create the NodeSRIOVDevicePluginConfig resource + +[role="_abstract"] +You can create a `NodeSRIOVDevicePluginConfig` custom resource to define how SR-IOV virtual functions on the management cluster worker nodes are allocated to DPF components. +This resource replaces the manual SR-IOV device plugin `DaemonSet` and control plane node patching from previous versions. + +.Prerequisites + +* The DPF Operator is installed. +* The `DPFOperatorConfig` resource is created. + +.Procedure + +. Create a file named `nodesriovdevicepluginconfig.yaml` with the following content: ++ +[source,yaml] +---- +apiVersion: noderesources.dpu.nvidia.com/v1alpha1 +kind: NodeSRIOVDevicePluginConfig +metadata: + name: bf3-p0-vfs + namespace: dpf-operator-system +spec: + devicePluginResources: + - name: bf3-p0-vfs-mgmt + type: vf + ranges: + - pfIndex: 0 + start: 1 + end: 1 + - name: bf3-p0-vfs + type: vf + options: + isRdma: true + ranges: + - pfIndex: 0 + start: 2 + end: 45 +---- ++ +where: ++ +`bf3-p0-vfs-mgmt`:: Reserves VF index 1 on PF0 for DPU management connectivity. +`bf3-p0-vfs`:: Allocates VF indices 2-45 on PF0 for workload traffic with RDMA enabled. The `pfIndex: 0` value refers to the first physical function of the BlueField-3 DPU. + +. Apply the resource file: ++ +[source,terminal] +---- +$ oc apply -f nodesriovdevicepluginconfig.yaml +---- + +.Verification + +* Verify that the `NodeSRIOVDevicePluginConfig` resource is created: ++ +[source,terminal] +---- +$ oc get nodesriovdevicepluginconfig -n dpf-operator-system +---- ++ +.Example output +[source,terminal] +---- +bf3-p0-vfs 30s +---- diff --git a/modules/nw-dpf-deploying-traffic-test-pods.adoc b/modules/nw-dpf-deploying-traffic-test-pods.adoc new file mode 100644 index 000000000000..4ef948e4514e --- /dev/null +++ b/modules/nw-dpf-deploying-traffic-test-pods.adoc @@ -0,0 +1,368 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-validation-telemetry.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-deploying-traffic-test-pods_{context}"] += Deploy traffic test pods and services + +[role="_abstract"] +You can deploy traffic test pods and services across the management cluster to validate end-to-end connectivity through the DPU data plane. +The test workloads include a master pod on a control plane node and worker pods on DPU-enabled nodes, with both standard and host-network configurations. + +.Prerequisites + +* The DPF Operator is installed and the DPU hosted cluster is provisioned. +* At least two DPU-enabled worker nodes are available. +* You have access to the management cluster as a user with the `cluster-admin` role. + +.Procedure + +. Create a file named `traffic-pods.yaml` with the following content: ++ +[source,yaml] +---- +# 1. Namespace +--- +apiVersion: v1 +kind: Namespace +metadata: + name: workload + +# 2. SCC RoleBinding (grants 'default' ServiceAccount in 'workload' NS access to 'privileged' SCC) +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: privileged-scc-default-sa + namespace: workload +subjects: +- kind: ServiceAccount + name: default + namespace: workload +roleRef: + kind: ClusterRole + name: system:openshift:scc:privileged + apiGroup: rbac.authorization.k8s.io + +# 3. Deployments and Services +# Deployment: traffic-test-master +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: traffic-test-master + namespace: workload + labels: + app: traffic-test-master +spec: + replicas: 1 + selector: + matchLabels: + app: traffic-test-master + template: + metadata: + labels: + app: traffic-test-master + spec: + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: traffic-test-master + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + containers: + - name: nginx + securityContext: + privileged: true + capabilities: + add: + - NET_ADMIN + image: nicolaka/netshoot + command: ["nc", "-kl", "5000"] + ports: + - containerPort: 5000 + name: tcp-server + resources: + requests: + cpu: 1 + memory: 1Gi + limits: + cpu: 1 + memory: 1Gi +--- +# Service: traffic-test-master +apiVersion: v1 +kind: Service +metadata: + name: traffic-test-master + namespace: workload + labels: + app: traffic-test-master +spec: + selector: + app: traffic-test-master + ports: + - protocol: TCP + port: 5000 + targetPort: 5000 +--- +# Service: traffic-test-master-nodeport +apiVersion: v1 +kind: Service +metadata: + name: traffic-test-master-nodeport + namespace: workload + labels: + app: traffic-test-master +spec: + type: NodePort + selector: + app: traffic-test-master + ports: + - protocol: TCP + port: 5000 + targetPort: 5000 +--- +# Deployment: traffic-test-worker +apiVersion: apps/v1 +kind: Deployment +metadata: + name: traffic-test-worker + namespace: workload + labels: + app: traffic-test-worker +spec: + replicas: 2 + selector: + matchLabels: + app: traffic-test-worker + template: + metadata: + labels: + app: traffic-test-worker + spec: + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: traffic-test-worker + nodeSelector: + feature.node.kubernetes.io/dpu-enabled: "" + containers: + - name: nginx + securityContext: + privileged: true + capabilities: + add: + - NET_ADMIN + image: nicolaka/netshoot + command: ["nc", "-kl", "5000"] + ports: + - containerPort: 5000 + name: tcp-server + resources: + requests: + cpu: 16 + memory: 6Gi + limits: + cpu: 16 + memory: 6Gi +--- +# Service: traffic-test-worker +apiVersion: v1 +kind: Service +metadata: + name: traffic-test-worker + namespace: workload + labels: + app: traffic-test-worker +spec: + selector: + app: traffic-test-worker + ports: + - protocol: TCP + port: 5000 + targetPort: 5000 +--- +# Service: traffic-test-worker-nodeport +apiVersion: v1 +kind: Service +metadata: + name: traffic-test-worker-nodeport + namespace: workload + labels: + app: traffic-test-worker +spec: + type: NodePort + selector: + app: traffic-test-worker + ports: + - protocol: TCP + port: 5000 + targetPort: 5000 +--- +# Deployment: traffic-test-worker-hostnetwork +apiVersion: apps/v1 +kind: Deployment +metadata: + name: traffic-test-worker-hostnetwork + namespace: workload + labels: + app: traffic-test-worker-hostnetwork +spec: + replicas: 2 + selector: + matchLabels: + app: traffic-test-worker-hostnetwork + template: + metadata: + labels: + app: traffic-test-worker-hostnetwork + spec: + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: traffic-test-worker-hostnetwork + nodeSelector: + feature.node.kubernetes.io/dpu-enabled: "" + hostNetwork: true + containers: + - name: nginx + securityContext: + privileged: true + capabilities: + add: + - NET_ADMIN + image: nicolaka/netshoot + command: ["nc", "-kl", "5000"] + ports: + - containerPort: 5000 + name: tcp-server + resources: + requests: + cpu: 1 + memory: 1Gi + limits: + cpu: 1 + memory: 1Gi +--- +# Service: traffic-test-worker-hostnetwork +apiVersion: v1 +kind: Service +metadata: + name: traffic-test-worker-hostnetwork + namespace: workload + labels: + app: traffic-test-worker-hostnetwork +spec: + selector: + app: traffic-test-worker-hostnetwork + ports: + - protocol: TCP + port: 5000 + targetPort: 5000 +--- +# Service: traffic-test-worker-hostnetwork-nodeport +apiVersion: v1 +kind: Service +metadata: + name: traffic-test-worker-hostnetwork-nodeport + namespace: workload + labels: + app: traffic-test-worker-hostnetwork +spec: + type: NodePort + selector: + app: traffic-test-worker-hostnetwork + ports: + - protocol: TCP + port: 5000 + targetPort: 5000 +---- ++ +The manifest creates the following resources: ++ +* A `workload` namespace for the test pods. +* A `RoleBinding` that grants the `default` service account in the `workload` namespace access to the `privileged` security context constraint. +* A `traffic-test-master` deployment and `ClusterIP` and `NodePort` services on a control plane node. +* A `traffic-test-worker` deployment with two replicas and `ClusterIP` and `NodePort` services on DPU-enabled worker nodes. +* A `traffic-test-worker-hostnetwork` deployment with two replicas that uses host networking on DPU-enabled worker nodes, with `ClusterIP` and `NodePort` services. + +. Apply the manifest: ++ +[source,terminal] +---- +$ oc apply -f traffic-pods.yaml +---- ++ +.Example output +[source,terminal] +---- +rolebinding.rbac.authorization.k8s.io/privileged-scc-default-sa created +deployment.apps/traffic-test-master created +service/traffic-test-master created +service/traffic-test-master-nodeport created +deployment.apps/traffic-test-worker created +service/traffic-test-worker created +service/traffic-test-worker-nodeport created +deployment.apps/traffic-test-worker-hostnetwork created +service/traffic-test-worker-hostnetwork created +service/traffic-test-worker-hostnetwork-nodeport created +---- + +.Verification + +. Verify that the test pods are running: ++ +[source,terminal] +---- +$ oc get pods -n workload -o wide +---- ++ +.Example output +[source,terminal] +---- +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +traffic-test-master-7448bb5cc-mdftd 1/1 Running 0 2m10s 10.129.0.145 master-2 +traffic-test-worker-776486fb68-krz54 1/1 Running 0 2m10s 10.128.2.9 host-worker1 +traffic-test-worker-776486fb68-lz8vf 1/1 Running 0 2m10s 10.131.0.9 host-worker2 +traffic-test-worker-hostnetwork-596d569d99-cjpns 1/1 Running 0 2m10s 10.0.110.11 host-worker1 +traffic-test-worker-hostnetwork-596d569d99-x6m7r 1/1 Running 0 2m10s 10.0.110.12 host-worker2 +---- ++ +Confirm that the `traffic-test-master` pod is on a control plane node and that the `traffic-test-worker` pods are distributed across different DPU-enabled worker nodes. + +. Verify that the services are created: ++ +[source,terminal] +---- +$ oc get svc -n workload +---- ++ +.Example output +[source,terminal] +---- +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +traffic-test-master ClusterIP 172.30.102.123 5000/TCP 13m +traffic-test-master-nodeport NodePort 172.30.98.22 5000:31368/TCP 13m +traffic-test-worker ClusterIP 172.30.187.147 5000/TCP 13m +traffic-test-worker-hostnetwork ClusterIP 172.30.122.242 5000/TCP 13m +traffic-test-worker-hostnetwork-nodeport NodePort 172.30.122.214 5000:30209/TCP 13m +traffic-test-worker-nodeport NodePort 172.30.108.72 5000:32570/TCP 13m +---- diff --git a/modules/nw-dpf-deployment-flow-overview.adoc b/modules/nw-dpf-deployment-flow-overview.adoc new file mode 100644 index 000000000000..9a15089b06e6 --- /dev/null +++ b/modules/nw-dpf-deployment-flow-overview.adoc @@ -0,0 +1,28 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/about-dpf-operator.adoc + +:_mod-docs-content-type: CONCEPT +[id="nw-dpf-deployment-flow-overview_{context}"] += DPF deployment flow overview + +[role="_abstract"] +The end-to-end deployment process for the NVIDIA DPF Operator follows a series of high-level steps, from management cluster setup through workload verification. + +The deployment flow consists of the following steps: + +. *Management cluster setup:* Install and configure a standard {product-title} cluster on x86 servers with control-plane nodes only. + +. *Management cluster configuration:* Configure nodes and cluster-level settings, then install and configure the required operators on the management cluster. + +. *DPF installation:* Deploy the DPF Operator, controllers, DPF resources, and DPU service definitions on the management cluster. You must install the DPF Operator before the `dpf-hcp-provisioner-operator` because the provisioner requires DPF custom resource definitions (CRDs) such as `DPUCluster`, `DPUFlavor`, `DPUDeployment`, and `DPFOperatorConfig`. + +. *Hosted cluster creation:* The `dpf-hcp-provisioner-operator` automates the creation of a hosted DPU cluster by using hosted control planes. The provisioner references the `DPUDeployment` resource during ignition generation. + +. *Worker node scale-out and DPU provisioning:* When worker nodes with DPUs are added to the cluster, the DPF Operator flashes the DPUs with a Red Hat Enterprise Linux CoreOS (RHCOS) image and configures them to join the hosted cluster as worker nodes. + +. *Service deployment:* After the DPU hosted cluster is operational, data plane DPU services and chains are deployed by DPF. + +. *Worker node integration:* Approve CSRs, monitor DPU provisioning, and configure security context constraint (SCC) bindings on the hosted cluster. + +. *Verification:* Validate end-to-end traffic flow, BGP peering, and OVS bridge configuration. diff --git a/modules/nw-dpf-dts-observability-overview.adoc b/modules/nw-dpf-dts-observability-overview.adoc new file mode 100644 index 000000000000..822822d76f77 --- /dev/null +++ b/modules/nw-dpf-dts-observability-overview.adoc @@ -0,0 +1,48 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-validation-telemetry.adoc + +:_mod-docs-content-type: CONCEPT +[id="nw-dpf-dts-observability-overview_{context}"] += DPU telemetry observability with DTS + +[role="_abstract"] +The DOCA Telemetry Service (DTS) exposes DPU hardware telemetry, such as PCIe link speed, uplink throughput, packets, errors, and NIC channel activity, as Prometheus metrics. +You can view these metrics by using the {product-title} web console or a Grafana dashboard. + +DTS runs on every DPU in the hosted cluster and collects counters from sysfs and ethtool providers. +{product-title} includes a built-in Prometheus instance, so you do not need to deploy a separate monitoring stack to scrape DTS metrics. + +== How DPF exposes DTS metrics to the management cluster + +DTS runs on the DPU hosted cluster, but Prometheus runs on the management cluster. +DPF bridges this gap with a built-in port-mirroring mechanism. + +When a `DPUService` resource declares a port in its `configPorts` field, DPF performs the following actions: + +* Publishes the service port as a `NodePort` on the DPU hosted cluster. +* Creates a mirror `Service` on the management cluster, labeled with `dpu.nvidia.com/exposed-port-for-dpucluster`. + +The management-cluster Prometheus then scrapes the mirror service. +This mechanism requires no additional configuration beyond the standard DTS deployment objects. + +== DTS deployment objects + +DTS is deployed through three standard DPF resources: + +`DPUServiceTemplate`:: +Defines the Helm chart for the DOCA Telemetry Service, the DTS container image, and the metrics port. The `configMapData.prometheus.port` field is set to `9189`. + +`DPUServiceConfiguration`:: +Declares the service port `httpserverport: 9189` under `configPorts`. +This declaration triggers the management-cluster port-mirroring mechanism described previously. + +`DPUDeployment`:: +References the template and configuration so that DTS is rolled out to the DPUs as a `DaemonSet` on the DPU hosted cluster. +DTS defaults to the `sysfs` and `ethtool` providers. + +[NOTE] +==== +In a standard DPF installation, these objects are applied automatically during the postinstallation step. +Apply them manually only when you are adding DTS to an existing cluster. +==== diff --git a/modules/nw-dpf-enabling-ovnk-dpu-host-mode.adoc b/modules/nw-dpf-enabling-ovnk-dpu-host-mode.adoc new file mode 100644 index 000000000000..c575e4925769 --- /dev/null +++ b/modules/nw-dpf-enabling-ovnk-dpu-host-mode.adoc @@ -0,0 +1,58 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-worker-nodes-provisioning.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-enabling-ovnk-dpu-host-mode_{context}"] += Enable OVN-Kubernetes DPU-Host mode + +[role="_abstract"] +You can enable DPU-Host mode on worker nodes that use accelerated OVN-Kubernetes CNI by creating `ConfigMap` resources on the management cluster. +These resources disable the network node identity feature and enable DPU acceleration support. + +.Prerequisites + +* You have access to the management cluster as a user with the `cluster-admin` role. +* The OpenShift CLI (`oc`) is installed. +* The OVN-Kubernetes resource injector is installed. + +.Procedure + +. Create a file named `ovnk-acc.yaml` with the following content: ++ +[source,yaml] +---- +apiVersion: v1 +kind: ConfigMap +metadata: + name: network-node-identity + namespace: openshift-network-operator +data: + enabled: "false" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: hardware-offload-config + namespace: openshift-network-operator +data: + dpu-host-mode-label: "feature.node.kubernetes.io/dpu-enabled=" + mgmt-port-resource-name: "openshift.io/bf3-p0-vfs-mgmt" +---- ++ +The first `ConfigMap` disables the network node identity feature. +The second `ConfigMap` enables hardware offload for DPU-equipped nodes by specifying the node label that identifies DPU-enabled nodes and the management port resource name. + +. Apply the resource file: ++ +[source,terminal] +---- +$ oc apply -f ovnk-acc.yaml +---- ++ +.Example output +[source,terminal] +---- +configmap/network-node-identity created +configmap/hardware-offload-config created +---- diff --git a/modules/nw-dpf-enabling-ovnk-resource-injector.adoc b/modules/nw-dpf-enabling-ovnk-resource-injector.adoc new file mode 100644 index 000000000000..533bc2abe29d --- /dev/null +++ b/modules/nw-dpf-enabling-ovnk-resource-injector.adoc @@ -0,0 +1,72 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-worker-nodes-provisioning.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-enabling-ovnk-resource-injector_{context}"] += Enable the OVN-Kubernetes resource injector + +[role="_abstract"] +You can install the OVN-Kubernetes resource injector by using Helm to deploy a `MutatingAdmissionPolicy` that automatically injects SR-IOV virtual function resource requests and network attachment annotations into each pod scheduled to a worker node. + +[NOTE] +==== +This step requires the `MutatingAdmissionPolicy` feature gate to be enabled on the management cluster. +Virtual function resource capacity on worker nodes is provided by the `NodeSRIOVDevicePluginConfig` resource, which replaces the manual SR-IOV device plugin `DaemonSet` and control plane node patching used in earlier DPF versions. +==== + +.Prerequisites + +* You have access to the management cluster as a user with the `cluster-admin` role. +* The OpenShift CLI (`oc`) is installed. +* The Helm CLI (`helm`) is installed. +* The DPF Operator environment variables are set. For details, see "DPF Operator installation environment variables". +* The `MutatingAdmissionPolicy` feature gate is enabled on the management cluster. +* The `NodeSRIOVDevicePluginConfig` resource is created. + +.Procedure + +. Install the OVN-Kubernetes resource injector by using Helm: ++ +[source,terminal] +---- +$ helm install ovn-kubernetes-resource-injector \ + $OVN_TEMPLATE_CHART_URL/ovn-kubernetes-chart \ + --namespace openshift-ovn-kubernetes \ + --version ${OVN_CHART_VERSION} \ + --set ovn-kubernetes-resource-injector.enabled=true \ + --set resourceName=openshift.io/bf3-p0-vfs \ + --set nodeWithDPUManifests.enabled=false \ + --set nodeWithoutDPUManifests.enabled=false \ + --set dpuManifests.enabled=false \ + --set controlPlaneManifests.enabled=false \ + --set commonManifests.enabled=false \ + --wait +---- ++ +.Example output +[source,terminal] +---- +NAME: ovn-kubernetes-resource-injector +LAST DEPLOYED: Sun Nov 2 17:10:29 2025 +NAMESPACE: openshift-ovn-kubernetes +STATUS: deployed +REVISION: 1 +TEST SUITE: None +---- + +.Verification + +* Verify the resource injector mutating webhook configuration was applied: ++ +[source,terminal] +---- +$ oc get mutatingwebhookconfiguration | grep ovn +---- ++ +.Example output +[source,terminal] +---- +NAME WEBHOOKS AGE +ovn-kubernetes-resource-injector 1 22h +---- diff --git a/modules/nw-dpf-enabling-user-workload-monitoring.adoc b/modules/nw-dpf-enabling-user-workload-monitoring.adoc new file mode 100644 index 000000000000..7c4abd84372c --- /dev/null +++ b/modules/nw-dpf-enabling-user-workload-monitoring.adoc @@ -0,0 +1,65 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-validation-telemetry.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-enabling-user-workload-monitoring_{context}"] += Enable user workload monitoring for DTS + +[role="_abstract"] +{product-title} includes Prometheus, but by default it only monitors {product-title} platform components. +You must enable user workload monitoring so that Prometheus can scrape user namespaces where DPF and DTS run, such as `dpf-operator-system`. + +.Prerequisites + +* A DPF cluster is deployed with at least one provisioned DPU. +* You have access to the management cluster as a user with the `cluster-admin` role. + +.Procedure + +. Create a `ConfigMap` to enable user workload monitoring in the `openshift-monitoring` namespace: ++ +[source,yaml] +---- +apiVersion: v1 +kind: ConfigMap +metadata: + name: cluster-monitoring-config + namespace: openshift-monitoring +data: + config.yaml: | + enableUserWorkload: true +---- ++ +[NOTE] +==== +If the `cluster-monitoring-config` `ConfigMap` already exists with other settings, edit it instead of replacing it. +Add only the `enableUserWorkload: true` line to the existing `config.yaml` data. +==== + +. Apply the `ConfigMap`: ++ +[source,terminal] +---- +$ oc apply -f cluster-monitoring-config.yaml +---- + +.Verification + +* Verify that the user workload monitoring pods are running in the `openshift-user-workload-monitoring` namespace: ++ +[source,terminal] +---- +$ oc -n openshift-user-workload-monitoring get pods +---- ++ +.Example output +[source,terminal] +---- +NAME READY STATUS RESTARTS AGE +prometheus-operator-... 1/1 Running 0 ... +prometheus-user-workload-0 ... Running 0 ... +thanos-ruler-user-workload-0 ... Running 0 ... +---- ++ +Confirm that pods named `prometheus-user-workload`, `thanos-ruler-user-workload`, and `prometheus-operator` are all in a `Running` state. diff --git a/modules/nw-dpf-environment-variables.adoc b/modules/nw-dpf-environment-variables.adoc new file mode 100644 index 000000000000..26f35451ad61 --- /dev/null +++ b/modules/nw-dpf-environment-variables.adoc @@ -0,0 +1,113 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-operator-installation.adoc + +:_mod-docs-content-type: REFERENCE +[id="nw-dpf-environment-variables_{context}"] += DPF Operator installation environment variables + +[role="_abstract"] +The following environment variables are used throughout the DPF Operator installation and configuration procedures. +These environment variables must be set before you apply DPF Operator installation and configuration manifests. + +.DPF Operator environment variables +[cols="1,2,1",options="header"] +|=== +|Variable |Description |Example value + +|`CLUSTER_NAME` +|The name of the management cluster. +|`mycluster` + +|`BASE_DOMAIN` +|The base domain for the management cluster. +|`example.com` + +|`HOST_CLUSTER_API` +|The API server hostname of the management cluster. Derived from `CLUSTER_NAME` and `BASE_DOMAIN`. +|`api.mycluster.example.com` + +|`TAG` +|The version tag for the DPF Operator Helm chart. +|`v26.4.1-beta.1` + +|`TARGETCLUSTER_API_SERVER_PORT` +|The port number of the hosted cluster API server. +|`6443` + +|`TARGETCLUSTER_NODE_CIDR` +|The CIDR range for the hosted cluster node network. +|`10.0.0.0/16` + +|`POD_CIDR` +|The CIDR range for the pod network. +|`10.128.0.0/14` + +|`SERVICE_CIDR` +|The CIDR range for the service network. +|`172.30.0.0/16` + +|`VTEP_CIDR` +|The CIDR range for the DPU high-speed network IP addresses. This range must match the network infrastructure design. +|`192.168.200.0/29` + +|`NUM_VFS` +|The number of SR-IOV virtual functions to create on each DPU. +|`46` + +|`HBN_HELM_REPO_URL` +|The Helm repository URL for the Host-Based Networking service chart. +|`\https://helm.ngc.nvidia.com/nvidia/doca` + +|`HBN_HELM_CHART_VERSION` +|The version of the HBN Helm chart. +|`3.4.0` + +|`HBN_IMAGE_REPO` +|The container image repository for the HBN service. +|`nvcr.io/nvidia/doca/doca_hbn` + +|`HBN_IMAGE_TAG` +|The container image tag for the HBN service. +|`3.4.0-doca3.4.0` + +|`DTS_HELM_REPO_URL` +|The Helm repository URL for the DOCA Telemetry Service chart. +|`\https://helm.ngc.nvidia.com/nvidia/doca` + +|`DTS_HELM_CHART_VERSION` +|The version of the DOCA Telemetry Service Helm chart. +|`1.25.5` + +|`DTS_IMAGE` +|The container image for the DOCA Telemetry Service. +|`nvcr.io/nvidia/doca/doca_telemetry:1.25.5-doca3.4.0` + +|`OVN_TEMPLATE_CHART_URL` +|The OCI chart URL for the OVN-Kubernetes Helm chart. +|`oci://ghcr.io/mellanox/charts` + +|`OVN_CHART_VERSION` +|The version of the OVN-Kubernetes Helm chart. +|`v26.4.0-ocpbeta` + +|`OVN_MTU` +|The MTU value for OVN-Kubernetes networking. Use `1400` for standard MTU or `8940` for jumbo frames. +|`1400` + +|`BFB_URL` +|The download URL for the BlueField Bootstream File image used to provision DPUs. Replace the placeholder with the full URL for the BFB image required by your deployment. +|`\https://content.mellanox.com/BlueField/BFBs/` + +|`HOSTED_CLUSTER_NAME` +|The name of the hosted cluster running on the DPUs. +|`dpf-hosted` + +|`REGISTRY` +|The Helm chart registry URL for the DPF Operator. Set during environment setup. +|Defined during environment setup + +|`NODES_MTU` +|The MTU value for the node network interfaces. Set during environment setup. +|Defined during environment setup +|=== diff --git a/modules/nw-dpf-hardware-requirements.adoc b/modules/nw-dpf-hardware-requirements.adoc new file mode 100644 index 000000000000..87a9a49685e6 --- /dev/null +++ b/modules/nw-dpf-hardware-requirements.adoc @@ -0,0 +1,103 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/about-dpf-operator.adoc + +:_mod-docs-content-type: REFERENCE +[id="nw-dpf-hardware-requirements_{context}"] += DPF hardware requirements + +[role="_abstract"] +The following hardware specifications are the minimum requirements for deploying DPF v26.4 on {product-title} {product-version}. + +== Workstation + +A workstation with the following command-line interface (CLI) tools installed: + +* OpenShift CLI (`oc`) is installed. You can download it from the link:https://mirror.openshift.com/pub/openshift-v4/clients/ocp/[OpenShift mirror]. +* Helm CLI (`helm`) is installed. For installation instructions, see the link:https://helm.sh/docs/intro/install/[Helm installation guide]. + +== Control plane nodes + +Three nodes form the control plane of the management cluster. + +.Control plane node requirements +[cols="1,3",options="header"] +|=== +|Component |Requirement + +|Form factor +|Virtual machines or physical servers + +|Memory +|60 GB RAM + +|CPU +|16 vCPUs (Intel/AMD x86_64) + +|Storage +|120 GB NVMe/SSD storage, plus an additional 80 GB disk for {lvms} + +|Networking +|1x 1GbE network interface + +|DPUs +|DPUs must not be installed on control plane nodes +|=== + +== Worker nodes + +Two physical x86 servers host the NVIDIA BlueField-3 DPUs and act as worker nodes for the management cluster. + +.Worker node requirements +[cols="1,3",options="header"] +|=== +|Component |Requirement + +|Memory +|256 GB RAM + +|CPU +|16 cores (Intel/AMD x86_64) + +|Storage +|500 GB NVMe/SSD minimum for the base operating system + +|DPU slot +|PCIe Gen 5 x16 slot required. Each server can have multiple DPUs but only one NVIDIA BlueField-3 DPU can be provisioned. + +|BIOS settings +|SR-IOV must be enabled. In-Band Manageability Interface must be enabled. +|=== + +[NOTE] +==== +As part of the installation process, a Linux bridge named `br-dpu` is automatically created on the worker node's physical management port by using a MachineConfig custom resource to facilitate control-plane traffic from the DPU through the host server. +==== + +== NVIDIA BlueField-3 DPUs + +One NVIDIA BlueField-3 DPU is required per worker node. + +.BlueField-3 DPU requirements +[cols="1,3",options="header"] +|=== +|Component |Requirement + +|Model +|BlueField-3: link:https://docs.nvidia.com/networking/display/bf3dpu/specifications#src-2449222537_Specifications-B3240DPUsSpecifications[B3240], link:https://docs.nvidia.com/networking/display/bf3dpu/specifications#src-2449222537_Specifications-B3220DPUsSpecifications[B3220], or link:https://docs.nvidia.com/networking/display/bf3dpu/specifications#src-2449222537_Specifications-B3210DPUsSpecifications[B3210] + +|Memory +|32 GB. Dual-port DPUs with 32 GB require an external power connection to the x86 server. + +|Networking +|Dual 200GbE ports per DPU. Both ports must be connected to the high-speed switch for ECMP routing. + +|Management +|The out-of-band management port is not used in this configuration. + +|Operating system +|The DPUs are provisioned with a BlueField Bootstream File (BFB) containing a Red Hat Enterprise Linux CoreOS (RHCOS) base image. + +|Firmware version +|32.49.1014 +|=== diff --git a/modules/nw-dpf-hcp-environment-variables.adoc b/modules/nw-dpf-hcp-environment-variables.adoc new file mode 100644 index 000000000000..9914f9c70578 --- /dev/null +++ b/modules/nw-dpf-hcp-environment-variables.adoc @@ -0,0 +1,83 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-hosted-cluster-provisioning.adoc + +:_mod-docs-content-type: REFERENCE +[id="nw-dpf-hcp-environment-variables_{context}"] += Hosted cluster provisioning environment variables + +[role="_abstract"] +The following environment variables are used throughout the hosted cluster provisioning procedures. +These environment variables must be set before you create secrets, the `DPUCluster` resource, or the `DPFHCPProvisioner` resource. + +.Hosted cluster provisioning environment variables +[cols="1,2,1",options="header"] +|=== +|Variable |Description |Example value + +|`HOSTED_CLUSTER_NAME` +|The name of the hosted DPU cluster. +|`dpf-hosted` + +|`OPENSHIFT_VERSION` +|The {product-title} version for the hosted cluster. +|`4.22.0` + +|`CLUSTERS_NAMESPACE` +|The namespace where hosted cluster resources are created. +|`clusters` + +|`BASE_DOMAIN` +|The base DNS domain for the hosted cluster. +|`example.com` + +|`BLUEFIELD_OCP_IMAGE` +|The BlueField {product-title} layer container image URL. When set, this value is used as `machineOSURL` in the `DPFHCPProvisioner` resource and skips automatic image lookup. This example value matches the GA draft reference deployment; confirm the supported image for your environment before use. +|`quay.io/eelgaev/bluefield-ocp:4.21.0_3.4.0-beta-v2` + +|`ETCD_STORAGE_CLASS` +|The storage class used for etcd persistent volume claims. +|`lvms-vg1` + +|`OCP_RELEASE_IMAGE` +|The {product-title} release image for the hosted cluster. Derived from `OPENSHIFT_VERSION`. +|`quay.io/openshift-release-dev/ocp-release:4.22.0-multi` + +|`HOSTED_CLUSTER_VIP` +|The virtual IP address for the hosted cluster API server. +|`203.0.113.200` + +|`PULL_SECRET_NAME` +|The name of the Kubernetes secret that contains the pull secret for the hosted cluster. +|`pull-secret` + +|`OPENSHIFT_PULL_SECRET` +|The file path to the pull secret JSON file on your workstation. +|`/root/pull-secret.txt` + +|`SSH_KEY_SECRET_NAME` +|The name of the Kubernetes secret that contains the SSH public key for the hosted cluster. +|`ssh-key` + +|`SSH_KEY` +|The file path to the SSH public key file on your workstation. +|`/root/.ssh/id_rsa.pub` +|=== + +Set all environment variables in your shell before proceeding: + +[source,terminal] +---- +$ export HOSTED_CLUSTER_NAME="dpf-hosted" +$ export OPENSHIFT_VERSION="4.22.0" +$ export CLUSTERS_NAMESPACE="clusters" +$ export BASE_DOMAIN="example.com" +$ export BLUEFIELD_OCP_IMAGE="quay.io/eelgaev/bluefield-ocp:4.21.0_3.4.0-beta-v2" +$ export ETCD_STORAGE_CLASS="lvms-vg1" +$ export OCP_RELEASE_IMAGE="quay.io/openshift-release-dev/ocp-release:${OPENSHIFT_VERSION}-multi" +$ export HOSTED_CLUSTER_VIP="203.0.113.200" +$ export PULL_SECRET_NAME="pull-secret" +$ export OPENSHIFT_PULL_SECRET="/root/pull-secret.txt" +$ export SSH_KEY_SECRET_NAME="ssh-key" +$ export SSH_KEY="/root/.ssh/id_rsa.pub" +---- diff --git a/modules/nw-dpf-hosted-cluster-overview.adoc b/modules/nw-dpf-hosted-cluster-overview.adoc new file mode 100644 index 000000000000..69f9e3d1d0b3 --- /dev/null +++ b/modules/nw-dpf-hosted-cluster-overview.adoc @@ -0,0 +1,35 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-hosted-cluster-provisioning.adoc + +:_mod-docs-content-type: CONCEPT +[id="nw-dpf-hosted-cluster-overview_{context}"] += DPU hosted cluster provisioning with the dpf-hcp-provisioner-operator + +[role="_abstract"] +The `dpf-hcp-provisioner-operator` abstracts hosted control plane complexity for DPF by orchestrating the full lifecycle of hosted clusters for DPU environments. +The operator treats the hosted control plane as a black box and maintains a 1:1:1 relationship: each `DPFHCPProvisioner` custom resource maps to exactly one `DPUCluster` and one `HostedCluster`. + +The operator provides the following capabilities: + +HostedCluster lifecycle management:: +Creates, updates, and deletes `HostedCluster`, `NodePool`, and associated secret resources. + +Automatic CSR approval:: +Approves Certificate Signing Requests from DPU worker nodes joining the hosted cluster. + +BlueField {product-title} layer image lookup:: +Matches {product-title} release images to corresponding BlueField container images by using container registry tag lookup. +This step is skipped when `machineOSURL` is provided in the `DPFHCPProvisioner` custom resource. + +Kubeconfig injection:: +Extracts the `HostedCluster` kubeconfig and injects it into the `DPUCluster` custom resource, enabling management cluster communication with the DPU hosted cluster. + +MetalLB configuration:: +Deploys `IPAddressPool` and `L2Advertisement` resources for `LoadBalancer` service exposure. + +Ignition generation:: +Generates BlueField-specific ignition configurations from hosted control plane ignition for DPU node provisioning. + +Status translation:: +Mirrors `HostedCluster` conditions to `DPFHCPProvisioner` status without exposing hosted control plane internals. diff --git a/modules/nw-dpf-installing-cert-manager.adoc b/modules/nw-dpf-installing-cert-manager.adoc new file mode 100644 index 000000000000..6801e115e465 --- /dev/null +++ b/modules/nw-dpf-installing-cert-manager.adoc @@ -0,0 +1,74 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-environment-setup.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-installing-cert-manager_{context}"] += Install the cert-manager Operator + +[role="_abstract"] +The cert-manager Operator automates the management, issuance, and renewal of TLS certificates within the cluster. +You must install it before deploying the DPF Operator. + +.Prerequisites + +* You have access to the cluster as a user with the `cluster-admin` role. +* You have installed the OpenShift CLI (`oc`). + +.Procedure + +. Create a file named `cert-manager-operator.yaml` with the following content: ++ +[source,yaml] +---- +apiVersion: v1 +kind: Namespace +metadata: + name: cert-manager +--- +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: openshift-cert-manager-operator + namespace: cert-manager +spec: + targetNamespaces: + - cert-manager +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: openshift-cert-manager-operator + namespace: cert-manager +spec: + channel: stable-v1 + name: openshift-cert-manager-operator + source: redhat-operators + sourceNamespace: openshift-marketplace +---- + +. Apply the file: ++ +[source,terminal] +---- +$ oc apply -f cert-manager-operator.yaml +---- + +.Verification + +* Verify that the operator pods are running: ++ +[source,terminal] +---- +$ oc get pods -n cert-manager +---- ++ +.Example output +[source,terminal] +---- +NAME READY STATUS RESTARTS AGE +cert-manager-7cfb4fbb84-wj2gg 1/1 Running 0 2d22h +cert-manager-cainjector-854f669657-zlrqd 1/1 Running 0 2d22h +cert-manager-operator-controller-manager-cd468b77f-9k24n 1/1 Running 0 2d22h +cert-manager-webhook-68fd6d5f5c-2pcq5 1/1 Running 0 2d22h +---- diff --git a/modules/nw-dpf-installing-dpf-operator.adoc b/modules/nw-dpf-installing-dpf-operator.adoc new file mode 100644 index 000000000000..165e82ad5d92 --- /dev/null +++ b/modules/nw-dpf-installing-dpf-operator.adoc @@ -0,0 +1,82 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-operator-installation.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-installing-dpf-operator_{context}"] += Install the DPF Operator + +[role="_abstract"] +You can install the DPF Operator by using Helm to deploy the operator into the `dpf-operator-system` namespace on your management cluster. + +[IMPORTANT] +==== +You must install the DPF Operator before the `dpf-hcp-provisioner-operator` because the provisioner operator requires the following DPF custom resource definitions to be available: `DPUCluster`, `DPUFlavor`, `DPUDeployment`, and `DPFOperatorConfig`. +==== + +.Prerequisites + +* You have access to the management cluster as a user with the `cluster-admin` role. +* The OpenShift CLI (`oc`) is installed. +* The Helm CLI (`helm`) is installed. +* The DPF Operator environment variables are set. For details, see "DPF Operator installation environment variables". + +.Procedure + +. Add the DPF Helm repository and update the local cache: ++ +[source,terminal] +---- +$ helm repo add --force-update dpf-repository ${REGISTRY} +---- ++ +[source,terminal] +---- +$ helm repo update +---- + +. Install the DPF Operator by using Helm: ++ +[source,terminal] +---- +$ helm upgrade --install dpf-operator dpf-repository/dpf-operator \ + --namespace dpf-operator-system \ + --version "${TAG}" \ + --set kamajiEtcdDefrag.enabled=false \ + --set isOpenshift=true \ + --set enableNodeFeatureRules=false \ + --wait +---- + +.Verification + +. Verify that the operator controller manager deployment has rolled out successfully: ++ +[source,terminal] +---- +$ oc rollout status deployment --namespace dpf-operator-system dpf-operator-controller-manager +---- ++ +.Example output +[source,terminal] +---- +deployment "dpf-operator-controller-manager" successfully rolled out +---- + +. Verify that all pods in the `dpf-operator-system` namespace are ready: ++ +[source,terminal] +---- +$ oc wait --for=condition=ready --namespace dpf-operator-system pods --all +---- ++ +.Example output +[source,terminal] +---- +pod/argocd-application-controller-0 condition met +pod/argocd-redis-b4f94bb8d-wr86b condition met +pod/argocd-repo-server-96765f997-79k9q condition met +pod/argocd-server-648c7ff85f-7frtg condition met +pod/dpf-operator-controller-manager-7bf9744c5f-cwrgc condition met +pod/maintenance-operator-585767f779-8k2lx condition met +---- diff --git a/modules/nw-dpf-installing-gitops-operator.adoc b/modules/nw-dpf-installing-gitops-operator.adoc new file mode 100644 index 000000000000..941d5d08cb52 --- /dev/null +++ b/modules/nw-dpf-installing-gitops-operator.adoc @@ -0,0 +1,81 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-environment-setup.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-installing-gitops-operator_{context}"] += Install the GitOps Operator + +[role="_abstract"] +The GitOps Operator enables ArgoCD-based deployment of applications and configurations on the management cluster. +You must install it before deploying the DPF Operator. + +.Prerequisites + +* You have access to the cluster as a user with the `cluster-admin` role. +* You have installed the OpenShift CLI (`oc`). + +.Procedure + +. Create a file named `gitops-operator.yaml` with the following content: ++ +[source,yaml] +---- +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-gitops-operator + labels: + openshift.io/cluster-monitoring: "true" +--- +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: openshift-gitops-operator + namespace: openshift-gitops-operator +spec: + upgradeStrategy: Default +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: openshift-gitops-operator + namespace: openshift-gitops-operator +spec: + channel: gitops-1.21 + config: + env: + - name: ARGOCD_CLUSTER_CONFIG_NAMESPACES + value: "openshift-gitops,dpf-operator-system" + - name: CONTROLLER_CLUSTER_ROLE + value: "cluster-admin" + - name: SERVER_CLUSTER_ROLE + value: "cluster-admin" + installPlanApproval: Automatic + name: openshift-gitops-operator + source: redhat-operators + sourceNamespace: openshift-marketplace +---- + +. Apply the file: ++ +[source,terminal] +---- +$ oc apply -f gitops-operator.yaml +---- + +.Verification + +* Verify that the operator pods are running: ++ +[source,terminal] +---- +$ oc get pods -n openshift-gitops-operator +---- ++ +.Example output +[source,terminal] +---- +NAME READY STATUS RESTARTS AGE +openshift-gitops-operator-controller-manager-759b6c9ff9-752sk 2/2 Running 0 2d18h +---- diff --git a/modules/nw-dpf-installing-hcp-provisioner.adoc b/modules/nw-dpf-installing-hcp-provisioner.adoc new file mode 100644 index 000000000000..97aec51db7b4 --- /dev/null +++ b/modules/nw-dpf-installing-hcp-provisioner.adoc @@ -0,0 +1,63 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-hosted-cluster-provisioning.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-installing-hcp-provisioner_{context}"] += Install the dpf-hcp-provisioner-operator + +[role="_abstract"] +You can install the `dpf-hcp-provisioner-operator` by using a Helm chart. +The operator manages the lifecycle of hosted clusters for DPU environments. + +.Prerequisites + +* The Multicluster Engine (MCE) Operator is installed and hosted control planes is enabled. +* The MetalLB Operator is installed and a `MetalLB` instance is created. +* A storage class is available for etcd persistent volumes, such as {lvms} or an equivalent. +* The DPF Operator is installed and DPF CRDs are available. +* The Helm CLI (`helm`) is installed on your workstation. + +.Procedure + +. Clone the `dpf-hcp-provisioner-operator` repository: ++ +[source,terminal] +---- +$ git clone https://github.com/rh-ecosystem-edge/dpf-hcp-provisioner-operator.git +---- + +. Change to the cloned repository directory: ++ +[source,terminal] +---- +$ cd dpf-hcp-provisioner-operator +---- + +. Install the operator by using Helm: ++ +[source,terminal] +---- +$ helm install dpf-hcp-provisioner-operator \ + helm/dpf-hcp-provisioner-operator \ + --namespace dpf-hcp-provisioner-system \ + --create-namespace \ + --set image.repository=registry.redhat.io/dpu-kit-for-nvidia-operator/dpf-hcp-provisioner-rhel10-operator \ + --set image.tag=v4.22 +---- + +.Verification + +* Verify that the operator pod is running: ++ +[source,terminal] +---- +$ oc get pods -n dpf-hcp-provisioner-system +---- ++ +.Example output +[source,terminal] +---- +NAME READY STATUS RESTARTS AGE +dpf-hcp-provisioner-controller-manager-xxx-yyy 1/1 Running 0 1m +---- diff --git a/modules/nw-dpf-installing-maintenance-operator.adoc b/modules/nw-dpf-installing-maintenance-operator.adoc new file mode 100644 index 000000000000..4ebdce87faa1 --- /dev/null +++ b/modules/nw-dpf-installing-maintenance-operator.adoc @@ -0,0 +1,72 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-environment-setup.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-installing-maintenance-operator_{context}"] += Install the NVIDIA Maintenance Operator + +[role="_abstract"] +The NVIDIA Maintenance Operator assists in performing maintenance tasks and gracefully draining DPU worker nodes. +You install this operator by using Helm. + +.Prerequisites + +* You have access to the cluster as a user with the `cluster-admin` role. +* You have installed the OpenShift CLI (`oc`). +* You have installed the Helm CLI (`helm`). + +.Procedure + +. Create a Helm values file named `maintenance-operator-values.yaml` with the following content: ++ +[source,yaml] +---- +operatorConfig: + maxParallelOperations: 60% +operator: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "node-role.kubernetes.io/master" + operator: Exists + - matchExpressions: + - key: "node-role.kubernetes.io/control-plane" + operator: Exists + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule +---- + +. Install the operator by using Helm: ++ +[source,terminal] +---- +$ helm upgrade --install maintenance-operator oci://ghcr.io/mellanox/maintenance-operator-chart \ + --namespace dpf-operator-system \ + --create-namespace \ + --version 0.3.0 \ + --values maintenance-operator-values.yaml \ + --wait +---- + +.Verification + +* Verify that the operator pod is running: ++ +[source,terminal] +---- +$ oc get pods -n dpf-operator-system +---- ++ +.Example output +[source,terminal] +---- +maintenance-operator-585767f779-kps9c 1/1 Running 0 2d23h +---- diff --git a/modules/nw-dpf-installing-metallb-operator.adoc b/modules/nw-dpf-installing-metallb-operator.adoc new file mode 100644 index 000000000000..d33a1de4a9f2 --- /dev/null +++ b/modules/nw-dpf-installing-metallb-operator.adoc @@ -0,0 +1,77 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-environment-setup.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-installing-metallb-operator_{context}"] += Install the MetalLB Operator + +[role="_abstract"] +The MetalLB Operator provides load balancing services for the management cluster. +You must install it before deploying the DPF Operator. + +.Prerequisites + +* You have access to the cluster as a user with the `cluster-admin` role. +* You have installed the OpenShift CLI (`oc`). + +.Procedure + +. Create a file named `metallb-operator.yaml` with the following content: ++ +[source,yaml] +---- +apiVersion: v1 +kind: Namespace +metadata: + name: metallb-system +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: metallb-operator + namespace: openshift-operators +spec: + channel: "stable" + name: metallb-operator + source: redhat-operators + sourceNamespace: openshift-marketplace + installPlanApproval: Automatic + config: + # Tolerate the taint on the control plane nodes + tolerations: + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + # Force scheduling only on nodes with the control-plane label + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" +---- + +. Apply the file: ++ +[source,terminal] +---- +$ oc apply -f metallb-operator.yaml +---- + +.Verification + +* Verify that the operator pods are running: ++ +[source,terminal] +---- +$ oc get pods -n openshift-operators | grep metallb +---- ++ +.Example output +[source,terminal] +---- +metallb-operator-controller-manager-76d599495-8rcf7 1/1 Running 0 2d19h +metallb-operator-webhook-server-5bb6fcbbb8-chg8z 1/1 Running 0 2d19h +---- diff --git a/modules/nw-dpf-installing-required-operators.adoc b/modules/nw-dpf-installing-required-operators.adoc new file mode 100644 index 000000000000..ce65b3a6ac2e --- /dev/null +++ b/modules/nw-dpf-installing-required-operators.adoc @@ -0,0 +1,10 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-environment-setup.adoc + +:_mod-docs-content-type: CONCEPT +[id="installing-operators_{context}"] += Required operators + +[role="_abstract"] +Before you install the DPF Operator, the management cluster must have the cert-manager Operator, MetalLB Operator, GitOps Operator, and Maintenance Operator installed. diff --git a/modules/nw-dpf-management-cluster-setup.adoc b/modules/nw-dpf-management-cluster-setup.adoc new file mode 100644 index 000000000000..cf19732efbfb --- /dev/null +++ b/modules/nw-dpf-management-cluster-setup.adoc @@ -0,0 +1,82 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-environment-setup.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-management-cluster-setup_{context}"] += Set up the management cluster + +[role="_abstract"] +The management cluster is a standard {product-title} {product-version} cluster installed by using the Assisted Installer. +This cluster hosts the DPF operators and the hosted control planes for managing the hosted cluster on DPUs. + +.Prerequisites + +* You have access to the link:https://console.redhat.com/openshift/create[{hybrid-console}]. +* You have the OpenShift CLI (`oc`) installed. + +.Procedure + +. Navigate to the link:https://console.redhat.com/openshift/create[{hybrid-console} cluster creation page] and create a cluster with control-plane nodes only. +Select *Data center* -> *Assisted Installer*. + +. Optional: Configure jumbo MTU for each control plane node. ++ +.. Under *Hosts' network configuration* in the Assisted Installer wizard, select *Static IP, bridges, and bonds*. +.. Set the *Static network configurations* section per node according to the following template, using the relevant MAC address and interface name for each node: ++ +[source,yaml] +---- +interfaces: + - ipv4: + dhcp: true + enabled: true + mac-address: + mtu: 9000 # Set to 9000 for jumbo frames or 1500 for standard MTU + name: + state: up + type: ethernet +---- ++ +[NOTE] +==== +* You can alternatively configure MTU allocation on the DHCP server that allocates IPs to the control plane nodes. +* If virtual machines are used for control-plane nodes, the MTU must be set on the bridge of the hypervisor used by the VMs. +* When using MTU 9000, ensure the switch ports that connect the cluster's control-plane nodes are set to handle jumbo frames. +==== + +. Select the following operators to install with the cluster: ++ +* *Storage* -> *Logical Volume Manager Storage* +* *Platform Operations & Lifecycle* -> *MultiCluster Engine* +* *Scheduling* -> *Node Feature Discovery* + +. Click *Add hosts* to add hosts to the cluster. +Only control plane nodes are required at this stage. + +. After the installation completes, download the `KUBECONFIG` file and save it. + +.Verification + +. Set the `KUBECONFIG` environment variable: ++ +[source,terminal] +---- +$ export KUBECONFIG=mgmt-kubeconfig +---- + +. Verify that all nodes are in a `Ready` state: ++ +[source,terminal] +---- +$ oc get nodes +---- ++ +.Example output +[source,terminal] +---- +NAME STATUS ROLES AGE VERSION +master-0 Ready control-plane,worker 10m v1.35.0 +master-1 Ready control-plane,worker 10m v1.35.0 +master-2 Ready control-plane,worker 10m v1.35.0 +---- diff --git a/modules/nw-dpf-network-requirements.adoc b/modules/nw-dpf-network-requirements.adoc new file mode 100644 index 000000000000..d4888a412358 --- /dev/null +++ b/modules/nw-dpf-network-requirements.adoc @@ -0,0 +1,34 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/about-dpf-operator.adoc + +:_mod-docs-content-type: REFERENCE +[id="nw-dpf-network-requirements_{context}"] += DPF network infrastructure requirements + +[role="_abstract"] +The following network infrastructure is required for a DPF deployment. + +== Switches + +Management switch:: Provides 1GbE connectivity for the control plane and worker node management interfaces. + +High-speed switch:: An NVIDIA SN3700 or similar switch providing 2x 200GbE connectivity per DPU. + +== Connectivity + +* All nodes must have full internet access, both from the host out-of-band and DPU high-speed interfaces. +* The management network and the high-speed DPU network, which is the VTEP CIDR, must be routable to each other. +* A dedicated IP range, which is the VTEP CIDR, must be allocated from the high-speed DPU network for DPU service IPs used by HBN and OVN tunnels. +* A Virtual IP (VIP) from the management subnet must be reserved for the hosted DPU cluster control-plane services. The VIP must have a DNS A record. + +== MTU configuration + +The deployment supports both standard MTU (1500) and jumbo frames (MTU 9000). Jumbo frames can reduce per-packet overhead for DPU-accelerated traffic, but require all network components, including switches, interfaces, and bridges, to support MTU 9000. + +[IMPORTANT] +==== +The MTU value is set during deployment and cannot be changed later. Ensure consistency across all environment components. When using VMs for control plane nodes, ensure the hypervisor bridge MTU matches the chosen value. +==== + +For detailed network topology, interface names, and bridge configurations, see the link:https://networking-docs.nvidia.com/sol/rdg-for-dpf-host-trusted-with-ovn-kubernetes-and-hbn-services[RDG for DPF with OVN-Kubernetes and HBN Services], specifically the "Solution Design" and "Node and Switch Definitions" sections. diff --git a/modules/nw-dpf-running-traffic-validation.adoc b/modules/nw-dpf-running-traffic-validation.adoc new file mode 100644 index 000000000000..a840d9a301c9 --- /dev/null +++ b/modules/nw-dpf-running-traffic-validation.adoc @@ -0,0 +1,60 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-validation-telemetry.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-running-traffic-validation_{context}"] += Run traffic validation tests + +[role="_abstract"] +You can run connectivity tests between the traffic test pods and services to verify that the DPU services and service chains are configured correctly. +A successful test confirms that end-to-end traffic flows through the DPU data plane as expected. + +.Prerequisites + +* The traffic test pods and services are deployed in the `workload` namespace and all pods are in a `Running` state. +* You have access to the management cluster as a user with the `cluster-admin` role. + +.Procedure + +. Run a ping connectivity test between pods on different worker nodes. ++ +In the following example, replace `` with the name of a `traffic-test-worker` pod and replace `` with the IP address of a `traffic-test-worker` pod on a different worker node: ++ +[source,terminal,subs="attributes+"] +---- +$ oc -n workload exec -it -- ping -c 4 +---- ++ +.Example output +[source,terminal] +---- +PING 10.131.0.9 (10.131.0.9) 56(84) bytes of data. +64 bytes from 10.131.0.9: icmp_seq=1 ttl=62 time=1.61 ms +64 bytes from 10.131.0.9: icmp_seq=2 ttl=62 time=0.876 ms +64 bytes from 10.131.0.9: icmp_seq=3 ttl=62 time=0.510 ms +64 bytes from 10.131.0.9: icmp_seq=4 ttl=62 time=0.421 ms + +--- 10.131.0.9 ping statistics --- +4 packets transmitted, 4 received, 0% packet loss, time 3028ms +rtt min/avg/max/mdev = 0.421/0.853/1.606/0.466 ms +---- ++ +Verify that all 4 packets are received with 0% packet loss. + +. Run a service connectivity test from a worker pod to a service on a control plane node. ++ +In the following example, replace `` with the name of a `traffic-test-worker` pod and replace `` with the cluster IP address of the `traffic-test-master` service: ++ +[source,terminal] +---- +$ oc -n workload exec -it -- nc -vz 5000 +---- ++ +.Example output +[source,terminal] +---- +Connection to 172.30.102.123 5000 port [tcp/*] succeeded! +---- ++ +A `succeeded` message confirms that the service is reachable through the DPU-accelerated network. diff --git a/modules/nw-dpf-software-requirements.adoc b/modules/nw-dpf-software-requirements.adoc new file mode 100644 index 000000000000..c6e5858c3fe8 --- /dev/null +++ b/modules/nw-dpf-software-requirements.adoc @@ -0,0 +1,43 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/about-dpf-operator.adoc + +:_mod-docs-content-type: REFERENCE +[id="nw-dpf-software-requirements_{context}"] += DPF software requirements + +[role="_abstract"] +The following software versions are required for deploying DPF v26.4. + +.Software version requirements +[cols="1,1",options="header"] +|=== +|Component |Required version + +|{product-title} +|{product-version} + +|OpenShift CLI (`oc`) +|{product-version} + +|Hosted control planes {product-title} cluster +|{product-version} + +|NVIDIA DPF Operator +|v26.4.0 + +|RHCOS BFB +|{product-version} +|=== + +== Access requirements + +* `cluster-admin` privileges are required for the management cluster. + +== Additional guidance + +For foundational hardware and software details regarding the BlueField-3 DPU, consult the official NVIDIA user guides: + +* link:https://github.com/NVIDIA/doca-platform/blob/public-release-v26.4/docs/public/getting-started/dpf-host-trusted.md[Get Started with DPF Host Trusted] +* link:https://github.com/NVIDIA/doca-platform/blob/public-release-v26.4/docs/public/user-guides/host-trusted/use-cases/hbn-ovnk/README.md[DPF OVN-Kubernetes with Host-Based Networking User Guide] +* link:https://docs.nvidia.com/networking/display/public/sol/rdg-for-dpf-host-trusted-with-ovn-kubernetes[RDG for DPF with OVN-Kubernetes and HBN Services] diff --git a/modules/nw-dpf-ts-dpu-object-state.adoc b/modules/nw-dpf-ts-dpu-object-state.adoc new file mode 100644 index 000000000000..4d0f54151900 --- /dev/null +++ b/modules/nw-dpf-ts-dpu-object-state.adoc @@ -0,0 +1,25 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-troubleshooting.adoc + +:_mod-docs-content-type: REFERENCE +[id="nw-dpf-ts-dpu-object-state_{context}"] += DPU objects remain in the `DPU Cluster Config` state + +[role="_abstract"] +If DPU objects remain in a `DPU Cluster Config` state and do not progress, the hosted cluster might have pending certificate signing requests (CSRs) that must be approved. + +Check for pending CSRs in the hosted cluster:: +Switch to the hosted cluster context and check for any pending CSRs: ++ +[source,terminal] +---- +$ export KUBECONFIG= +---- ++ +[source,terminal] +---- +$ oc get csr -A +---- ++ +Review the output and approve any CSRs that show a `Pending` status. diff --git a/modules/nw-dpf-ts-dpu-provisioning.adoc b/modules/nw-dpf-ts-dpu-provisioning.adoc new file mode 100644 index 000000000000..edf99a3732de --- /dev/null +++ b/modules/nw-dpf-ts-dpu-provisioning.adoc @@ -0,0 +1,79 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-troubleshooting.adoc + +:_mod-docs-content-type: REFERENCE +[id="nw-dpf-ts-dpu-provisioning_{context}"] += DPU provisioning does not start + +[role="_abstract"] +If DPU provisioning does not start immediately after you add worker nodes to the management cluster, verify that certificate signing requests (CSRs), controller pods, Node Feature Discovery (NFD) labels, and DPF resource objects are in the correct state. + +Verify that all worker CSRs are approved:: +Run the following command to list the CSR status on the management cluster: ++ +[source,terminal] +---- +$ oc get csr +---- ++ +Ensure that all CSRs for the worker nodes show an `Approved` status. + +Verify that all DPF controller pods are running:: +Run the following command to check the status of the DPF Operator pods: ++ +[source,terminal] +---- +$ oc get pod -n dpf-operator-system +---- ++ +Ensure that all pods are in a `Running` state. + +Verify that worker nodes are labeled for DPU provisioning by NFD:: +Run the following command to confirm that the `dpu-enabled` label is present on the worker nodes: ++ +[source,terminal] +---- +$ oc get nodes -l feature.node.kubernetes.io/dpu-enabled="" +---- ++ +The output lists the worker nodes that NFD has labeled for DPU provisioning. +For example: ++ +[source,terminal] +---- +NAME STATUS ROLES AGE VERSION +host-worker1.example.com NotReady worker 62s v1.33.5 +host-worker2.example.com NotReady worker 66s v1.33.5 +---- + +Check BFB object status:: +Run the following command to verify that the BlueField Bootstream File (BFB) image is downloaded and ready: ++ +[source,terminal] +---- +$ oc describe bfb -n dpf-operator-system bf-bundle +---- + +Check `DPUDeployment` object status:: +Inspect the `DPUDeployment` object for information about the following resources: ++ +-- +* BFB object state +* `DPUServiceTemplate` objects state +* `DPUServiceConfiguration` objects state +-- ++ +Run the following command to view the full `DPUDeployment` status: ++ +[source,terminal] +---- +$ oc get dpudeployments -n dpf-operator-system dpudeployment -o yaml +---- ++ +Alternatively, run the following `dpfctl` command for a summarized view: ++ +[source,terminal] +---- +$ oc -n dpf-operator-system exec deploy/dpf-operator-controller-manager -- /dpfctl describe dpudeployments +---- diff --git a/modules/nw-dpf-ts-node-readiness.adoc b/modules/nw-dpf-ts-node-readiness.adoc new file mode 100644 index 000000000000..da2058bfef1f --- /dev/null +++ b/modules/nw-dpf-ts-node-readiness.adoc @@ -0,0 +1,38 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-troubleshooting.adoc + +:_mod-docs-content-type: REFERENCE +[id="nw-dpf-ts-node-readiness_{context}"] += Management cluster nodes do not become ready + +[role="_abstract"] +If management cluster nodes do not reach a `Ready` state after DPU provisioning completes, the OVN-Kubernetes CNI pods might not be running correctly on the management cluster or the hosted cluster. + +Check OVN-Kubernetes pods on the management cluster:: +Switch to the management cluster context and verify that all OVN-Kubernetes pods are running on the x86_64 worker nodes and control plane nodes: ++ +[source,terminal] +---- +$ export KUBECONFIG= +---- ++ +[source,terminal] +---- +$ oc get pods -n openshift-ovn-kubernetes -o wide +---- + +Check OVN-Kubernetes pods on the hosted cluster:: +Switch to the hosted cluster context and verify that all OVN-Kubernetes pods are running on the DPU workers: ++ +[source,terminal] +---- +$ export KUBECONFIG= +---- ++ +[source,terminal] +---- +$ oc get pods -n openshift-ovn-kubernetes -o wide +---- ++ +Ensure that all pods in the `openshift-ovn-kubernetes` namespace are in a `Running` state on both clusters. diff --git a/modules/nw-dpf-unsupported-ovnk-features.adoc b/modules/nw-dpf-unsupported-ovnk-features.adoc new file mode 100644 index 000000000000..be6f5eccdc3c --- /dev/null +++ b/modules/nw-dpf-unsupported-ovnk-features.adoc @@ -0,0 +1,25 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-troubleshooting.adoc + +:_mod-docs-content-type: REFERENCE +[id="nw-dpf-unsupported-ovnk-features_{context}"] += Unsupported OVN-Kubernetes features with DPF + +[role="_abstract"] +Certain OVN-Kubernetes features are not supported with the DPF networking configuration. + +[IMPORTANT] +==== +The following OVN-Kubernetes features are not supported as part of the CNI used in this technical preview: + +* User-Defined Networks (UDN) +* Egress IP +* `AdminNetworkPolicy` (ANP) +* Hybrid overlay +* Egress firewall +* Quality of Service (QoS) +* Border Gateway Protocol (BGP) +* Multicast +* Local gateway mode +==== diff --git a/modules/nw-dpf-validating-traffic-overview.adoc b/modules/nw-dpf-validating-traffic-overview.adoc new file mode 100644 index 000000000000..65a8a5785ff8 --- /dev/null +++ b/modules/nw-dpf-validating-traffic-overview.adoc @@ -0,0 +1,10 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-validation-telemetry.adoc + +:_mod-docs-content-type: CONCEPT +[id="traffic-validation_{context}"] += Traffic validation + +[role="_abstract"] +You can validate end-to-end traffic flow over the DPU-accelerated network by deploying test pods and running connectivity tests. diff --git a/modules/nw-dpf-verifying-dpu-provisioning.adoc b/modules/nw-dpf-verifying-dpu-provisioning.adoc new file mode 100644 index 000000000000..1e70edfbe504 --- /dev/null +++ b/modules/nw-dpf-verifying-dpu-provisioning.adoc @@ -0,0 +1,68 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-worker-nodes-provisioning.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-verifying-dpu-provisioning_{context}"] += Verify DPU provisioning + +[role="_abstract"] +After worker nodes are labeled for the DPU `MachineConfigPool`, the `DPUSet` controller automatically detects nodes with the `dpu-enabled` label, creates a `DPU` object for each node, and starts the provisioning process. +You can monitor the provisioning stages to verify progress. + +.Prerequisites + +* You have access to the management cluster as a user with the `cluster-admin` role. +* The OpenShift CLI (`oc`) is installed. +* Worker nodes are labeled with `node-role.kubernetes.io/worker-dpu=""`. + +.Procedure + +. Watch for `DPU` object creation: ++ +[source,terminal] +---- +$ oc get dpu -n dpf-operator-system -w +---- ++ +The `DPU` objects progress through the following provisioning stages: ++ +-- +Initializing:: The `DPU` object is created. +OS Installing:: The BFB installation is in progress. +Rebooting:: The host and DPU are resetting. +DPU Cluster Config:: The DPU Kubernetes node join procedure is in progress. Manual CSR approval is required during this stage. +Host Network Configuration:: Networking configuration adjustments are applied on the host. +Ready:: The DPU is successfully provisioned and ready to use. +Error:: Provisioning failed. Check events and conditions for details. +-- ++ +[IMPORTANT] +==== +When the provisioning stage reaches `DPU Cluster Config`, proceed to "Configure authorization for the hosted cluster" and "Approve DPU node CSRs" to complete the DPU node join process. +==== + +. Monitor detailed provisioning progress: ++ +[source,terminal] +---- +$ oc -n dpf-operator-system exec deploy/dpf-operator-controller-manager -- /dpfctl describe dpudeployments +---- + +. Optional: View detailed status for a specific `DPU` object: ++ +In the following command, replace `` with the name of the `DPU` resource: ++ +[source,terminal] +---- +$ oc describe dpu -n dpf-operator-system +---- + +. Optional: Follow the provisioning logs for a specific DPU: ++ +In the following command, replace `` with the name of the `DPU` resource: ++ +[source,terminal] +---- +$ oc logs -n dpf-operator-system -l dpf.nvidia.com/dpu= -f +---- diff --git a/modules/nw-dpf-verifying-dpu-services.adoc b/modules/nw-dpf-verifying-dpu-services.adoc new file mode 100644 index 000000000000..90989a89001c --- /dev/null +++ b/modules/nw-dpf-verifying-dpu-services.adoc @@ -0,0 +1,109 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-operator-installation.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-verifying-dpu-services_{context}"] += Verify DPU service resources + +[role="_abstract"] +After you create all DPU service resources, you can verify that the services, IPAM pools, interfaces, and service chains have been reconciled successfully. + +.Prerequisites + +* The DPF Operator is installed and configured. +* All DPU service resources are created: `DPUServiceTemplate`, `DPUServiceConfiguration`, `DPUServiceCredentialRequest`, `DPUServiceInterface`, `DPUServiceNAD`, and `DPUServiceIPAM`. +* The `DPUDeployment` resource is created. + +.Procedure + +. Verify that the DPU services are reconciled: ++ +[source,terminal] +---- +$ oc wait --for=condition=ApplicationsReconciled \ + --namespace dpf-operator-system dpuservices \ + -l svc.dpu.nvidia.com/owned-by-dpudeployment=dpf-operator-system_dpudeployment +---- ++ +.Example output +[source,terminal] +---- +dpuservice.svc.dpu.nvidia.com/hbn-v4ffl condition met +dpuservice.svc.dpu.nvidia.com/ovn-xmkjj condition met +---- + +. Verify that the DPU service IPAM pools are reconciled: ++ +[source,terminal] +---- +$ oc wait --for=condition=DPUIPAMObjectReconciled \ + --namespace dpf-operator-system dpuserviceipam --all +---- ++ +.Example output +[source,terminal] +---- +dpuserviceipam.svc.dpu.nvidia.com/loopback condition met +dpuserviceipam.svc.dpu.nvidia.com/pool1 condition met +---- + +. Verify that the DPU service interfaces are reconciled: ++ +[source,terminal] +---- +$ oc wait --for=condition=ServiceInterfaceSetReconciled \ + --namespace dpf-operator-system dpuserviceinterface --all +---- ++ +.Example output +[source,terminal] +---- +dpuserviceinterface.svc.dpu.nvidia.com/hbn-p0-if-qqlws condition met +dpuserviceinterface.svc.dpu.nvidia.com/hbn-p1-if-9x29m condition met +dpuserviceinterface.svc.dpu.nvidia.com/hbn-pf2dpu2-if-vnjtl condition met +dpuserviceinterface.svc.dpu.nvidia.com/ovn condition met +dpuserviceinterface.svc.dpu.nvidia.com/p0 condition met +dpuserviceinterface.svc.dpu.nvidia.com/p1 condition met +---- + +. Verify that the DPU service chains are reconciled: ++ +[source,terminal] +---- +$ oc wait --for=condition=ServiceChainSetReconciled \ + --namespace dpf-operator-system dpuservicechain --all +---- ++ +.Example output +[source,terminal] +---- +dpuservicechain.svc.dpu.nvidia.com/dpudeployment-sqx9h condition met +---- + +. Verify the overall status of all DPU services: ++ +[source,terminal] +---- +$ oc get dpuservice -n dpf-operator-system +---- ++ +.Example output +[source,terminal] +---- +NAME READY PHASE AGE +flannel True Success 26h +hbn-gffmv True Success 25m +nvidia-k8s-ipam False Pending 26h +ovn-f49zx True Success 17m +ovs-cni True Success 26h +servicechainset-controller True Success 26h +servicechainset-rbac-and-crds True Success 138m +sfc-controller True Success 26h +sriov-device-plugin True Success 26h +---- ++ +[NOTE] +==== +The `nvidia-k8s-ipam` service is expected to show a `Pending` phase at this stage. It transitions to `Success` after DPU provisioning is complete. +==== diff --git a/modules/nw-dpf-verifying-hosted-cluster.adoc b/modules/nw-dpf-verifying-hosted-cluster.adoc new file mode 100644 index 000000000000..bec206a78cb9 --- /dev/null +++ b/modules/nw-dpf-verifying-hosted-cluster.adoc @@ -0,0 +1,61 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-hosted-cluster-provisioning.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-verifying-hosted-cluster_{context}"] += Verify hosted cluster creation + +[role="_abstract"] +After creating the `DPFHCPProvisioner` resource, you can monitor its status to verify that the hosted cluster is provisioned and becomes ready. +The provisioning process can take up to 30 minutes. + +.Prerequisites + +* The `DPFHCPProvisioner` resource is created in the clusters namespace. + +.Procedure + +. Monitor the `DPFHCPProvisioner` status: ++ +[source,terminal] +---- +$ oc get dpfhcpprovisioner -n ${CLUSTERS_NAMESPACE} +---- ++ +.Example output +[source,terminal] +---- +NAME AGE STATUS +dpf-hosted 2m Provisioning +---- + +. Wait for the `DPFHCPProvisioner` to reach the `Ready` phase: ++ +[source,terminal] +---- +$ oc wait dpfhcpprovisioner ${HOSTED_CLUSTER_NAME} -n ${CLUSTERS_NAMESPACE} \ + --for=jsonpath='{.status.phase}'=Ready --timeout=30m +---- ++ +.Example output +[source,terminal] +---- +dpfhcpprovisioner.provisioning.dpu.hcp.io/dpf-hosted condition met +---- + +.Verification + +* Confirm that the `DPFHCPProvisioner` shows `Ready` status with `READY` set to `True`: ++ +[source,terminal] +---- +$ oc get dpfhcpprovisioner ${HOSTED_CLUSTER_NAME} -n ${CLUSTERS_NAMESPACE} +---- ++ +.Example output +[source,terminal] +---- +NAME PHASE READY HOSTEDCLUSTER AGE +dpf-hosted Ready True dpf-hosted 10m +---- diff --git a/modules/nw-dpf-verifying-system-readiness.adoc b/modules/nw-dpf-verifying-system-readiness.adoc new file mode 100644 index 000000000000..947c3e2f03a9 --- /dev/null +++ b/modules/nw-dpf-verifying-system-readiness.adoc @@ -0,0 +1,95 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-worker-nodes-provisioning.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-verifying-system-readiness_{context}"] += Verify full system readiness + +[role="_abstract"] +After the DPU provisioning process completes, you can verify that all worker nodes, SR-IOV virtual functions, and DPU services are operational on the management cluster. + +.Prerequisites + +* You have access to the management cluster as a user with the `cluster-admin` role. +* The OpenShift CLI (`oc`) is installed. +* DPU provisioning has completed. + +.Procedure + +. Switch back to the management cluster context: ++ +[source,terminal] +---- +$ export KUBECONFIG=mgmt-kubeconfig +---- + +. Verify that all worker nodes are in a `Ready` state: ++ +[source,terminal] +---- +$ oc get node +---- ++ +.Example output +[source,terminal] +---- +NAME STATUS ROLES AGE VERSION +host-worker1 Ready worker 57m v1.35 +host-worker2 Ready worker 57m v1.35 +master-0 Ready control-plane,master,worker 4d23h v1.35 +master-1 Ready control-plane,master,worker 4d22h v1.35 +master-2 Ready control-plane,master,worker 4d23h v1.35 +---- + +. Verify that SR-IOV virtual functions are registered as Kubernetes node resources on the worker nodes: ++ +[source,terminal] +---- +$ oc get nodes -l 'node-role.kubernetes.io/worker,!node-role.kubernetes.io/control-plane' -o json | \ + jq '.items[] | {name: .metadata.name, capacity: .status.capacity."openshift.io/bf3-p0-vfs", allocatable: .status.allocatable."openshift.io/bf3-p0-vfs"}' +---- ++ +.Example output +[source,terminal] +---- +{ + "name": "host-worker1", + "capacity": "44", + "allocatable": "44" +} +{ + "name": "host-worker2", + "capacity": "44", + "allocatable": "44" +} +---- + +. Verify that all DPU services are in a `Success` phase: ++ +[source,terminal] +---- +$ oc get dpuservices -n dpf-operator-system +---- ++ +.Example output +[source,terminal] +---- +NAME READY PHASE AGE +flannel True Success 4h31m +hbn-cmlph True Success 3h47m +nvidia-k8s-ipam True Success 4h31m +ovn-xcv67 True Success 3h47m +ovs-cni True Success 4h31m +servicechainset-controller True Success 4h31m +servicechainset-rbac-and-crds True Success 4h30m +sfc-controller True Success 4h31m +sriov-device-plugin True Success 4h31m +---- + +. Optional: View detailed DPU service status: ++ +[source,terminal] +---- +$ oc -n dpf-operator-system exec deploy/dpf-operator-controller-manager -- /dpfctl describe all --show-resources=dpuservice --grouping=false +---- diff --git a/modules/nw-dpf-viewing-dts-metrics.adoc b/modules/nw-dpf-viewing-dts-metrics.adoc new file mode 100644 index 000000000000..be38d72c4198 --- /dev/null +++ b/modules/nw-dpf-viewing-dts-metrics.adoc @@ -0,0 +1,104 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-validation-telemetry.adoc + +:_mod-docs-content-type: REFERENCE +[id="nw-dpf-viewing-dts-metrics_{context}"] += DTS metrics and dashboards + +[role="_abstract"] +After you configure the DTS `ServiceMonitor`, you can view DPU telemetry metrics by using the {product-title} web console, `PromQL` queries, or Grafana dashboards. + +== Verify the DTS DPUService status + +Before viewing metrics, confirm that the DTS `DPUService` is ready on the management cluster. +The object name carries a generated suffix, so select it by its stable label: + +[source,terminal] +---- +$ oc -n dpf-operator-system get dpuservice \ + -l svc.dpu.nvidia.com/dpudeployment-service=doca-telemetry-service +---- + +.Example output +[source,terminal] +---- +NAME READY PHASE AGE +doca-telemetry-service-89p28 True Success ... +---- + +A status of `READY: True` and `PHASE: Success` confirms that DTS is deployed and running. + +== View metrics in the {product-title} web console + +The {product-title} web console reads from the cluster's built-in Prometheus instance. +No additional tools are required. + +To run an ad hoc query, go to *Observe* -> *Metrics* in the web console, enter a DTS `PromQL` query, and click *Run queries*. + +.DTS `PromQL` queries +[cols="2,3",options="header"] +|=== +|Query |Description + +|`current_link_speed{job=~"doca-telemetry-service.*"}` +|Returns the current PCIe link speed for each DPU. Each series includes a `source` label that identifies the DPU node name. + +|`rate(p0_eth_rx_bytes{job=~"doca-telemetry-service.*"}[5m]) * 8` +|Calculates the uplink receive throughput in bits per second over a 5-minute window. + +|`rate(ch_poll{job=~"doca-telemetry-service.*"}[5m])` +|Calculates the NIC channel polling activity rate over a 5-minute window. +|=== + +To view the built-in DTS dashboard, go to *Observe* -> *Dashboards* in the web console and select *DOCA DPU Telemetry (DTS)* from the dashboard dropdown. +The dashboard displays PCIe link speed and width, uplink throughput, packets per second, errors and drops per second, and NIC channel activity, with each DPU as its own line. + +== View metrics in Grafana + +DTS installs a Grafana instance through the Grafana Operator, pre-configured with a data source that points at the cluster's Prometheus instance. + +To open Grafana, retrieve the route URL: + +[source,terminal] +---- +$ echo "https://$(oc -n dpf-operator-system get route dpf-grafana-route -o jsonpath='{.spec.host}')" +---- + +Open the URL in a browser. Anonymous access provides read-only viewer permissions. +To edit dashboards, click *Sign in* and use `admin` / `admin` as the default credentials set in the Grafana custom resource. + +[IMPORTANT] +==== +Change the default Grafana credentials for non-lab clusters. +==== + +In Grafana, go to *Dashboards* and open *DOCA DPU Telemetry (DTS)*. +Use the *DPU (source)* dropdown to focus on a specific DPU or select *All*. +Adjust the time range by using the time-range control on the dashboard toolbar. The dashboard refreshes every 30 seconds. + +== DPF framework dashboards + +The DPF Operator installs framework dashboards that track DPU lifecycle and control-plane health separately from the DTS hardware telemetry dashboard. +These dashboards are loaded into Grafana automatically through `GrafanaDashboard` resources created from `ConfigMaps`. + +.DPF framework dashboards +[cols="2,3",options="header"] +|=== +|Dashboard |Description + +|DOCA Platform DPU Fleet Health +|Fleet-wide DPU health, provisioning state, and version distribution. + +|DOCA Platform DPU Health Detail +|Per-DPU status, conditions, and history timelines. + +|DOCA Platform Framework State +|Inventory and readiness of every DPF resource type. + +|DOCA Platform Framework Performance +|Time for DPF resources to reach their conditions, including reconcile and provisioning timings. + +|Controller Runtime +|DPF controller internals: CPU and memory usage, reconcile rates, queues, and errors. +|=== diff --git a/modules/nw-dpf-worker-machineconfig.adoc b/modules/nw-dpf-worker-machineconfig.adoc new file mode 100644 index 000000000000..ce8f70638839 --- /dev/null +++ b/modules/nw-dpf-worker-machineconfig.adoc @@ -0,0 +1,445 @@ +// Module included in the following assemblies: +// +// * networking/networking_operators/dpf_operator/dpf-environment-setup.adoc + +:_mod-docs-content-type: PROCEDURE +[id="nw-dpf-worker-machineconfig_{context}"] += Configure worker nodes for DPUs + +[role="_abstract"] +You must create a `MachineConfig` resource for worker nodes with DPUs before adding those nodes to the management cluster. +The Machine Config Operator stores this configuration and automatically applies it when worker nodes with the `worker-dpu` label are added to the cluster. + +The `MachineConfig` resource performs several configuration tasks required by DPF: + +Bridge configuration:: Creates a `br-dpu` bridge interface that enables communication between the DPU and the hosted cluster control plane running on the management cluster. For more information, see the link:https://github.com/NVIDIA/doca-platform/blob/public-release-v26.4/docs/public/user-guides/host-trusted/prerequisites/host-network-configuration-prerequisite.md[DPF Operator prerequisites]. +OVS service management:: Disables the default OVS services on x86 worker nodes. This is required for OVN-Kubernetes DPU Host mode operation, where networking functions are offloaded to the DPU rather than running on the host CPU. +IP rules configuration:: Sets routing rules required for pod-to-host control-plane traffic. + +.Prerequisites + +* You have access to the cluster as a user with the `cluster-admin` role. +* You have installed the OpenShift CLI (`oc`). + +.Procedure + +. Create a file named `worker-dpu-mcp.yaml` with the following `MachineConfigPool` definition: ++ +[source,yaml] +---- +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfigPool +metadata: + name: worker-dpu +spec: + machineConfigSelector: + matchExpressions: + - {key: machineconfiguration.openshift.io/role, operator: In, values: [worker, worker-dpu]} + maxUnavailable: 1 + nodeSelector: + matchLabels: + node-role.kubernetes.io/worker-dpu: "" + paused: false +---- + +. Create the script files that are embedded in the `MachineConfig`. ++ +.. Create the bridge configuration script file named `apply-nmstate-bridge.sh`: ++ +[source,bash,subs="verbatim"] +---- +# A script for detecting the default network interface and creating the br-dpu bridge +cat > apply-nmstate-bridge.sh << 'EOF' +#!/bin/bash +set -e +BRIDGE_NAME="br-dpu" +IP_HINT_FILE="/run/nodeip-configuration/primary-ip" +TARGET_MTU="$1" +NODE_IP="" +read_node_ip() { + if [[ ! -f "$IP_HINT_FILE" ]]; then + echo "ERROR: IP hint file not found: $IP_HINT_FILE" >&2 + return 1 + fi + NODE_IP=$(tr -d '[:space:]' < "$IP_HINT_FILE") + if [[ -z "$NODE_IP" ]]; then + echo "ERROR: IP hint file is empty: $IP_HINT_FILE" >&2 + return 1 + fi + echo "INFO: Node IP from hint file: $NODE_IP" +} +wait_for_bridge_ip() { + local bridge="$1" + local timeout=120 + local interval=2 + local elapsed=0 + echo "INFO: Waiting up to ${timeout}s for $bridge to acquire $NODE_IP..." + while (( elapsed < timeout )); do + if ip -o addr show dev "$bridge" | grep -qw "$NODE_IP"; then + echo "INFO: $bridge has $NODE_IP." + ip addr show dev "$bridge" + return 0 + fi + sleep "$interval" + elapsed=$(( elapsed + interval )) + done + echo "ERROR: $bridge did not acquire $NODE_IP within ${timeout}s." >&2 + return 1 +} +set_bridge_rp_filter_loose() { + local bridge="$1" + echo "INFO: Setting rp_filter=2 (loose mode) on $bridge" + sysctl -w "net.ipv4.conf.${bridge}.rp_filter=2" +} +validate_bridge_exists() { + if ip link show "$BRIDGE_NAME" &> /dev/null; then + echo "INFO: Bridge '$BRIDGE_NAME' already exists, waiting for IP..." + wait_for_bridge_ip "$BRIDGE_NAME" + set_bridge_rp_filter_loose "$BRIDGE_NAME" + exit 0 + fi +} +get_nodeip_hint_interface() { + local iface + iface=$(ip -j addr | jq -r --arg ip "$NODE_IP" --arg br "$BRIDGE_NAME" \ + 'first(.[] | select(any(.addr_info[]; .local==$ip) and .ifname!=$br)) | .ifname') + if [[ -z "${iface}" || "${iface}" == "null" ]]; then + echo "ERROR: No interface found with IP $NODE_IP" >&2 + return 1 + fi + echo "${iface}" +} +apply_linux_bridge() { + local iface="$1" + local bridge="$BRIDGE_NAME" + local mtu_arg="$2" + if [ -z "$iface" ]; then + echo "ERROR: No physical interface matches the Node IP in $IP_HINT_FILE." >&2 + exit 1 + fi + echo "INFO: Target interface: $iface" + echo "INFO: MTU policy: ${mtu_arg:+Set to $mtu_arg (user override)}${mtu_arg:-Inherit from physical interface}" + local routes_json + routes_json=$(nmstatectl show --json | jq -c --arg phys "$iface" \ + '[.routes.config // [] | .[] | select(.["next-hop-interface"] == $phys)]') + echo "INFO: Generating NMState desired state..." + nmstatectl show "$iface" --json | jq \ + --arg br "$bridge" \ + --arg phys "$iface" \ + --arg mtu_val "$mtu_arg" \ + --argjson phys_routes "$routes_json" \ + ' + .interfaces[0] as $p | + (if $mtu_val != "" then {"mtu": ($mtu_val | tonumber)} else {} end) as $mtu_obj | + { + "interfaces": [ + ({ + "name": $br, + "type": "linux-bridge", + "state": "up", + "mac-address": $p."mac-address", + "ipv4": ($p.ipv4 | del(.forwarding)), + "ipv6": ($p.ipv6 | del(.forwarding)), + "bridge": { + "options": { "stp": { "enabled": false } }, + "port": [{ "name": $phys }] + } + } + $mtu_obj), + ({ + "name": $phys, + "type": $p.type, + "state": "up", + "ipv4": { "enabled": false }, + "ipv6": { "enabled": false } + } + $mtu_obj + + if $p["link-aggregation"] then + { "link-aggregation": $p["link-aggregation"] } + else {} end) + ] + } + + if ($phys_routes | length) > 0 then + { "routes": { "config": [$phys_routes[] | .["next-hop-interface"] = $br] } } + else {} end + ' > /tmp/br-dpu-config.yml + echo "--- Generated NMState desired state ---" + cat /tmp/br-dpu-config.yml + echo "---------------------------------------" + echo "INFO: Applying configuration via nmstatectl..." + if nmstatectl apply /tmp/br-dpu-config.yml; then + echo "SUCCESS: Bridge $bridge created successfully." + wait_for_bridge_ip "$bridge" + ip addr show "$bridge" + set_bridge_rp_filter_loose "$bridge" + else + echo "ERROR: Failed to apply NMState configuration." >&2 + rm -f /tmp/br-dpu-config.yml + exit 1 + fi + rm -f /tmp/br-dpu-config.yml +} +# --- Main --- +read_node_ip +validate_bridge_exists +SELECTED_IFACE=$(get_nodeip_hint_interface) +apply_linux_bridge "$SELECTED_IFACE" "$TARGET_MTU" +EOF +---- ++ +.. Create the P0 routing script file named `configure-p0-routing.sh`: ++ +[source,bash,subs="verbatim"] +---- +cat > configure-p0-routing.sh << 'EOF' +#!/bin/bash +set -e +# Long-running service that ensures routing table 100 stays configured for +# OVN-K traffic via br-dpu. Continuously monitors and re-applies rules/routes +# in case they are removed (e.g. after NMState reconfiguration). +CHECK_INTERVAL=2 +RECONCILE_INTERVAL=60 +PRIMARY_IP_FILE="/run/nodeip-configuration/primary-ip" +echo "Waiting for primary IP file to determine IP version..." +while [ ! -f "$PRIMARY_IP_FILE" ] || [ ! -s "$PRIMARY_IP_FILE" ]; do + sleep $CHECK_INTERVAL +done +br_dpu_ip=$(tr -d '[:space:]' < "$PRIMARY_IP_FILE") +echo "Using br-dpu IP from $PRIMARY_IP_FILE: $br_dpu_ip" +if [[ "$br_dpu_ip" =~ : ]]; then + IP_VERSION="6" + IP_FLAG="-6" + PREFIX_LEN="128" + LINK_LOCAL_PATTERN="^fe80:" + echo "Detected IPv6 configuration" +else + IP_VERSION="4" + IP_FLAG="-4" + PREFIX_LEN="32" + LINK_LOCAL_PATTERN="^169[.]254" + echo "Detected IPv4 configuration" +fi +ensure_rule() { + if ip $IP_FLAG -j rule list | jq -e --arg src "$br_dpu_ip" '.[] | select(.src == $src and .table == "100")' > /dev/null 2>&1; then + return 0 + fi + echo "Adding rule: from $br_dpu_ip/$PREFIX_LEN lookup 100" + ip $IP_FLAG rule add from $br_dpu_ip/$PREFIX_LEN lookup 100 +} +ensure_route() { + local dst="$1"; shift + if ip $IP_FLAG -j route show table 100 | jq -e --arg dst "$dst" '.[] | select(.dst == $dst)' > /dev/null 2>&1; then + return 0 + fi + echo "Adding route: $dst $* table 100" + ip $IP_FLAG route add $dst "$@" table 100 +} +configure_routing() { + local ovnk_iface="$1" + local ovnk_ip="$2" + local br_dpu_network + br_dpu_network=$(ip $IP_FLAG -j route show dev br-dpu | jq -r '.[] | select(.protocol == "kernel") | .dst' | head -n1) + if [ -z "$br_dpu_network" ]; then + echo "Warning: Could not find br-dpu network, will retry" + return 1 + fi + local br_dpu_gateway + br_dpu_gateway=$(ip $IP_FLAG -j route | jq -r '.[] | select(.dst == "default" and .dev == "br-dpu") | .gateway' | head -n1) + if [ -z "$br_dpu_gateway" ]; then + echo "Warning: Could not find gateway for br-dpu, will retry" + return 1 + fi + local ovnk_subnet + ovnk_subnet=$(ip $IP_FLAG -j route | jq --arg dev "$ovnk_iface" --arg pattern "$LINK_LOCAL_PATTERN" -r ' + .[] | select( + .dev == $dev + and .dst != null + and .dst != "default" + and (.dst | test($pattern) | not) + and .gateway == null + ) | .dst' | head -n1) + if [ -z "$ovnk_subnet" ]; then + echo "Warning: Could not find subnet for $ovnk_iface, will retry" + return 1 + fi + local br_dpu_metric + br_dpu_metric=$(ip $IP_FLAG -j route show dev br-dpu | jq -r '.[] | select(.protocol == "kernel") | .metric // 425' | head -n1) + ensure_rule + ensure_route "$ovnk_subnet" via "$br_dpu_gateway" + ensure_route "$br_dpu_network" dev br-dpu proto kernel scope link src "$br_dpu_ip" metric "$br_dpu_metric" + return 0 +} +echo "Waiting for OVN-K interface (with link-local address) to get an IP address..." +configured=false +while true; do + ovnk_ifaces=$(ip -j addr show | jq --arg pattern "$LINK_LOCAL_PATTERN" -r '.[] | select(.addr_info[]? | .local | test($pattern)) | .ifname' | sort -u) + for ovnk_iface in $ovnk_ifaces; do + ovnk_ip=$(ip $IP_FLAG -j addr show "$ovnk_iface" | jq --arg pattern "$LINK_LOCAL_PATTERN" -r '.[] | .addr_info[]? | select(.local | test($pattern) | not) | .local' | head -n1) + if [ -n "$ovnk_ip" ]; then + if [ "$configured" = "false" ]; then + echo "Found OVN-K interface: $ovnk_iface with IPv${IP_VERSION}: $ovnk_ip" + fi + if configure_routing "$ovnk_iface" "$ovnk_ip"; then + if [ "$configured" = "false" ]; then + echo "Routing configuration completed, entering reconcile loop" + configured=true + fi + fi + break + fi + done + if [ "$configured" = "true" ]; then + sleep $RECONCILE_INTERVAL + else + sleep $CHECK_INTERVAL + fi +done +EOF +---- ++ +.. Create the NetworkManager configuration file named `unmanage-ovnk-interface.conf`: ++ +[source,bash,subs="verbatim"] +---- +cat > unmanage-ovnk-interface.conf << 'EOF' +[keyfile] +unmanaged-devices=interface-name:ovn-k8s-* +EOF +---- + +. Encode the scripts to base64: ++ +[source,terminal] +---- +$ BRIDGE_SCRIPT_BASE64=$(base64 -w 0 apply-nmstate-bridge.sh) +$ ROUTING_SCRIPT_BASE64=$(base64 -w 0 configure-p0-routing.sh) +$ NETWORKMANAGER_CONFIG_BASE64=$(base64 -w 0 unmanage-ovnk-interface.conf) +---- + +. Create a file named `dpu-worker-configuration.yaml` with the following `MachineConfig` definition that embeds the base64-encoded scripts: ++ +[source,bash,subs="verbatim"] +---- +# Create the MachineConfig with substituted base64 content +envsubst << EOF > dpu-worker-configuration.yaml +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + machineconfiguration.openshift.io/role: worker-dpu + name: dpu-worker-configuration +spec: + config: + ignition: + version: 3.2.0 + storage: + files: + - contents: + source: data:text/plain;charset=utf-8;base64,$BRIDGE_SCRIPT_BASE64 + mode: 0755 + overwrite: true + path: /usr/local/bin/apply-nmstate-bridge.sh + - contents: + source: data:text/plain;charset=utf-8;base64,$ROUTING_SCRIPT_BASE64 + mode: 0755 + overwrite: true + path: /usr/local/bin/configure-p0-routing.sh + - contents: + source: data:text/plain;charset=utf-8;base64,$NETWORKMANAGER_CONFIG_BASE64 + mode: 0644 + overwrite: true + path: "/etc/NetworkManager/conf.d/unmanage-ovnk-interface.conf" + systemd: + units: + - contents: | + [Unit] + Description=Apply NMState bridge configuration + After=network.target NetworkManager.service nodeip-configuration.service nmstate.service + Before=crio.service kubelet.service + [Service] + Type=oneshot + ExecStart=/usr/local/bin/apply-nmstate-bridge.sh \$NODES_MTU + RemainAfterExit=yes + TimeoutStartSec=150s + Restart=on-failure + RestartSec=10 + [Install] + WantedBy=multi-user.target + enabled: true + name: nmstate-bridge.service + - name: ovs-configuration.service + enabled: false + - name: openvswitch.service + enabled: false + mask: true + - name: wait-for-br-ex-up.service + enabled: false + - name: p0-routing.service + enabled: true + contents: | + [Unit] + Description=Configure and maintain p0 interface routing + After=kubelet.service network-online.target nmstate-bridge.service + Wants=network-online.target + [Service] + Type=simple + ExecStart=/usr/local/bin/configure-p0-routing.sh + StandardOutput=journal + StandardError=journal + Restart=always + RestartSec=10 + [Install] + WantedBy=multi-user.target +EOF +---- + +. Set the MTU environment variable and apply both configuration files. +Set `NODES_MTU` to `9000` for jumbo frames or `1500` for standard MTU: ++ +[source,terminal] +---- +$ export NODES_MTU=1500 +---- ++ +[source,terminal] +---- +$ oc apply -f worker-dpu-mcp.yaml +---- ++ +[source,terminal] +---- +$ envsubst < dpu-worker-configuration.yaml | oc apply -f - +---- + +.Verification + +. Verify that the `MachineConfig` was created: ++ +[source,terminal] +---- +$ oc get machineconfig dpu-worker-configuration +---- ++ +.Example output +[source,terminal] +---- +NAME GENERATEDBYCONTROLLER IGNITIONVERSION AGE +dpu-worker-configuration 3.2.0 2m +---- + +. Verify that the `MachineConfigPool` exists: ++ +[source,terminal] +---- +$ oc get machineconfigpool worker-dpu +---- ++ +.Example output +[source,terminal] +---- +NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT +worker-dpu True False False 0 0 +---- + +[NOTE] +==== +The Machine Config Operator automatically reboots worker nodes to apply these changes after nodes with the `worker-dpu` label are added to the cluster. +==== diff --git a/networking/networking_operators/dpf_operator/_attributes b/networking/networking_operators/dpf_operator/_attributes new file mode 120000 index 000000000000..20cc1dcb77bf --- /dev/null +++ b/networking/networking_operators/dpf_operator/_attributes @@ -0,0 +1 @@ +../../_attributes/ \ No newline at end of file diff --git a/networking/networking_operators/dpf_operator/about-dpf-operator.adoc b/networking/networking_operators/dpf_operator/about-dpf-operator.adoc new file mode 100644 index 000000000000..d6bef87927dc --- /dev/null +++ b/networking/networking_operators/dpf_operator/about-dpf-operator.adoc @@ -0,0 +1,43 @@ +:_mod-docs-content-type: ASSEMBLY +[id="about-dpf-operator"] += About the NVIDIA DPF Operator +include::_attributes/common-attributes.adoc[] +:context: about-dpf-operator + +toc::[] + +[role="_abstract"] +The NVIDIA DOCA Platform Framework (DPF) Operator enables hardware-accelerated networking on {product-title} by offloading OVN-Kubernetes data plane operations to NVIDIA BlueField-3 Data Processing Units (DPUs). The DPF deployment creates a dual-cluster topology consisting of a management cluster running on x86 servers and a hosted DPU cluster running on BlueField-3 DPUs. + +[NOTE] +==== +In the current release, Host-Based Networking (HBN) with OVN-Kubernetes is the only supported DPU service combination. + +The NVIDIA DPF Operator is distinct from the Red{nbsp}Hat DPU Operator. The Red{nbsp}Hat DPU Operator manages supported non-NVIDIA DPU devices. NVIDIA BlueField-3 deployments use the DPF Operator and related components. +==== + +// Architecture overview +include::modules/nw-dpf-architecture-overview.adoc[leveloffset=+1] + +// Component placement +include::modules/nw-dpf-component-placement.adoc[leveloffset=+1] + +// Deployment flow overview +include::modules/nw-dpf-deployment-flow-overview.adoc[leveloffset=+1] + +// Hardware requirements +include::modules/nw-dpf-hardware-requirements.adoc[leveloffset=+1] + +// Network requirements +include::modules/nw-dpf-network-requirements.adoc[leveloffset=+1] + +// Software requirements +include::modules/nw-dpf-software-requirements.adoc[leveloffset=+1] + +[role="_additional-resources"] +.Additional resources + +* xref:../dpu-operator/dpu-operator.adoc#dpu-operator[DPU Operator] +* link:https://github.com/NVIDIA/doca-platform/tree/public-release-v26.4[DOCA Platform Framework (DPF) GitHub] +* link:https://github.com/NVIDIA/doca-platform/blob/public-release-v26.4/docs/public/getting-started/dpf-host-trusted.md[Get Started with DPF Host Trusted] +* link:https://github.com/NVIDIA/doca-platform/blob/public-release-v26.4/docs/public/user-guides/host-trusted/use-cases/hbn-ovnk/README.md[DPF OVN-Kubernetes with Host-Based Networking User Guide] diff --git a/networking/networking_operators/dpf_operator/dpf-environment-setup.adoc b/networking/networking_operators/dpf_operator/dpf-environment-setup.adoc new file mode 100644 index 000000000000..09c54c37a28a --- /dev/null +++ b/networking/networking_operators/dpf_operator/dpf-environment-setup.adoc @@ -0,0 +1,55 @@ +:_mod-docs-content-type: ASSEMBLY +[id="dpf-environment-setup"] += Setting up the environment for DPF +include::_attributes/common-attributes.adoc[] +:context: dpf-environment-setup + +toc::[] + +[role="_abstract"] +Before installing the NVIDIA DPF Operator, you must set up the management cluster, configure worker nodes, and install and configure the required operators. + +// Management cluster setup +include::modules/nw-dpf-management-cluster-setup.adoc[leveloffset=+1] + +// Configuring worker nodes +include::modules/nw-dpf-configuring-worker-nodes.adoc[leveloffset=+1] + +// Worker MachineConfig +include::modules/nw-dpf-worker-machineconfig.adoc[leveloffset=+2] + +// Create DPF namespace +include::modules/nw-dpf-creating-dpf-namespace.adoc[leveloffset=+2] + +// Installing the required operators +include::modules/nw-dpf-installing-required-operators.adoc[leveloffset=+1] + +// Cert Manager Operator +include::modules/nw-dpf-installing-cert-manager.adoc[leveloffset=+2] + +// MetalLB Operator +include::modules/nw-dpf-installing-metallb-operator.adoc[leveloffset=+2] + +// GitOps Operator +include::modules/nw-dpf-installing-gitops-operator.adoc[leveloffset=+2] + +// Maintenance Operator +include::modules/nw-dpf-installing-maintenance-operator.adoc[leveloffset=+2] + +// Configuring the required operators +include::modules/nw-dpf-configuring-required-operators.adoc[leveloffset=+1] + +// MultiCluster Engine +include::modules/nw-dpf-configuring-mce.adoc[leveloffset=+2] + +// Node Feature Discovery +include::modules/nw-dpf-configuring-nfd.adoc[leveloffset=+2] + +// MetalLB +include::modules/nw-dpf-configuring-metallb.adoc[leveloffset=+2] + +// GitOps +include::modules/nw-dpf-configuring-gitops.adoc[leveloffset=+2] + +// Cluster Network Operator +include::modules/nw-dpf-configuring-cno.adoc[leveloffset=+2] diff --git a/networking/networking_operators/dpf_operator/dpf-hosted-cluster-provisioning.adoc b/networking/networking_operators/dpf_operator/dpf-hosted-cluster-provisioning.adoc new file mode 100644 index 000000000000..2eaf712e1207 --- /dev/null +++ b/networking/networking_operators/dpf_operator/dpf-hosted-cluster-provisioning.adoc @@ -0,0 +1,31 @@ +:_mod-docs-content-type: ASSEMBLY +[id="dpf-hosted-cluster-provisioning"] += Provisioning the DPU hosted cluster +include::_attributes/common-attributes.adoc[] +:context: dpf-hosted-cluster-provisioning + +toc::[] + +[role="_abstract"] +The `dpf-hcp-provisioner-operator` automates the creation and lifecycle management of a hosted control planes cluster for DPU nodes. This operator abstracts the complexity of setting up a hosted control plane for the DPU cluster. + +// Hosted cluster overview +include::modules/nw-dpf-hosted-cluster-overview.adoc[leveloffset=+1] + +// Install HCP provisioner +include::modules/nw-dpf-installing-hcp-provisioner.adoc[leveloffset=+1] + +// HCP environment variables +include::modules/nw-dpf-hcp-environment-variables.adoc[leveloffset=+1] + +// Create secrets +include::modules/nw-dpf-creating-hcp-secrets.adoc[leveloffset=+1] + +// Create DPUCluster +include::modules/nw-dpf-creating-dpucluster.adoc[leveloffset=+1] + +// Create DPFHCPProvisioner +include::modules/nw-dpf-creating-dpfhcpprovisioner.adoc[leveloffset=+1] + +// Verify hosted cluster +include::modules/nw-dpf-verifying-hosted-cluster.adoc[leveloffset=+1] diff --git a/networking/networking_operators/dpf_operator/dpf-operator-installation.adoc b/networking/networking_operators/dpf_operator/dpf-operator-installation.adoc new file mode 100644 index 000000000000..6aea3e63ad0a --- /dev/null +++ b/networking/networking_operators/dpf_operator/dpf-operator-installation.adoc @@ -0,0 +1,66 @@ +:_mod-docs-content-type: ASSEMBLY +[id="dpf-operator-installation"] += Installing and configuring the DPF Operator +include::_attributes/common-attributes.adoc[] +:context: dpf-operator-installation + +toc::[] + +[role="_abstract"] +After setting up the environment, install the NVIDIA DPF Operator and create the required DPF resources and DPU services. + +[IMPORTANT] +==== +You must install the DPF Operator before the `dpf-hcp-provisioner-operator` because the provisioner requires DPF CRDs such as `DPUCluster`, `DPUFlavor`, `DPUDeployment`, and `DPFOperatorConfig`. +==== + +// Environment variables reference +include::modules/nw-dpf-environment-variables.adoc[leveloffset=+1] + +// Install DPF Operator +include::modules/nw-dpf-installing-dpf-operator.adoc[leveloffset=+1] + +// Creating DPF resources +include::modules/nw-dpf-creating-dpf-resources-overview.adoc[leveloffset=+1] + +// DPFOperatorConfig +include::modules/nw-dpf-creating-dpfoperatorconfig.adoc[leveloffset=+2] + +// NodeSRIOVDevicePluginConfig +include::modules/nw-dpf-creating-sriov-config.adoc[leveloffset=+2] + +// DPUFlavor +include::modules/nw-dpf-creating-dpuflavor.adoc[leveloffset=+2] + +// BFB +include::modules/nw-dpf-creating-bfb.adoc[leveloffset=+2] + +// DPUDeployment +include::modules/nw-dpf-creating-dpudeployment.adoc[leveloffset=+2] + +// Creating DPU services +include::modules/nw-dpf-creating-dpu-services-overview.adoc[leveloffset=+1] + +// HBN DPU Service +include::modules/nw-dpf-creating-hbn-service.adoc[leveloffset=+2] + +// OVN-Kubernetes DPU Service +include::modules/nw-dpf-creating-ovnk-service.adoc[leveloffset=+2] + +// DOCA Telemetry Service +include::modules/nw-dpf-creating-dts-service.adoc[leveloffset=+2] + +// OVN-Kubernetes credentials +include::modules/nw-dpf-creating-ovnk-credentials.adoc[leveloffset=+2] + +// DPU service interfaces +include::modules/nw-dpf-creating-service-interfaces.adoc[leveloffset=+2] + +// DPU service NADs +include::modules/nw-dpf-creating-service-nads.adoc[leveloffset=+2] + +// DPU service IPAM +include::modules/nw-dpf-creating-service-ipam.adoc[leveloffset=+2] + +// Verify DPU services +include::modules/nw-dpf-verifying-dpu-services.adoc[leveloffset=+1] diff --git a/networking/networking_operators/dpf_operator/dpf-troubleshooting.adoc b/networking/networking_operators/dpf_operator/dpf-troubleshooting.adoc new file mode 100644 index 000000000000..201a4d785b06 --- /dev/null +++ b/networking/networking_operators/dpf_operator/dpf-troubleshooting.adoc @@ -0,0 +1,22 @@ +:_mod-docs-content-type: ASSEMBLY +[id="dpf-troubleshooting"] += Troubleshooting DPF +include::_attributes/common-attributes.adoc[] +:context: dpf-troubleshooting + +toc::[] + +[role="_abstract"] +Use the following information to troubleshoot common issues with the NVIDIA DPF Operator deployment. + +// Troubleshooting DPU provisioning +include::modules/nw-dpf-ts-dpu-provisioning.adoc[leveloffset=+1] + +// Troubleshooting DPU object state +include::modules/nw-dpf-ts-dpu-object-state.adoc[leveloffset=+1] + +// Troubleshooting node readiness +include::modules/nw-dpf-ts-node-readiness.adoc[leveloffset=+1] + +// Unsupported OVN-Kubernetes features +include::modules/nw-dpf-unsupported-ovnk-features.adoc[leveloffset=+1] diff --git a/networking/networking_operators/dpf_operator/dpf-validation-telemetry.adoc b/networking/networking_operators/dpf_operator/dpf-validation-telemetry.adoc new file mode 100644 index 000000000000..c7f3a16720dd --- /dev/null +++ b/networking/networking_operators/dpf_operator/dpf-validation-telemetry.adoc @@ -0,0 +1,34 @@ +:_mod-docs-content-type: ASSEMBLY +[id="dpf-validation-telemetry"] += Validating traffic and configuring telemetry +include::_attributes/common-attributes.adoc[] +:context: dpf-validation-telemetry + +toc::[] + +[role="_abstract"] +After provisioning the DPUs and verifying system readiness, validate end-to-end traffic flow and configure DPU telemetry observability. + +// Validating traffic +include::modules/nw-dpf-validating-traffic-overview.adoc[leveloffset=+1] + +// Deploy traffic test pods +include::modules/nw-dpf-deploying-traffic-test-pods.adoc[leveloffset=+2] + +// Run traffic validation tests +include::modules/nw-dpf-running-traffic-validation.adoc[leveloffset=+2] + +// Configuring DPU telemetry +include::modules/nw-dpf-configuring-dpu-telemetry.adoc[leveloffset=+1] + +// DTS observability overview +include::modules/nw-dpf-dts-observability-overview.adoc[leveloffset=+2] + +// Enable user workload monitoring +include::modules/nw-dpf-enabling-user-workload-monitoring.adoc[leveloffset=+2] + +// Configure DTS ServiceMonitor +include::modules/nw-dpf-configuring-dts-servicemonitor.adoc[leveloffset=+2] + +// View DTS metrics +include::modules/nw-dpf-viewing-dts-metrics.adoc[leveloffset=+2] diff --git a/networking/networking_operators/dpf_operator/dpf-worker-nodes-provisioning.adoc b/networking/networking_operators/dpf_operator/dpf-worker-nodes-provisioning.adoc new file mode 100644 index 000000000000..cfb16bb4cadd --- /dev/null +++ b/networking/networking_operators/dpf_operator/dpf-worker-nodes-provisioning.adoc @@ -0,0 +1,43 @@ +:_mod-docs-content-type: ASSEMBLY +[id="dpf-worker-nodes-provisioning"] += Adding worker nodes and provisioning DPUs +include::_attributes/common-attributes.adoc[] +:context: dpf-worker-nodes-provisioning + +toc::[] + +[role="_abstract"] +After the DPF Operator and the hosted cluster are configured, adjust the OVN-Kubernetes CNI settings, add DPU-equipped worker nodes to the management cluster, and provision the DPUs. + +// Adjusting OVN-Kubernetes CNI settings +include::modules/nw-dpf-adjusting-ovnk-cni-overview.adoc[leveloffset=+1] + +// Enable OVN-K resource injector +include::modules/nw-dpf-enabling-ovnk-resource-injector.adoc[leveloffset=+2] + +// Enable DPU-host mode +include::modules/nw-dpf-enabling-ovnk-dpu-host-mode.adoc[leveloffset=+2] + +// Adding worker nodes +include::modules/nw-dpf-adding-worker-nodes-overview.adoc[leveloffset=+1] + +// Adding workers via Assisted Installer +include::modules/nw-dpf-adding-workers-assisted-installer.adoc[leveloffset=+2] + +// Adding workers via Baremetal Operator +include::modules/nw-dpf-adding-workers-baremetal-operator.adoc[leveloffset=+2] + +// Approve worker CSRs +include::modules/nw-dpf-approving-worker-csrs.adoc[leveloffset=+1] + +// Verify DPU provisioning +include::modules/nw-dpf-verifying-dpu-provisioning.adoc[leveloffset=+1] + +// Configure hosted cluster auth +include::modules/nw-dpf-configuring-hosted-cluster-auth.adoc[leveloffset=+1] + +// Approve DPU CSRs +include::modules/nw-dpf-approving-dpu-csrs.adoc[leveloffset=+1] + +// Verify system readiness +include::modules/nw-dpf-verifying-system-readiness.adoc[leveloffset=+1] diff --git a/networking/networking_operators/dpf_operator/images b/networking/networking_operators/dpf_operator/images new file mode 120000 index 000000000000..847b03ed0541 --- /dev/null +++ b/networking/networking_operators/dpf_operator/images @@ -0,0 +1 @@ +../../images/ \ No newline at end of file diff --git a/networking/networking_operators/dpf_operator/modules b/networking/networking_operators/dpf_operator/modules new file mode 120000 index 000000000000..36719b9de743 --- /dev/null +++ b/networking/networking_operators/dpf_operator/modules @@ -0,0 +1 @@ +../../modules/ \ No newline at end of file diff --git a/networking/networking_operators/dpf_operator/snippets b/networking/networking_operators/dpf_operator/snippets new file mode 120000 index 000000000000..5a3f5add140e --- /dev/null +++ b/networking/networking_operators/dpf_operator/snippets @@ -0,0 +1 @@ +../../snippets/ \ No newline at end of file