diff --git a/README.md b/README.md index f9e62b6..724f4da 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,36 @@ find the following examples of 64 GPU (8 node) runs: - [H100](./mpi-operator/nccl-test-distributed-h100-64-mpijob.yaml) - [H100 with SHARP](./mpi-operator/nccl-test-distributed-h100-64-sharp-mpijob.yaml) - [GB200 NVL72](./mpi-operator/nccl-test-distributed-gb200-nvl72-mpijob.yaml) +- [GB200 NVL72 Multi-rack](./mpi-operator/nccl-test-distributed-gb200-128-multirack-mpijob.yaml) + +#### Generating Custom Manifests + +For custom configurations, use `nccl.nu` to generate manifests dynamically. +Requires [Nushell](https://www.nushell.sh/). + +```bash +# Generate manifest for 64 H100 GPUs +./nccl.nu generate --gpu h100 --scale 64 + +# Generate with SHARP and multi-rack topology +./nccl.nu generate --gpu h100 --scale 128 --sharp --multirack + +# Generate and apply directly to cluster +./nccl.nu generate --gpu a100 --scale 64 --apply + +# Use custom container image +./nccl.nu generate --gpu h100 --scale 64 --image ghcr.io/custom/image:tag +``` + +Available options: +- `--gpu` - GPU type (a100, h100, a40, gb200, rtxp6000_8x) +- `--scale` - Total GPU count (must be divisible by GPUs per node) +- `--multirack` - Enable multi-rack topology spread constraints +- `--noib` - Disable InfiniBand +- `--gdrcopy` - Enable GDRCopy +- `--sharp` - Enable SHARP/COLLNET +- `--apply` - Apply manifest directly to cluster +- `--image` - Override container image #### Running Jobs diff --git a/cue.mod/module.cue b/cue.mod/module.cue new file mode 100644 index 0000000..f243535 --- /dev/null +++ b/cue.mod/module.cue @@ -0,0 +1,9 @@ +module: "github.com/coreweave/nccl-tests" +language: { + version: "v0.15.3" +} +deps: { + "cue.dev/x/k8s.io@v0": { + v: "v0.6.0" + } +} diff --git a/generate.cue b/generate.cue new file mode 100644 index 0000000..b6331e8 --- /dev/null +++ b/generate.cue @@ -0,0 +1,223 @@ +package main + +import ( + "strings" + + corev1 "cue.dev/x/k8s.io/api/core/v1" + metav1 "cue.dev/x/k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/coreweave/nccl-tests/gpus" +) + +// Input parameters (set via -t flag) +gpu: *"a100" | string @tag(gpu) +scale: *64 | int @tag(scale,type=int) +name: *"" | string @tag(name) +multirack: *false | bool @tag(multirack,type=bool) +noib: *false | bool @tag(noib,type=bool) +gdrcopy: *false | bool @tag(gdrcopy,type=bool) +sharp: *false | bool @tag(sharp,type=bool) +image: *"" | string @tag(image) +defaultImage: string @tag(defaultImage) + +// Lookup the GPU config +_gpu: gpus.gpus[gpu] + +// Calculate workers (integer division) +_workers: scale div _gpu.gpusPerNode + +// Resolve image (override takes precedence, then GPU-specific, then default) +_image: defaultImage +if image != "" { + _image: image +} + +// Generate job name (lowercase) +_name: "nccl-test-\(scale)-\(strings.ToLower(_gpu.name))" +if name != "" { + _name: name +} + +// Build env var flags string from mpiEnv list +_gpuEnvFlags: strings.Join([for k, v in _gpu.mpiEnv {"-x \(k)=\(v)"}], " \\\n ") + +// Build gdrcopy env flags +_gdrcopyEnvFlags: string | *"" +if gdrcopy { + _gdrcopyEnvFlags: "-x NCCL_GDRCOPY_ENABLE=1 \\\n -x NCCL_DEBUG=INFO \\\n -x NCCL_DEBUG_SUBSYS=INIT" +} + +// Build sharp env flags (COLLNET + ALGO for H100) +_sharpEnvFlags: string | *"" +if sharp { + if gpu == "h100" { + _sharpEnvFlags: "-x NCCL_COLLNET_ENABLE=1 \\\n -x NCCL_ALGO=COLLNETCHAIN" + } + if gpu != "h100" { + _sharpEnvFlags: "-x NCCL_COLLNET_ENABLE=1" + } +} + +// Combine all extra env flags +_allExtraEnv: [for s in [_gpuEnvFlags, _gdrcopyEnvFlags, _sharpEnvFlags] if s != "" {s}] +_extraEnvStr: strings.Join(_allExtraEnv, " \\\n ") +_extraEnv: string | *"" +if _extraEnvStr != "" { + _extraEnv: " \\\n \(_extraEnvStr)" +} + +// Determine IB HCA setting (eth0 when noib, else ibp) +_ibHca: string | *"ibp" +if noib { + _ibHca: "eth0" +} + +// Build UCX_NET_DEVICES part (only when has ibDevices and not noib and not gdrcopy) +_ucxPart: string +if _gpu.ibDevices != "" && !noib && !gdrcopy { + _ucxPart: " \\\n -x UCX_NET_DEVICES=\(_gpu.ibDevices)" +} +if _gpu.ibDevices == "" || noib || gdrcopy { + _ucxPart: "" +} + +// Build bind-to part (only when not gdrcopy) +_bindPart: string | *" \\\n -bind-to none" +if gdrcopy { + _bindPart: "" +} + +// Build the mpirun command +_mpirunArgs: "mpirun \\\n -np \(scale)\(_bindPart) \\\n -x LD_LIBRARY_PATH \\\n -x NCCL_SOCKET_IFNAME=eth0 \\\n -x NCCL_IB_HCA=\(_ibHca)\(_ucxPart)\(_extraEnv) \\\n /opt/nccl_tests/build/all_reduce_perf -b 512M -e 8G -f 2 -g 1\n" + +// Launcher container spec (typed) +_launcherContainer: corev1.#Container & { + name: "nccl" + image: _image + env: [ + {name: "OMPI_ALLOW_RUN_AS_ROOT", value: "1"}, + {name: "OMPI_ALLOW_RUN_AS_ROOT_CONFIRM", value: "1"}, + ] + command: ["/bin/bash", "-c"] + args: [_mpirunArgs] + resources: { + requests: {cpu: "2", memory: "128Mi"} + limits: {cpu: "2", memory: "128Mi"} + } +} + +// Determine if we should request IB resources (GPU supports IB and noib is not set) +_useIB: _gpu.useIB && !noib + +// Worker volume mounts (always dshm, optionally gdrdrv) +_workerVolumeMounts: [...corev1.#VolumeMount] +if !gdrcopy { + _workerVolumeMounts: [{mountPath: "/dev/shm", name: "dshm"}] +} +if gdrcopy { + _workerVolumeMounts: [ + {mountPath: "/dev/shm", name: "dshm"}, + {mountPath: "/dev/gdrdrv", name: "gdrdrv"}, + ] +} + +// Worker container spec (typed) +_workerContainer: corev1.#Container & { + name: "nccl" + image: _image + resources: { + requests: { + cpu: "\(_gpu.cpu)" + memory: _gpu.memory + "nvidia.com/gpu": "\(_gpu.gpusPerNode)" + if _useIB {"rdma/ib": "1"} + } + limits: { + "nvidia.com/gpu": "\(_gpu.gpusPerNode)" + if _useIB {"rdma/ib": "1"} + memory: _gpu.memory + cpu: "\(_gpu.cpu)" + } + } + volumeMounts: _workerVolumeMounts +} + +// Worker pod affinity (typed) +_workerAffinity: corev1.#Affinity & { + if _gpu.podAffinity { + podAffinity: preferredDuringSchedulingIgnoredDuringExecution: [{ + weight: 100 + podAffinityTerm: { + labelSelector: matchLabels: "metadata.coreweave.cloud/job": "nccl-test" + topologyKey: "topology.kubernetes.io/zone" + } + }] + } + nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: [{ + matchExpressions: [{ + key: _gpu.nodeKey + operator: "In" + values: [_gpu.nodeType] + }] + }] +} + +// Worker volumes (always dshm, optionally gdrdrv) +_workerVolumes: [...corev1.#Volume] +if !gdrcopy { + _workerVolumes: [{name: "dshm", emptyDir: medium: "Memory"}] +} +if gdrcopy { + _workerVolumes: [ + {name: "dshm", emptyDir: medium: "Memory"}, + {name: "gdrdrv", hostPath: path: "/dev/gdrdrv"}, + ] +} + +// Topology spread constraints (typed) +_topologySpreadConstraints: [...corev1.#TopologySpreadConstraint] +if multirack { + _topologySpreadConstraints: [{ + maxSkew: 1 + minDomains: 2 + topologyKey: "topology.kubernetes.io/zone" + whenUnsatisfiable: "DoNotSchedule" + labelSelector: matchLabels: "metadata.coreweave.cloud/job": "nccl-test" + }] +} + +// The MPIJob manifest +manifest: { + apiVersion: "kubeflow.org/v2beta1" + kind: "MPIJob" + metadata: metav1.#ObjectMeta & { + name: _name + } + spec: { + slotsPerWorker: _gpu.gpusPerNode + runPolicy: cleanPodPolicy: "Running" + mpiReplicaSpecs: { + Launcher: { + replicas: 1 + template: { + spec: corev1.#PodSpec & { + containers: [_launcherContainer] + } + } + } + Worker: { + replicas: _workers + template: { + metadata: metav1.#ObjectMeta & { + labels: "metadata.coreweave.cloud/job": "nccl-test" + } + spec: corev1.#PodSpec & { + topologySpreadConstraints: _topologySpreadConstraints + containers: [_workerContainer] + affinity: _workerAffinity + volumes: _workerVolumes + } + } + } + } + } +} diff --git a/gpus/a100.cue b/gpus/a100.cue new file mode 100644 index 0000000..a96fbe4 --- /dev/null +++ b/gpus/a100.cue @@ -0,0 +1,10 @@ +package gpus + +gpus: a100: #GPU & { + name: "A100" + gpusPerNode: 8 + nodeType: "gd-8xa100-i128" + cpu: 112 + memory: "1000Gi" + ibDevices: "ibp0:1,ibp1:1,ibp2:1,ibp3:1" +} diff --git a/gpus/a40.cue b/gpus/a40.cue new file mode 100644 index 0000000..8322e9d --- /dev/null +++ b/gpus/a40.cue @@ -0,0 +1,11 @@ +package gpus + +gpus: a40: #GPU & { + name: "A40" + gpusPerNode: 8 + nodeType: "A40" + nodeKey: "gpu.nvidia.com/model" + cpu: 90 + memory: "400Gi" + ibDevices: "ibp0:1,ibp1:1,ibp2:1,ibp3:1" +} diff --git a/gpus/gb200.cue b/gpus/gb200.cue new file mode 100644 index 0000000..f8b88f9 --- /dev/null +++ b/gpus/gb200.cue @@ -0,0 +1,21 @@ +package gpus + +gpus: gb200: #GPU & { + name: "GB200" + gpusPerNode: 4 + nodeType: "gb200-4x" + cpu: 140 + memory: "900Gi" + useIB: false + podAffinity: true + mpiEnv: { + NVIDIA_IMEX_CHANNELS: "0" + NCCL_NET_GDR_C2C: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_SHM_DISABLE: "0" + UCX_TLS: "tcp" + UCX_NET_DEVICES: "eth0" + OMPI_MCA_coll_hcoll_enable: "0" + } +} diff --git a/gpus/gpus.cue b/gpus/gpus.cue new file mode 100644 index 0000000..09a4d83 --- /dev/null +++ b/gpus/gpus.cue @@ -0,0 +1,14 @@ +package gpus + +import "github.com/coreweave/nccl-tests/schema" + +// Default image passed via tag from nccl.nu (which reads workflow YAML and git SHA) +defaultImage: string @tag(defaultImage) + +// Alias for schema.#GPU with our default image injected. +// Individual GPU configs can optionally override the default image. +#GPU: schema.#GPU & { + image: string | *defaultImage +} + +gpus: [string]: #GPU diff --git a/gpus/h100.cue b/gpus/h100.cue new file mode 100644 index 0000000..e8b4c89 --- /dev/null +++ b/gpus/h100.cue @@ -0,0 +1,11 @@ +package gpus + +gpus: h100: #GPU & { + name: "H100" + gpusPerNode: 8 + nodeType: "gd-8xh100ib-i128" + cpu: 110 + memory: "960Gi" + ibDevices: "ibp0:1,ibp1:1,ibp2:1,ibp3:1,ibp4:1,ibp5:1,ibp6:1,ibp7:1" + mpiEnv: SHARP_COLL_ENABLE_PCI_RELAXED_ORDERING: "1" +} diff --git a/gpus/rtxp6000_8x.cue b/gpus/rtxp6000_8x.cue new file mode 100644 index 0000000..07bbbe9 --- /dev/null +++ b/gpus/rtxp6000_8x.cue @@ -0,0 +1,10 @@ +package gpus + +gpus: rtxp6000_8x: #GPU & { + name: "RTXP6000-8X" + gpusPerNode: 8 + nodeType: "rtxp6000-8x" + cpu: 112 + memory: "960Gi" + useIB: false +} diff --git a/nccl.nu b/nccl.nu new file mode 100755 index 0000000..05cf1b5 --- /dev/null +++ b/nccl.nu @@ -0,0 +1,223 @@ +#!/usr/bin/env nu + +# NCCL Test CLI - Sync cluster GPU types and generate manifests + +# Get default image from workflow file and git SHA +def get-default-image [] { + let workflow = (open .github/workflows/ubuntu-22.yml) + let job = $workflow.jobs.cu129.with + let sha = (git rev-parse --short HEAD) + $"ghcr.io/coreweave/nccl-tests:($job.base-tag)-nccl($job.nccl-version)-($sha)" +} + +# Get image tag arguments for CUE commands +def get-image-tags [] { + ["-t", $"defaultImage=(get-default-image)"] +} + +# Generate CUE config file content from node data using cue eval +def generate_cue_file [node: record] { + let labels = $node.metadata.labels + let cap = $node.status.capacity + + let gpu_count = $labels | get "gpu.nvidia.com/count" | into int + let node_type = $labels | get "node.coreweave.cloud/type" + let cpu = $labels | get "cpu.coreweave.cloud/cores" | into int + let mem_ki = $cap.memory | str replace "Ki" "" | into int + let mem_gi = ($mem_ki / 1048576 | math floor) + let ib_speed = $labels | get -o "ib.coreweave.cloud/speed" | default "0G" + let has_ib = $ib_speed != "0G" + let config_name = $node_type | str replace "-" "_" --all + # Detect IB devices from node labels (pattern: ib.coreweave.cloud/neighbors.current.ibp.device) + let ib_device_labels = ($labels | columns | where { |k| $k =~ 'ib.coreweave.cloud/neighbors.current.ibp\d+\.device' }) + let ib_devices = if ($ib_device_labels | length) > 0 { + $ib_device_labels + | each { |k| $k | parse --regex 'ibp(?P\d+)' | get num.0 | into int } + | sort + | each { |n| $"ibp($n):1" } + | str join "," + } else { + "" + } + + let img_tags = (get-image-tags) + let content = (cue eval ./utils/... -e newGpu + ...$img_tags + -t $"nodeType=($node_type)" + -t $"gpuCount=($gpu_count)" + -t $"cpu=($cpu)" + -t $"memGi=($mem_gi)" + -t $"hasIB=($has_ib)" + -t $"ibDevices=($ib_devices)") + + { + name: $config_name + content: $"package gpus\n\n($content)\n" + } +} + +# Sync cluster GPU types with CUE config +def "main sync" [ + --kubeconfig: string = "", # Path to kubeconfig file +] { + let kubeconfig = if $kubeconfig != "" { $kubeconfig | path expand } else { "" } + let kc = if $kubeconfig != "" { ["--kubeconfig", $kubeconfig] } else { [] } + + # Get GPU types from cluster + print -e "Fetching GPU nodes from cluster..." + let cluster_nodes = (kubectl ...$kc get nodes -l node.coreweave.cloud/class=gpu -o json + | from json + | get items) + + let cluster_gpus = ($cluster_nodes + | each { $in.metadata.labels | get "node.coreweave.cloud/type" } + | uniq) + + # Get GPU configs from CUE + print -e "Reading GPU configs from CUE..." + let img_tags = (get-image-tags) + let cue_gpus_raw = (cue export ./gpus/... -e gpus ...$img_tags | from json) + let cue_gpu_names = ($cue_gpus_raw | columns) + + let cue_gpus = ($cue_gpu_names | each { |name| + let cfg = ($cue_gpus_raw | get $name) + {name: $name, nodeType: $cfg.nodeType} + }) + + let cue_node_types = ($cue_gpus | get nodeType) + + # Build status table + let all_types = ($cluster_gpus | append $cue_node_types | uniq | sort) + + let status = ($all_types | each { |t| + let in_cluster = $t in $cluster_gpus + let in_config = $t in $cue_node_types + let config_name = if $in_config { + ($cue_gpus | where nodeType == $t | get name.0) + } else { "" } + + { + "GPU Type": $t + "Cluster": (if $in_cluster { "✓" } else { "✗" }) + "Config": (if $in_config { $"✓ \(($config_name)\)" } else { "✗" }) + } + }) + + print "" + print ($status | table) + + # Find missing configs + let missing = ($cluster_gpus | where { |t| not ($t in $cue_node_types) }) + + if ($missing | length) > 0 { + print $"\n($missing | length) GPU type\(s\) in cluster without config:\n" + + for gpu_type in $missing { + let create = (input $"Create config for ($gpu_type)? [Y/n] ") + if ($create | str downcase) in ["", "y"] { + # Get node details + let node = (kubectl ...$kc get nodes -l $"node.coreweave.cloud/type=($gpu_type)" -o json + | from json + | get items.0) + + let cue_file = (generate_cue_file $node) + let file_path = $"gpus/($cue_file.name).cue" + $cue_file.content | save -f $file_path + print $"Created ($file_path)" + } + } + } else { + print "\nAll cluster GPU types have configs ✓" + } +} + +# Generate NCCL test manifest +def "main generate" [ + --kubeconfig: string = "", # Path to kubeconfig file + --gpu: string = "", # GPU type (skip prompt if provided) + --scale: int = 0, # Number of GPUs (skip prompt if provided) + --multirack, # Enable multi-rack topology spread + --noib, # Disable InfiniBand + --gdrcopy, # Enable GDRCopy + --sharp, # Enable SHARP/COLLNET + --apply, # Apply manifest to cluster + --image: string = "", # Override container image +] { + let kubeconfig = if $kubeconfig != "" { $kubeconfig | path expand } else { "" } + + # Get image tags from workflow + let img_tags = (get-image-tags) + + # Get available GPUs from CUE + let gpus_json = (cue export ./gpus/... -e gpus ...$img_tags | from json) + let gpu_names = ($gpus_json | columns) + + # Select GPU + let selected_gpu = if $gpu != "" { $gpu } else { + $gpu_names | input list "Select GPU type:" + } + + if $selected_gpu not-in $gpu_names { + print -e $"Error: GPU '($selected_gpu)' not found. Available: ($gpu_names | str join ', ')" + exit 1 + } + + let gpu_info = ($gpus_json | get $selected_gpu) + let gpus_per_node = $gpu_info.gpusPerNode + + # Select scale + let selected_scale = if $scale > 0 { $scale } else { + let default = $gpus_per_node * 8 + let scale_input = (input $"Number of GPUs [default: ($default), ($gpus_per_node)/node]: ") + if $scale_input == "" { $default } else { $scale_input | into int } + } + + # Validate scale is divisible by gpusPerNode + if ($selected_scale mod $gpus_per_node) != 0 { + print -e $"Error: Scale ($selected_scale) must be divisible by gpusPerNode ($gpus_per_node)" + exit 1 + } + + let workers = ($selected_scale / $gpus_per_node | into int) + print -e $"Generating: ($selected_gpu) × ($selected_scale) GPUs \(($workers) workers\)\n" + + # Generate manifest using CUE + let mr = if $multirack { ["-t", "multirack=true"] } else { [] } + let noib_flag = if $noib { ["-t", "noib=true"] } else { [] } + let gdrcopy_flag = if $gdrcopy { ["-t", "gdrcopy=true"] } else { [] } + let sharp_flag = if $sharp { ["-t", "sharp=true"] } else { [] } + let img_override = if $image != "" { ["-t", $"image=($image)"] } else { [] } + let manifest = (cue export -e manifest --out yaml -t $"gpu=($selected_gpu)" -t $"scale=($selected_scale)" ...$mr ...$noib_flag ...$gdrcopy_flag ...$sharp_flag ...$img_tags ...$img_override .) + + if $apply { + let kc = if $kubeconfig != "" { ["--kubeconfig", $kubeconfig] } else { [] } + $manifest | kubectl ...$kc apply -f - + } else { + print $manifest + } +} + +def main [] { + print "NCCL Test CLI + +Commands: + sync - Compare cluster GPU types vs CUE configs, create missing + generate - Interactive manifest generation + +Options: + --kubeconfig - Path to kubeconfig file + --noib - Disable InfiniBand + --gdrcopy - Enable GDRCopy + --sharp - Enable SHARP/COLLNET + +Examples: + ./nccl.nu sync --kubeconfig ~/Downloads/kubeconfig + ./nccl.nu generate + ./nccl.nu generate --gpu a100 --scale 64 + ./nccl.nu generate --gpu a100 --scale 64 --noib + ./nccl.nu generate --gpu a100 --scale 64 --gdrcopy + ./nccl.nu generate --gpu a100 --scale 64 --sharp + ./nccl.nu generate --gpu h100 --scale 64 --sharp + ./nccl.nu generate --gpu h100 --scale 128 --apply --kubeconfig ~/kube.conf + ./nccl.nu generate --gpu h100 --scale 64 --image ghcr.io/coreweave/nccl-tests:custom-tag" +} diff --git a/schema/gpu.cue b/schema/gpu.cue new file mode 100644 index 0000000..54e9bb1 --- /dev/null +++ b/schema/gpu.cue @@ -0,0 +1,15 @@ +package schema + +#GPU: { + name: string + image: string + gpusPerNode: int + nodeType: string + nodeKey: string | *"node.coreweave.cloud/type" + cpu: int + memory: string + ibDevices: string | *"" + useIB: bool | *true + mpiEnv: [string]: string + podAffinity: bool | *false +} diff --git a/utils/new_gpu.cue b/utils/new_gpu.cue new file mode 100644 index 0000000..e100ea0 --- /dev/null +++ b/utils/new_gpu.cue @@ -0,0 +1,36 @@ +package utils + +import ( + "strings" + "github.com/coreweave/nccl-tests/schema" +) + +// Input parameters via tags +_nodeType: string @tag(nodeType) +_gpuCount: int @tag(gpuCount,type=int) +_cpu: int @tag(cpu,type=int) +_memGi: int @tag(memGi,type=int) +_hasIB: bool @tag(hasIB,type=bool) +_ibDevices: string @tag(ibDevices) +_defaultImage: string @tag(defaultImage) + +_configName: strings.Replace(_nodeType, "-", "_", -1) + +// The generated config - validated against #GPU schema +_newConfig: schema.#GPU & { + name: strings.ToUpper(_nodeType) + image: _defaultImage + gpusPerNode: _gpuCount + nodeType: _nodeType + cpu: _cpu + memory: "\(_memGi)Gi" + if _hasIB { + ibDevices: _ibDevices + } + if !_hasIB { + useIB: false + } +} + +// Output structure for file generation +newGpu: gpus: (_configName): _newConfig