Skip to content

Commit 75150ce

Browse files
iankouls-awsiankouls-aws
committed
Add hyperpod-cli mnist example
1 parent f9c1b3e commit 75150ce

10 files changed

Lines changed: 189 additions & 0 deletions

File tree

‎Container-Root/hyperpod/deployment/eks/model-training/pytorchjob/mnist/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
./result

‎Container-Root/hyperpod/deployment/eks/model-training/pytorchjob/mnist/job-describe.sh‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/bin/bash
2+
3+
CMD="hyperpod get-job --job-name mnist-cpu"
4+
5+
if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
6+
eval "${CMD}"
7+

‎Container-Root/hyperpod/deployment/eks/model-training/pytorchjob/mnist/job-exec.sh‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/bin/bash
2+
3+
usage(){
4+
echo ""
5+
echo "Usage: $0 [worker_index] [command]"
6+
echo ""
7+
echo " worker_index - 0,1,etc the worker number to exec into"
8+
echo " command - bash command to execute"
9+
echo ""
10+
}
11+
12+
if [ "$1" == "" ]; then
13+
usage
14+
else
15+
16+
if [ "$2" == "" ]; then
17+
num=0
18+
else
19+
num=$1
20+
shift
21+
fi
22+
23+
EXEC=$@
24+
25+
CMD="hyperpod exec --job-name mnist-cpu --pod mnist-cpu-worker-$num $EXEC"
26+
27+
if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
28+
eval "${CMD}"
29+
fi

‎Container-Root/hyperpod/deployment/eks/model-training/pytorchjob/mnist/job-list.sh‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/bin/bash
2+
3+
CMD="hyperpod list-jobs"
4+
5+
if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
6+
eval "${CMD}"
7+

‎Container-Root/hyperpod/deployment/eks/model-training/pytorchjob/mnist/job-log.sh‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/bin/bash
2+
3+
if [ "$1" == "" ]; then
4+
num=0
5+
else
6+
num=$1
7+
fi
8+
9+
CMD="hyperpod get-log --job-name mnist-cpu --pod mnist-cpu-worker-$num"
10+
11+
if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
12+
eval "${CMD}"
13+

‎Container-Root/hyperpod/deployment/eks/model-training/pytorchjob/mnist/job-pods.sh‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/bin/bash
2+
3+
CMD="hyperpod list-pods --job-name mnist-cpu"
4+
5+
if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
6+
eval "${CMD}"
7+

‎Container-Root/hyperpod/deployment/eks/model-training/pytorchjob/mnist/job-start.sh‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/bin/bash
2+
3+
CMD="hyperpod start-job --config-file ./mnist-cpu.yaml"
4+
5+
if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
6+
eval "${CMD}"
7+

‎Container-Root/hyperpod/deployment/eks/model-training/pytorchjob/mnist/job-stop.sh‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/bin/bash
2+
3+
CMD="hyperpod cancel-job --job-name mnist-cpu"
4+
5+
if [ ! "$VERBOSE" == "false" ]; then echo -e "\n${CMD}\n"; fi
6+
eval "${CMD}"
7+

‎Container-Root/hyperpod/deployment/eks/model-training/pytorchjob/mnist/mnist-cpu.yaml‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License"). You
4+
# may not use this file except in compliance with the License. A copy of
5+
# the License is located at
6+
#
7+
# http://aws.amazon.com/apache2.0/
8+
#
9+
# or in the "license" file accompanying this file. This file is
10+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11+
# ANY KIND, either express or implied. See the License for the specific
12+
# language governing permissions and limitations under the License.
13+
14+
# Hydra configurations. SageMaker Hyperpod CLI use Hydra to manage start-job YAML files.
15+
defaults:
16+
- override hydra/job_logging: stdout
17+
18+
hydra:
19+
run:
20+
dir: .
21+
output_subdir: null
22+
23+
training_cfg:
24+
# entry_script: Required. Path to the entry script of training/fine-tuning, this path should be inside container.
25+
# Mapping to '--entry-script' argument in 'start-job' command.
26+
entry_script: /opt/pytorch-mnist/mnist.py
27+
# script_args: Optional. List of script arguments. Mapping to '--script-args' argument in 'start-job' command.
28+
# Example of usage:
29+
# script_args:
30+
# - --max_context_width: 4096
31+
# - --num_layers: 32
32+
script_args: ["--epochs: 100"]
33+
run:
34+
# name: Required. Current Training Job name. Mapping to '--job-name' argument in 'start-job' command.
35+
name: mnist-cpu
36+
# nodes: Required. Number of nodes to use for current training. Mapping to '--node-count' argument in 'start-job' command.
37+
nodes: 2
38+
# ntasks_per_node: Optional. Number of devices to use per node.
39+
# For CPU instances, default value will be 8; For GPU or TRN instances, default value
40+
# will be the accelerator cores number on the instance. Mapping to '--tasks-per-node' argument in 'start-job' command.
41+
ntasks_per_node: 1
42+
cluster:
43+
# cluster_type: Required. Currently, only support k8s cluster type.
44+
cluster_type: k8s
45+
# instance_type: Required. SageMaker Hyperpod supported instance type only.
46+
# Mapping to '--instance-type' argument in 'start-job' command.
47+
instance_type: ml.m5.2xlarge
48+
# cluster_config: Required. Fields related to cluster configuration for each Training job run.
49+
cluster_config:
50+
# annotations: Optional. The annotations attached to the Training job.
51+
# To use SageMaker Hyperpod Job Auto Resume feature, set annotations like following example:
52+
# annotations:
53+
# sagemaker.amazonaws.com/enable-job-auto-resume: True
54+
# sagemaker.amazonaws.com/job-max-retry-count: 1
55+
annotations: null
56+
# service_account_name: Optional. The name of service account associated with the namespace.
57+
# Mapping to '--service-account-name' argument in 'start-job' command.
58+
service_account_name: null
59+
# persistent_volume_claims: Optional. The persistent volume claims, usually used to mount FSx.
60+
# Mapping to '--persistent-volume-claims' argument in 'start-job' command.
61+
persistent_volume_claims: null
62+
# namespace: Optional. The namespace to submit job. If not specify, Training job will submit to
63+
# the current namespace from Kubernetes context.
64+
# Mapping to '--namespace' argument in 'start-job' command.
65+
namespace: null
66+
# custom_labels: Optional. Used to specify the name of the queue, which is created by the cluster admin users.
67+
# custom_labels:
68+
# kueue.x-k8s.io/queue-name: low-priority-queue2
69+
custom_labels: null
70+
# priority_class_name: Optional. The priority for the job, which is created by the cluster admin users.
71+
# Mapping to '--priority' argument in 'start-job' command.
72+
priority_class_name: null
73+
# volumes: Optional. Used to mount temp path to container. Mapping to '--volumes' argument in 'start-job' command.
74+
# Example of usage:
75+
# volumes:
76+
# - volumeName: v1
77+
# hostPath: /data
78+
# mountPath: /data
79+
volumes: null
80+
# labal_selector: Optional. Defines Kubernetes node affinity to select nodes with labels. Following
81+
# config will choose SageMaker HyperPod health labels and prefer nodes with SageMaker Hyperpod burn-in
82+
# test passed label. Mapping to '--label-selector' argument in 'start-job' command.
83+
label_selector:
84+
required:
85+
sagemaker.amazonaws.com/node-health-status:
86+
- Schedulable
87+
preferred:
88+
sagemaker.amazonaws.com/deep-health-check-status:
89+
- Passed
90+
weights:
91+
- 100
92+
# pullPolicy: Required. Kubernetes PyTorchJob pull policy to pull container, can be Always, IfNotPresent and Never.
93+
# Mapping to '--pull-policy' argument in 'start-job' command.
94+
pullPolicy: IfNotPresent
95+
# restartPolicy: Required. Kubernetes PyTorchJob restart policy. Can be OnFailure, Always or Never.
96+
# To use SageMaker Hyperpod AutoResume functionality, please set it to OnFailure.
97+
# Mapping to '--restart-policy' argument in 'start-job' command.
98+
restartPolicy: OnFailure
99+
100+
# base_results_dir: Optional. Location to store the results, checkpoints and logs.
101+
# Mapping to '--results-dir' argument in 'start-job' command.
102+
base_results_dir: ./result
103+
# container: Required. Docker image to be used for Training Job
104+
# Mapping to '--image' argument in 'start-job' command.
105+
container: docker.io/kubeflowkatib/pytorch-mnist-cpu:v1beta1-bc09cfd
106+
107+
# env_vars: Optional. Environment variables passed to the training job.
108+
# Mapping to '--environment' argument in 'start-job' command.
109+
env_vars:
110+
NCCL_DEBUG: INFO # Logging level for NCCL. Set to "INFO" for debug information

‎Container-Root/hyperpod/setup/eks/install-kubeps1.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ alias gu='nvtop.sh'
5252
alias gpu-util='nvtop.sh'
5353
alias nu='neurontop.sh'
5454
alias neuron-util='neurontop.sh'
55+
alias hp='hyperpod'
5556
5657
if [ -f ~/.kubeon ]; then
5758
source ~/kube-ps1.sh

0 commit comments

Comments
 (0)